From 64b037d0ade460abb8abddbdaa51e79ed2c2d8bd Mon Sep 17 00:00:00 2001 From: Adam Spofford <93943719+adamspofford-dfinity@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:40:00 -0800 Subject: [PATCH 1/4] feat: Low memory threshold (#4052) --- CHANGELOG.md | 6 ++ Cargo.lock | 96 ++++++++++++++----- Cargo.toml | 6 +- docs/cli-reference/dfx-canister.mdx | 2 + docs/dfx-json-schema.json | 16 +++- .../src/e2e_project_backend/src/lib.rs | 5 + e2e/tests-dfx/update_settings.bash | 17 ++++ src/dfx-core/src/config/model/dfinity.rs | 28 ++++++ src/dfx-core/src/error/dfx_config.rs | 6 ++ .../rust/src/__backend_name__/Cargo.toml | 4 +- src/dfx/src/commands/canister/call.rs | 1 + src/dfx/src/commands/canister/create.rs | 30 +++++- src/dfx/src/commands/canister/delete.rs | 1 + src/dfx/src/commands/canister/status.rs | 31 +++++- .../src/commands/canister/update_settings.rs | 26 ++++- src/dfx/src/lib/ic_attributes/mod.rs | 34 +++++++ src/dfx/src/lib/migrate.rs | 1 + .../operations/canister/create_canister.rs | 1 + .../operations/canister/deploy_canisters.rs | 1 + src/dfx/src/lib/operations/cmc.rs | 1 + 20 files changed, 280 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ad5db1bd..231c12ea15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # UNRELEASED +### feat: `dfx canister [create|update-settings] --wasm-memory-threshold` + +This adds support for the WASM memory threshold, used in conjunction with `--wasm-memory-limit`. +When the remaining memory until the limit falls below the threshold, the canister's +`on_low_wasm_memory` handler is run. + ### fix: `dfx deploy --by-proposal` no longer sends chunk data in ProposeCommitBatch Recently we made `dfx deploy` include some chunk data in CommitBatch, in order to streamline diff --git a/Cargo.lock b/Cargo.lock index ed6c78d48d..34d63ca273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,6 +303,12 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "argon2" version = "0.4.1" @@ -351,6 +357,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + [[package]] name = "async-io" version = "1.13.0" @@ -402,6 +419,15 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "async-watch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a078faf4e27c0c6cc0efb20e5da59dcccc04968ebf2801d8e0b2195124cdcdb2" +dependencies = [ + "event-listener 2.5.3", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1582,7 +1608,7 @@ dependencies = [ "ic-asset", "ic-cdk", "ic-identity-hsm", - "ic-utils 0.39.0", + "ic-utils 0.39.2", "ic-wasm", "icrc-ledger-types", "idl2json", @@ -1657,7 +1683,7 @@ dependencies = [ "humantime-serde", "ic-agent", "ic-identity-hsm", - "ic-utils 0.39.0", + "ic-utils 0.39.2", "itertools 0.10.5", "k256 0.11.6", "keyring", @@ -2732,12 +2758,14 @@ dependencies = [ [[package]] name = "ic-agent" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158138fcb769fe6288e63d5db221c904e472cfb7d376aba13a38c060f2984e63" +version = "0.39.2" +source = "git+https://github.com/dfinity/agent-rs?rev=0a51f2a65dde7d9e1790c378bd60e1768e3be257#0a51f2a65dde7d9e1790c378bd60e1768e3be257" dependencies = [ + "arc-swap", + "async-channel", "async-lock 3.4.0", "async-trait", + "async-watch", "backoff", "cached 0.52.0", "candid", @@ -2749,8 +2777,8 @@ dependencies = [ "hex", "http 1.2.0", "http-body 1.0.1", - "ic-certification 2.6.0", - "ic-transport-types 0.39.1", + "ic-certification 3.0.2", + "ic-transport-types 0.39.2", "ic-verify-bls-signature", "k256 0.13.4", "leb128", @@ -2767,7 +2795,8 @@ dependencies = [ "serde_repr", "sha2 0.10.8", "simple_asn1", - "thiserror 1.0.69", + "stop-token", + "thiserror 2.0.6", "time", "tokio", "tower-service", @@ -2789,7 +2818,7 @@ dependencies = [ "globset", "hex", "ic-agent", - "ic-utils 0.39.0", + "ic-utils 0.39.2", "itertools 0.10.5", "json5", "mime", @@ -2933,6 +2962,18 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "ic-certification" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eae40f26fcac9c141cad54d9aa5f423efffde78ac371057c53d275ebbcad443" +dependencies = [ + "hex", + "serde", + "serde_bytes", + "sha2 0.10.8", +] + [[package]] name = "ic-certification-testing" version = "2.3.0" @@ -3176,16 +3217,15 @@ dependencies = [ [[package]] name = "ic-identity-hsm" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8722411845f0a4b2c526b906049d45f87c62c78ddf1c38ecdc03ecbd34ffc1d6" +version = "0.39.2" +source = "git+https://github.com/dfinity/agent-rs?rev=0a51f2a65dde7d9e1790c378bd60e1768e3be257#0a51f2a65dde7d9e1790c378bd60e1768e3be257" dependencies = [ "hex", "ic-agent", "pkcs11", "sha2 0.10.8", "simple_asn1", - "thiserror 1.0.69", + "thiserror 2.0.6", ] [[package]] @@ -3294,20 +3334,19 @@ dependencies = [ [[package]] name = "ic-transport-types" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d8789a5c176bb1b925fa58ca97c651a3995d504e76101e93d2a17f558bdcf66" +version = "0.39.2" +source = "git+https://github.com/dfinity/agent-rs?rev=0a51f2a65dde7d9e1790c378bd60e1768e3be257#0a51f2a65dde7d9e1790c378bd60e1768e3be257" dependencies = [ "candid", "hex", - "ic-certification 2.6.0", + "ic-certification 3.0.2", "leb128", "serde", "serde_bytes", "serde_cbor", "serde_repr", "sha2 0.10.8", - "thiserror 1.0.69", + "thiserror 2.0.6", ] [[package]] @@ -3365,9 +3404,8 @@ dependencies = [ [[package]] name = "ic-utils" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1da4a68c45146018b8496c157ad94126b9c202ab4400c6c0a9030c1ef0f0ba" +version = "0.39.2" +source = "git+https://github.com/dfinity/agent-rs?rev=0a51f2a65dde7d9e1790c378bd60e1768e3be257#0a51f2a65dde7d9e1790c378bd60e1768e3be257" dependencies = [ "async-trait", "candid", @@ -3380,7 +3418,7 @@ dependencies = [ "sha2 0.10.8", "strum 0.26.3", "strum_macros 0.26.4", - "thiserror 1.0.69", + "thiserror 2.0.6", "time", "tokio", ] @@ -3617,7 +3655,7 @@ dependencies = [ "humantime", "ic-agent", "ic-asset", - "ic-utils 0.39.0", + "ic-utils 0.39.2", "libflate 1.4.0", "num-traits", "pem 1.1.1", @@ -6081,6 +6119,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stop-token" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b" +dependencies = [ + "async-channel", + "cfg-if", + "futures-core", + "pin-project-lite", +] + [[package]] name = "string_cache" version = "0.8.7" diff --git a/Cargo.toml b/Cargo.toml index 6b3e137c27..58bbde2814 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,11 +22,11 @@ license = "Apache-2.0" candid = "0.10.11" candid_parser = "0.1.4" dfx-core = { path = "src/dfx-core", version = "0.1.0" } -ic-agent = "0.39" +ic-agent = { version = "0.39", git = "https://github.com/dfinity/agent-rs", rev = "0a51f2a65dde7d9e1790c378bd60e1768e3be257" } ic-asset = { path = "src/canisters/frontend/ic-asset", version = "0.21.0" } ic-cdk = "0.13.1" -ic-identity-hsm = "0.39" -ic-utils = "0.39" +ic-identity-hsm = { version = "0.39", git = "https://github.com/dfinity/agent-rs", rev = "0a51f2a65dde7d9e1790c378bd60e1768e3be257" } +ic-utils = { version = "0.39", git = "https://github.com/dfinity/agent-rs", rev = "0a51f2a65dde7d9e1790c378bd60e1768e3be257" } aes-gcm = "0.10.3" anyhow = "1.0.56" diff --git a/docs/cli-reference/dfx-canister.mdx b/docs/cli-reference/dfx-canister.mdx index 5b6f7d3b8d..301d26ec93 100644 --- a/docs/cli-reference/dfx-canister.mdx +++ b/docs/cli-reference/dfx-canister.mdx @@ -280,6 +280,7 @@ You can use the following options with the `dfx canister create` command. | `--memory-allocation ` | Specifies how much memory the canister is allowed to use in total. This should be a value in the range [0..12 GiB]. A setting of 0 means the canister will have access to memory on a “best-effort” basis: It will only be charged for the memory it uses, but at any point in time may stop running if it tries to allocate more memory when there isn’t space available on the subnet. | | `--reserved-cycles-limit ` | Specifies the upper limit for the canister's reserved cycles. | | `--wasm-memory-limit ` | Specifies a soft upper limit for the canister's heap memory. | +| `--wasm-memory-threshold ` | Specifies a threshold remaining amount of memory before the canister's low-memory hook runs. | | `--log-viewer ` | Specifies the principal as an allowed viewers. Can be specified more than once. Cannot be used with `--log-visibility`. | | `--log-visibility ` | Specifies who can read the canister's logs: "controllers" or "public". For custom allowed viewers, use `--log-viewer`. | | `--no-wallet` | Performs the call with the user Identity as the Sender of messages. Bypasses the Wallet canister. Enabled by default. | @@ -1150,6 +1151,7 @@ You can specify the following options for the `dfx canister update-settings` com | `--remove-controller ` | Removes a principal from the list of controllers of the canister. | | `--remove-log-viewer ` | Removes a principal from the list of log viewers of the canister. Can be specified more than once to remove multiple log viewers. | | `--freezing-threshold ` | Set the [freezing threshold](https://internetcomputer.org/docs/current/references/ic-interface-spec/#ic-create_canister) in seconds for a canister. This should be a value in the range [0..2^64^-1]. Very long thresholds require the `--confirm-very-long-freezing-threshold` option. | +| `--wasm-memory-threshold ` | Specifies a threshold remaining amount of memory before the canister's low-memory hook runs. | | `-y`, `--yes` | Skips yes/no checks by answering 'yes'. Such checks can result in loss of control, so this is not recommended outside of CI. | ### Arguments diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index aa98486369..b49c3a3b6a 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -447,7 +447,8 @@ "log_visibility": null, "memory_allocation": null, "reserved_cycles_limit": null, - "wasm_memory_limit": null + "wasm_memory_limit": null, + "wasm_memory_threshold": null }, "allOf": [ { @@ -1020,6 +1021,19 @@ "type": "null" } ] + }, + "wasm_memory_threshold": { + "title": "Wasm Memory Threshold", + "description": "Specifies a threshold (in bytes) on the Wasm memory usage of the canister, as a distance from `wasm_memory_limit`.\n\nWhen the remaining memory before the limit drops below this threshold, its `on_low_wasm_memory` hook will be invoked. This enables it to self-optimize, or raise an alert, or otherwise attempt to prevent itself from reaching `wasm_memory_limit`.\n\nMust be a number of bytes between 0 and 2^48 (i.e. 256 TiB), inclusive. Can be specified as an integer, or as an SI unit string (e.g. \"4KB\", \"2 MiB\")", + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/Byte" + }, + { + "type": "null" + } + ] } } }, diff --git a/e2e/assets/allocate_memory/src/e2e_project_backend/src/lib.rs b/e2e/assets/allocate_memory/src/e2e_project_backend/src/lib.rs index 7ea906db17..3d9a93c679 100644 --- a/e2e/assets/allocate_memory/src/e2e_project_backend/src/lib.rs +++ b/e2e/assets/allocate_memory/src/e2e_project_backend/src/lib.rs @@ -7,3 +7,8 @@ fn greet(s: String) -> String { fn greet_update(s: String) -> String { format!("Hello, {s}!") } + +#[ic_cdk::on_low_wasm_memory] +fn on_low_wasm_memory() { + ic_cdk::println!("Low memory!"); +} diff --git a/e2e/tests-dfx/update_settings.bash b/e2e/tests-dfx/update_settings.bash index 74d668cf61..c96b82e663 100644 --- a/e2e/tests-dfx/update_settings.bash +++ b/e2e/tests-dfx/update_settings.bash @@ -61,6 +61,23 @@ teardown() { assert_contains "Canister exceeded its current Wasm memory limit of 8 bytes" } +@test "set wasm memory threshold" { + dfx_new_rust + install_asset allocate_memory + dfx_start + assert_command dfx canister create e2e_project_backend --no-wallet --wasm-memory-threshold 2MiB --wasm-memory-limit 2MiB + assert_command dfx deploy e2e_project_backend + assert_command dfx canister status e2e_project_backend + assert_contains "Wasm memory threshold: 2_097_152 Bytes" + assert_command dfx canister update-settings e2e_project_backend --wasm-memory-threshold 1MiB + assert_command dfx canister status e2e_project_backend + assert_contains "Wasm memory threshold: 1_048_576 Bytes" + assert_command dfx canister call e2e_project_backend greet_update '("alice")' + sleep 1 + assert_command dfx canister logs e2e_project_backend + assert_contains "Low memory!" +} + @test "set log visibility" { dfx_new dfx_start diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index e26edada97..e05b1ecfd5 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -16,11 +16,13 @@ use crate::error::dfx_config::GetRemoteCanisterIdError::GetRemoteCanisterIdFaile use crate::error::dfx_config::GetReservedCyclesLimitError::GetReservedCyclesLimitFailed; use crate::error::dfx_config::GetSpecifiedIdError::GetSpecifiedIdFailed; use crate::error::dfx_config::GetWasmMemoryLimitError::GetWasmMemoryLimitFailed; +use crate::error::dfx_config::GetWasmMemoryThresholdError::GetWasmMemoryThresholdFailed; use crate::error::dfx_config::{ AddDependenciesError, GetCanisterConfigError, GetCanisterNamesWithDependenciesError, GetComputeAllocationError, GetFreezingThresholdError, GetLogVisibilityError, GetMemoryAllocationError, GetPullCanistersError, GetRemoteCanisterIdError, GetReservedCyclesLimitError, GetSpecifiedIdError, GetWasmMemoryLimitError, + GetWasmMemoryThresholdError, }; use crate::error::fs::CanonicalizePathError; use crate::error::load_dfx_config::LoadDfxConfigError; @@ -477,6 +479,21 @@ pub struct InitializationValues { #[schemars(with = "Option")] pub wasm_memory_limit: Option, + /// # Wasm Memory Threshold + /// + /// Specifies a threshold (in bytes) on the Wasm memory usage of the canister, + /// as a distance from `wasm_memory_limit`. + /// + /// When the remaining memory before the limit drops below this threshold, its + /// `on_low_wasm_memory` hook will be invoked. This enables it to self-optimize, + /// or raise an alert, or otherwise attempt to prevent itself from reaching + /// `wasm_memory_limit`. + /// + /// Must be a number of bytes between 0 and 2^48 (i.e. 256 TiB), inclusive. + /// Can be specified as an integer, or as an SI unit string (e.g. "4KB", "2 MiB") + #[schemars(with = "Option")] + pub wasm_memory_threshold: Option, + /// # Log Visibility /// Specifies who is allowed to read the canister's logs. /// @@ -1004,6 +1021,17 @@ impl ConfigInterface { .wasm_memory_limit) } + pub fn get_wasm_memory_threshold( + &self, + canister_name: &str, + ) -> Result, GetWasmMemoryThresholdError> { + Ok(self + .get_canister_config(canister_name) + .map_err(|e| GetWasmMemoryThresholdFailed(canister_name.to_string(), e))? + .initialization_values + .wasm_memory_threshold) + } + pub fn get_log_visibility( &self, canister_name: &str, diff --git a/src/dfx-core/src/error/dfx_config.rs b/src/dfx-core/src/error/dfx_config.rs index fa2e59c051..dabd5aea76 100644 --- a/src/dfx-core/src/error/dfx_config.rs +++ b/src/dfx-core/src/error/dfx_config.rs @@ -58,6 +58,12 @@ pub enum GetWasmMemoryLimitError { GetWasmMemoryLimitFailed(String, #[source] GetCanisterConfigError), } +#[derive(Error, Debug)] +pub enum GetWasmMemoryThresholdError { + #[error("Failed to get Wasm memory threshold for canister '{0}'")] + GetWasmMemoryThresholdFailed(String, #[source] GetCanisterConfigError), +} + #[derive(Error, Debug)] pub enum GetLogVisibilityError { #[error("Failed to get log visibility for canister '{0}'")] diff --git a/src/dfx/assets/project_templates/rust/src/__backend_name__/Cargo.toml b/src/dfx/assets/project_templates/rust/src/__backend_name__/Cargo.toml index 413f5fa468..f0a6ce3715 100644 --- a/src/dfx/assets/project_templates/rust/src/__backend_name__/Cargo.toml +++ b/src/dfx/assets/project_templates/rust/src/__backend_name__/Cargo.toml @@ -10,5 +10,5 @@ crate-type = ["cdylib"] [dependencies] candid = "0.10" -ic-cdk = "0.16" -ic-cdk-timers = "0.10" # Feel free to remove this dependency if you don't need timers +ic-cdk = "0.17" +ic-cdk-timers = "0.11" # Feel free to remove this dependency if you don't need timers diff --git a/src/dfx/src/commands/canister/call.rs b/src/dfx/src/commands/canister/call.rs index 1a740f7307..aecf4d9a56 100644 --- a/src/dfx/src/commands/canister/call.rs +++ b/src/dfx/src/commands/canister/call.rs @@ -145,6 +145,7 @@ pub fn get_effective_canister_id( | MgmtMethod::BitcoinGetUtxos | MgmtMethod::BitcoinSendTransaction | MgmtMethod::BitcoinGetCurrentFeePercentiles + | MgmtMethod::BitcoinGetBlockHeaders | MgmtMethod::EcdsaPublicKey | MgmtMethod::SignWithEcdsa | MgmtMethod::NodeMetricsHistory => Ok(CanisterId::management_canister()), diff --git a/src/dfx/src/commands/canister/create.rs b/src/dfx/src/commands/canister/create.rs index 368ee604f5..a0e5a610c1 100644 --- a/src/dfx/src/commands/canister/create.rs +++ b/src/dfx/src/commands/canister/create.rs @@ -4,7 +4,7 @@ use crate::lib::environment::Environment; use crate::lib::error::{DfxError, DfxResult}; use crate::lib::ic_attributes::{ get_compute_allocation, get_freezing_threshold, get_log_visibility, get_memory_allocation, - get_reserved_cycles_limit, get_wasm_memory_limit, CanisterSettings, + get_reserved_cycles_limit, get_wasm_memory_limit, get_wasm_memory_threshold, CanisterSettings, }; use crate::lib::operations::canister::{create_canister, skip_remote_canister}; use crate::lib::root_key::fetch_root_key_if_needed; @@ -92,6 +92,18 @@ pub struct CanisterCreateOpts { #[arg(long, value_parser = wasm_memory_limit_parser, hide = true)] wasm_memory_limit: Option, + /// Specifies a threshold (in bytes) on the Wasm memory usage of the canister, + /// as a distance from `wasm_memory_limit`. + /// + /// When the remaining memory before the limit drops below this threshold, its + /// `on_low_wasm_memory` hook will be invoked. This enables it to self-optimize, + /// or raise an alert, or otherwise attempt to prevent itself from reaching + /// `wasm_memory_limit`. + /// + /// Must be a number between 0 B and 256 TiB, inclusive. Can include units, e.g. "4KiB". + #[arg(long, value_parser = wasm_memory_limit_parser)] + wasm_memory_threshold: Option, + /// Specifies who is allowed to read the canister's logs. /// Can be either "controllers" or "public". #[arg(long, value_parser = log_visibility_parser, conflicts_with("log_viewer"))] @@ -208,6 +220,12 @@ pub async fn exec( Some(canister_name), ) .with_context(|| format!("Failed to read Wasm memory limit of {canister_name}."))?; + let wasm_memory_threshold = get_wasm_memory_threshold( + opts.wasm_memory_threshold, + Some(config_interface), + Some(canister_name), + ) + .with_context(|| format!("Failed to read Wasm memory threshold of {canister_name}."))?; let log_visibility = get_log_visibility( env, LogVisibilityOpt::from(&opts.log_visibility, &opts.log_viewer).as_ref(), @@ -231,6 +249,7 @@ pub async fn exec( freezing_threshold, reserved_cycles_limit, wasm_memory_limit, + wasm_memory_threshold, log_visibility, }, opts.created_at_time, @@ -285,6 +304,14 @@ pub async fn exec( Some(canister_name), ) .with_context(|| format!("Failed to read Wasm memory limit of {canister_name}."))?; + let wasm_memory_threshold = get_wasm_memory_threshold( + opts.wasm_memory_threshold, + Some(config_interface), + Some(canister_name), + ) + .with_context(|| { + format!("Failed to read Wasm memory threshold of {canister_name}.") + })?; let log_visibility = get_log_visibility( env, LogVisibilityOpt::from(&opts.log_visibility, &opts.log_viewer).as_ref(), @@ -308,6 +335,7 @@ pub async fn exec( freezing_threshold, reserved_cycles_limit, wasm_memory_limit, + wasm_memory_threshold, log_visibility, }, opts.created_at_time, diff --git a/src/dfx/src/commands/canister/delete.rs b/src/dfx/src/commands/canister/delete.rs index 02d3dcb30f..c99c5df90d 100644 --- a/src/dfx/src/commands/canister/delete.rs +++ b/src/dfx/src/commands/canister/delete.rs @@ -189,6 +189,7 @@ async fn delete_canister( freezing_threshold: Some(FreezingThreshold::try_from(0u8).unwrap()), reserved_cycles_limit: None, wasm_memory_limit: None, + wasm_memory_threshold: None, log_visibility: None, }; info!(log, "Setting the controller to identity principal."); diff --git a/src/dfx/src/commands/canister/status.rs b/src/dfx/src/commands/canister/status.rs index ecd8d7773d..9218774219 100644 --- a/src/dfx/src/commands/canister/status.rs +++ b/src/dfx/src/commands/canister/status.rs @@ -52,6 +52,11 @@ async fn canister_status( } else { "Not Set".to_string() }; + let wasm_memory_threshold = if let Some(threshold) = status.settings.wasm_memory_threshold { + format!("{} Bytes", threshold) + } else { + "Not Set".to_string() + }; let log_visibility = match status.settings.log_visibility { LogVisibility::Controllers => "controllers".to_string(), LogVisibility::Public => "public".to_string(), @@ -66,7 +71,27 @@ async fn canister_status( } }; - println!("Canister status call result for {canister}.\nStatus: {status}\nControllers: {controllers}\nMemory allocation: {memory_allocation}\nCompute allocation: {compute_allocation}\nFreezing threshold: {freezing_threshold}\nIdle cycles burned per day: {idle_cycles_burned_per_day}\nMemory Size: {memory_size:?}\nBalance: {balance} Cycles\nReserved: {reserved} Cycles\nReserved cycles limit: {reserved_cycles_limit}\nWasm memory limit: {wasm_memory_limit}\nModule hash: {module_hash}\nNumber of queries: {queries_total}\nInstructions spent in queries: {query_instructions_total}\nTotal query request payload size (bytes): {query_req_payload_total}\nTotal query response payload size (bytes): {query_resp_payload_total}\nLog visibility: {log_visibility}", + println!( + "\ +Canister status call result for {canister}. +Status: {status} +Controllers: {controllers} +Memory allocation: {memory_allocation} +Compute allocation: {compute_allocation} +Freezing threshold: {freezing_threshold} +Idle cycles burned per day: {idle_cycles_burned_per_day} +Memory Size: {memory_size:?} +Balance: {balance} Cycles +Reserved: {reserved} Cycles +Reserved cycles limit: {reserved_cycles_limit} +Wasm memory limit: {wasm_memory_limit} +Wasm memory threshold: {wasm_memory_threshold} +Module hash: {module_hash} +Number of queries: {queries_total} +Instructions spent in queries: {query_instructions_total} +Total query request payload size (bytes): {query_req_payload_total} +Total query response payload size (bytes): {query_resp_payload_total} +Log visibility: {log_visibility}", status = status.status, controllers = controllers.join(" "), memory_allocation = status.settings.memory_allocation, @@ -76,7 +101,9 @@ async fn canister_status( memory_size = status.memory_size, balance = status.cycles, reserved = status.reserved_cycles, - module_hash = status.module_hash.map_or_else(|| "None".to_string(), |v| format!("0x{}", hex::encode(v))), + module_hash = status + .module_hash + .map_or_else(|| "None".to_string(), |v| format!("0x{}", hex::encode(v))), queries_total = status.query_stats.num_calls_total, query_instructions_total = status.query_stats.num_instructions_total, query_req_payload_total = status.query_stats.request_payload_bytes_total, diff --git a/src/dfx/src/commands/canister/update_settings.rs b/src/dfx/src/commands/canister/update_settings.rs index db3a3184eb..1d18cb1555 100644 --- a/src/dfx/src/commands/canister/update_settings.rs +++ b/src/dfx/src/commands/canister/update_settings.rs @@ -4,7 +4,7 @@ use crate::lib::environment::Environment; use crate::lib::error::{DfxError, DfxResult}; use crate::lib::ic_attributes::{ get_compute_allocation, get_freezing_threshold, get_log_visibility, get_memory_allocation, - get_reserved_cycles_limit, get_wasm_memory_limit, CanisterSettings, + get_reserved_cycles_limit, get_wasm_memory_limit, get_wasm_memory_threshold, CanisterSettings, }; use crate::lib::operations::canister::{ get_canister_status, skip_remote_canister, update_settings, @@ -91,6 +91,18 @@ pub struct UpdateSettingsOpts { #[arg(long, value_parser = wasm_memory_limit_parser)] wasm_memory_limit: Option, + /// Specifies a threshold (in bytes) on the Wasm memory usage of the canister, + /// as a distance from `wasm_memory_limit`. + /// + /// When the remaining memory before the limit drops below this threshold, its + /// `on_low_wasm_memory` hook will be invoked. This enables it to self-optimize, + /// or raise an alert, or otherwise attempt to prevent itself from reaching + /// `wasm_memory_limit`. + /// + /// Must be a number between 0 B and 256 TiB, inclusive. Can include units, e.g. "4KiB". + #[arg(long, value_parser = wasm_memory_limit_parser)] + wasm_memory_threshold: Option, + #[command(flatten)] log_visibility_opt: Option, @@ -158,6 +170,8 @@ pub async fn exec( get_reserved_cycles_limit(opts.reserved_cycles_limit, config_interface, canister_name)?; let wasm_memory_limit = get_wasm_memory_limit(opts.wasm_memory_limit, config_interface, canister_name)?; + let wasm_memory_threshold = + get_wasm_memory_threshold(opts.wasm_memory_threshold, config_interface, canister_name)?; let mut current_status: Option = None; if let Some(log_visibility) = &opts.log_visibility_opt { if log_visibility.require_current_settings() { @@ -214,6 +228,7 @@ pub async fn exec( freezing_threshold, reserved_cycles_limit, wasm_memory_limit, + wasm_memory_threshold, log_visibility, }; update_settings(env, canister_id, settings, call_sender).await?; @@ -266,6 +281,14 @@ pub async fn exec( Some(canister_name), ) .with_context(|| format!("Failed to get Wasm memory limit for {canister_name}."))?; + let wasm_memory_threshold = get_wasm_memory_threshold( + opts.wasm_memory_threshold, + Some(config_interface), + Some(canister_name), + ) + .with_context(|| { + format!("Failed to get Wasm memory threshold for {canister_name}.") + })?; let mut current_status: Option = None; if let Some(log_visibility) = &opts.log_visibility_opt { if log_visibility.require_current_settings() { @@ -325,6 +348,7 @@ pub async fn exec( freezing_threshold, reserved_cycles_limit, wasm_memory_limit, + wasm_memory_threshold, log_visibility, }; update_settings(env, canister_id, settings, call_sender).await?; diff --git a/src/dfx/src/lib/ic_attributes/mod.rs b/src/dfx/src/lib/ic_attributes/mod.rs index a7cf379a02..f287132866 100644 --- a/src/dfx/src/lib/ic_attributes/mod.rs +++ b/src/dfx/src/lib/ic_attributes/mod.rs @@ -22,6 +22,7 @@ pub struct CanisterSettings { pub freezing_threshold: Option, pub reserved_cycles_limit: Option, pub wasm_memory_limit: Option, + pub wasm_memory_threshold: Option, pub log_visibility: Option, } @@ -51,6 +52,10 @@ impl From .wasm_memory_limit .map(u64::from) .map(candid::Nat::from), + wasm_memory_threshold: value + .wasm_memory_threshold + .map(u64::from) + .map(candid::Nat::from), log_visibility: value.log_visibility, } } @@ -107,6 +112,14 @@ impl TryFrom, + config_interface: Option<&ConfigInterface>, + canister_name: Option<&str>, +) -> DfxResult> { + let wasm_memory_threshold = match (wasm_memory_threshold, config_interface, canister_name) { + (Some(memory_threshold), _, _) => Some(memory_threshold), + (None, Some(config_interface), Some(canister_name)) => { + config_interface.get_wasm_memory_threshold(canister_name)? + } + _ => None, + }; + wasm_memory_threshold + .map(|arg| { + u64::try_from(arg.get_bytes()) + .map_err(|e| anyhow!(e)) + .and_then(|n| Ok(WasmMemoryLimit::try_from(n)?)) + .context("Wasm memory limit must be between 0 and 2^48 (i.e 256TB), inclusively.") + }) + .transpose() +} pub fn get_log_visibility( env: &dyn Environment, diff --git a/src/dfx/src/lib/migrate.rs b/src/dfx/src/lib/migrate.rs index f9ceda72a4..5a3c584952 100644 --- a/src/dfx/src/lib/migrate.rs +++ b/src/dfx/src/lib/migrate.rs @@ -120,6 +120,7 @@ async fn migrate_canister( memory_allocation: None, reserved_cycles_limit: None, wasm_memory_limit: None, + wasm_memory_threshold: None, log_visibility: None, }, },)), diff --git a/src/dfx/src/lib/operations/canister/create_canister.rs b/src/dfx/src/lib/operations/canister/create_canister.rs index 8c4952afc7..03c1e3928f 100644 --- a/src/dfx/src/lib/operations/canister/create_canister.rs +++ b/src/dfx/src/lib/operations/canister/create_canister.rs @@ -216,6 +216,7 @@ async fn create_with_management_canister( .with_optional_freezing_threshold(settings.freezing_threshold) .with_optional_reserved_cycles_limit(settings.reserved_cycles_limit) .with_optional_wasm_memory_limit(settings.wasm_memory_limit) + .with_optional_wasm_memory_threshold(settings.wasm_memory_threshold) .with_optional_log_visibility(settings.log_visibility) .await; const NEEDS_WALLET: &str = "In order to create a canister on this network, you must use a wallet in order to allocate cycles to the new canister. \ diff --git a/src/dfx/src/lib/operations/canister/deploy_canisters.rs b/src/dfx/src/lib/operations/canister/deploy_canisters.rs index b1bdff2a3a..b3689e7722 100644 --- a/src/dfx/src/lib/operations/canister/deploy_canisters.rs +++ b/src/dfx/src/lib/operations/canister/deploy_canisters.rs @@ -277,6 +277,7 @@ async fn register_canisters( freezing_threshold, reserved_cycles_limit, wasm_memory_limit, + wasm_memory_threshold: None, log_visibility, }, created_at_time, diff --git a/src/dfx/src/lib/operations/cmc.rs b/src/dfx/src/lib/operations/cmc.rs index 947a08649e..9194177eb4 100644 --- a/src/dfx/src/lib/operations/cmc.rs +++ b/src/dfx/src/lib/operations/cmc.rs @@ -75,6 +75,7 @@ pub async fn notify_create( freezing_threshold: None, reserved_cycles_limit: None, wasm_memory_limit: None, + wasm_memory_threshold: None, log_visibility: None, }) }) From 102066c521c1ae07dd0859902c5aaa4343c8523c Mon Sep 17 00:00:00 2001 From: Vincent Zhang <118719397+vincent-dfinity@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:15:07 +0800 Subject: [PATCH 2/4] feat: add pre-install hooks to canisters. (#4055) * Add pre-install hooks to canisters. * Exclude SC2317 to workaround the warnings introduced after shellcheck has been updated to '0.9.0' on the CI. --- .github/workflows/publish-manifest.yml | 3 +- CHANGELOG.md | 4 ++ docs/dfx-json-schema.json | 10 +++ .../dfx.json | 11 +++- .../main.mo | 0 .../postinstall.sh | 2 +- e2e/assets/pre_post_install/preinstall.sh | 3 + e2e/tests-dfx/install.bash | 63 +++++++++++++------ src/dfx-core/src/config/model/dfinity.rs | 6 ++ src/dfx/src/lib/canister_info.rs | 7 +++ .../operations/canister/install_canister.rs | 56 +++++++++++++---- 11 files changed, 131 insertions(+), 34 deletions(-) rename e2e/assets/{post_install => pre_post_install}/dfx.json (57%) rename e2e/assets/{post_install => pre_post_install}/main.mo (100%) rename e2e/assets/{post_install => pre_post_install}/postinstall.sh (77%) create mode 100755 e2e/assets/pre_post_install/preinstall.sh diff --git a/.github/workflows/publish-manifest.yml b/.github/workflows/publish-manifest.yml index 9f51af1611..1f3bf1dbc0 100644 --- a/.github/workflows/publish-manifest.yml +++ b/.github/workflows/publish-manifest.yml @@ -26,7 +26,8 @@ jobs: run: go install mvdan.cc/sh/v3/cmd/shfmt@latest - name: Generate run: | - shellcheck --shell=sh public/install-dfxvm.sh --exclude SC2154,SC2034,SC3003,SC3014,SC3043 + shellcheck -V + shellcheck --shell=sh public/install-dfxvm.sh --exclude SC2154,SC2034,SC3003,SC3014,SC3043,SC2317 ~/go/bin/shfmt -d -p -i 4 -ci -bn -s public/install-dfxvm.sh sed -i "s/@revision@/${GITHUB_SHA}/" public/install-dfxvm.sh mkdir _out diff --git a/CHANGELOG.md b/CHANGELOG.md index 231c12ea15..c54a65832e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,10 @@ Your principal for ICP wallets and decentralized exchanges: ueuar-wxbnk-bdcsr-dn (run `dfx identity get-principal` to display) ``` +### feat: Add pre-install tasks + +Add pre-install tasks, which can be defined by the new `pre-install` key for canister objects in `dfx.json` with a command or list of commands. + ## Dependencies ### Frontend canister diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index b49c3a3b6a..6f6fdc505f 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -496,6 +496,16 @@ } ] }, + "pre_install": { + "title": "Pre-Install Commands", + "description": "One or more commands to run pre canister installation. These commands are executed in the root of the project.", + "default": [], + "allOf": [ + { + "$ref": "#/definitions/SerdeVec_for_String" + } + ] + }, "pullable": { "title": "Pullable", "description": "Defines required properties so that this canister is ready for `dfx deps pull` by other projects.", diff --git a/e2e/assets/post_install/dfx.json b/e2e/assets/pre_post_install/dfx.json similarity index 57% rename from e2e/assets/post_install/dfx.json rename to e2e/assets/pre_post_install/dfx.json index 13f72c635f..e414a26503 100644 --- a/e2e/assets/post_install/dfx.json +++ b/e2e/assets/pre_post_install/dfx.json @@ -1,9 +1,18 @@ { "version": 1, "canisters": { + "preinstall": { + "main": "main.mo", + "pre_install": "echo hello-pre-file" + }, + "preinstall_script": { + "main": "main.mo", + "pre_install": "preinstall.sh", + "dependencies": ["preinstall"] + }, "postinstall": { "main": "main.mo", - "post_install": "echo hello-file" + "post_install": "echo hello-post-file" }, "postinstall_script": { "main": "main.mo", diff --git a/e2e/assets/post_install/main.mo b/e2e/assets/pre_post_install/main.mo similarity index 100% rename from e2e/assets/post_install/main.mo rename to e2e/assets/pre_post_install/main.mo diff --git a/e2e/assets/post_install/postinstall.sh b/e2e/assets/pre_post_install/postinstall.sh similarity index 77% rename from e2e/assets/post_install/postinstall.sh rename to e2e/assets/pre_post_install/postinstall.sh index a20c8dc438..d18ad161fd 100755 --- a/e2e/assets/post_install/postinstall.sh +++ b/e2e/assets/pre_post_install/postinstall.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash echo "working directory of post-install script: '$(pwd)'" -echo hello-script +echo hello-post-script diff --git a/e2e/assets/pre_post_install/preinstall.sh b/e2e/assets/pre_post_install/preinstall.sh new file mode 100755 index 0000000000..386574a9a2 --- /dev/null +++ b/e2e/assets/pre_post_install/preinstall.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo "working directory of pre-install script: '$(pwd)'" +echo hello-pre-script diff --git a/e2e/tests-dfx/install.bash b/e2e/tests-dfx/install.bash index e947e8e22d..1286493d92 100644 --- a/e2e/tests-dfx/install.bash +++ b/e2e/tests-dfx/install.bash @@ -90,26 +90,36 @@ teardown() { assert_command_fail dfx canister install --all --wasm "${archive:?}/wallet/0.10.0/wallet.wasm" } -@test "install runs post-install tasks" { - install_asset post_install +@test "install runs pre-post-install tasks" { + install_asset pre_post_install dfx_start assert_command dfx canister create --all assert_command dfx build + assert_command dfx canister install preinstall + assert_match 'hello-pre-file' + + assert_command dfx canister install preinstall_script + assert_match 'hello-pre-script' + + echo 'return 1' >> preinstall.sh + assert_command_fail dfx canister install preinstall_script --mode upgrade + assert_match 'hello-pre-script' + assert_command dfx canister install postinstall - assert_match 'hello-file' + assert_match 'hello-post-file' assert_command dfx canister install postinstall_script - assert_match 'hello-script' + assert_match 'hello-post-script' echo 'return 1' >> postinstall.sh assert_command_fail dfx canister install postinstall_script --mode upgrade - assert_match 'hello-script' + assert_match 'hello-post-script' } -@test "post-install tasks run in project root" { - install_asset post_install +@test "pre-post-install tasks run in project root" { + install_asset pre_post_install dfx_start assert_command dfx canister create --all @@ -117,42 +127,57 @@ teardown() { cd src/e2e_project_backend + assert_command dfx canister install preinstall_script + assert_match 'hello-pre-script' + assert_match "working directory of pre-install script: '.*/working-dir/e2e_project'" + assert_command dfx canister install postinstall_script - assert_match 'hello-script' + assert_match 'hello-post-script' assert_match "working directory of post-install script: '.*/working-dir/e2e_project'" } -@test "post-install tasks receive environment variables" { - install_asset post_install +@test "pre-post-install tasks receive environment variables" { + install_asset pre_post_install dfx_start + echo "echo hello \$CANISTER_ID" >> preinstall.sh echo "echo hello \$CANISTER_ID" >> postinstall.sh assert_command dfx canister create --all assert_command dfx build - id=$(dfx canister id postinstall_script) + id_pre=$(dfx canister id preinstall_script) + id_post=$(dfx canister id postinstall_script) assert_command dfx canister install --all - assert_match "hello $id" + assert_match "hello $id_post" + assert_command dfx canister install preinstall_script --mode upgrade + assert_match "hello $id_pre" assert_command dfx canister install postinstall_script --mode upgrade - assert_match "hello $id" + assert_match "hello $id_post" assert_command dfx deploy - assert_match "hello $id" + assert_match "hello $id_post" + assert_command dfx deploy preinstall_script + assert_match "hello $id_pre" assert_command dfx deploy postinstall_script - assert_match "hello $id" + assert_match "hello $id_post" } -@test "post-install tasks discover dependencies" { - install_asset post_install +@test "pre-post-install tasks discover dependencies" { + install_asset pre_post_install dfx_start + echo "echo hello \$CANISTER_ID_PREINSTALL" >> preinstall.sh echo "echo hello \$CANISTER_ID_POSTINSTALL" >> postinstall.sh assert_command dfx canister create --all assert_command dfx build - id=$(dfx canister id postinstall) + id_pre=$(dfx canister id preinstall) + id_post=$(dfx canister id postinstall) + + assert_command dfx canister install preinstall_script + assert_match "hello $id_pre" assert_command dfx canister install postinstall_script - assert_match "hello $id" + assert_match "hello $id_post" } @test "can install gzip wasm" { diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index e05b1ecfd5..edee38d405 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -271,6 +271,12 @@ pub struct ConfigCanistersCanister { #[serde(flatten)] pub type_specific: CanisterTypeProperties, + /// # Pre-Install Commands + /// One or more commands to run pre canister installation. + /// These commands are executed in the root of the project. + #[serde(default)] + pub pre_install: SerdeVec, + /// # Post-Install Commands /// One or more commands to run post canister installation. /// These commands are executed in the root of the project. diff --git a/src/dfx/src/lib/canister_info.rs b/src/dfx/src/lib/canister_info.rs index df3dd1e027..95b4808af9 100644 --- a/src/dfx/src/lib/canister_info.rs +++ b/src/dfx/src/lib/canister_info.rs @@ -49,6 +49,7 @@ pub struct CanisterInfo { type_specific: CanisterTypeProperties, dependencies: Vec, + pre_install: Vec, post_install: Vec, main: Option, shrink: Option, @@ -171,6 +172,7 @@ impl CanisterInfo { _ => build_defaults.get_args(), }; + let pre_install = canister_config.pre_install.clone().into_vec(); let post_install = canister_config.post_install.clone().into_vec(); let metadata = CanisterMetadataConfig::new(&canister_config.metadata, &network_name); @@ -190,6 +192,7 @@ impl CanisterInfo { args, type_specific, dependencies, + pre_install, post_install, main: canister_config.main.clone(), shrink: canister_config.shrink, @@ -262,6 +265,10 @@ impl CanisterInfo { &self.packtool } + pub fn get_pre_install(&self) -> &[String] { + &self.pre_install + } + pub fn get_post_install(&self) -> &[String] { &self.post_install } diff --git a/src/dfx/src/lib/operations/canister/install_canister.rs b/src/dfx/src/lib/operations/canister/install_canister.rs index 3591021dbb..1d446fe2fb 100644 --- a/src/dfx/src/lib/operations/canister/install_canister.rs +++ b/src/dfx/src/lib/operations/canister/install_canister.rs @@ -55,6 +55,17 @@ pub async fn install_canister( let log = env.get_logger(); let agent = env.get_agent(); let network = env.get_network_descriptor(); + if !canister_info.get_pre_install().is_empty() { + let config = env.get_config()?; + run_customized_install_tasks( + env, + canister_info, + true, + network, + pool, + env_file.or_else(|| config.as_ref()?.get_config().output_env_file.as_deref()), + )?; + } if !network.is_ic && named_canister::get_ui_canister_id(canister_id_store).is_none() { named_canister::install_ui_canister(env, canister_id_store, None).await?; } @@ -282,9 +293,10 @@ The command line value will be used.", } if !canister_info.get_post_install().is_empty() { let config = env.get_config()?; - run_post_install_tasks( + run_customized_install_tasks( env, canister_info, + false, network, pool, env_file.or_else(|| config.as_ref()?.get_config().output_env_file.as_deref()), @@ -435,14 +447,16 @@ fn check_stable_compatibility( }) } -#[context("Failed to run post-install tasks")] -fn run_post_install_tasks( +#[context("Failed to run {}-install tasks", if is_pre_install { "pre" } else { "post" })] +fn run_customized_install_tasks( env: &dyn Environment, canister: &CanisterInfo, + is_pre_install: bool, network: &NetworkDescriptor, pool: Option<&CanisterPool>, env_file: Option<&Path>, ) -> DfxResult { + let pre_or_post = if is_pre_install { "pre" } else { "post" }; let tmp; let pool = match pool { Some(pool) => pool, @@ -450,8 +464,9 @@ fn run_post_install_tasks( let config = env.get_config_or_anyhow()?; let canisters_to_load = all_project_canisters_with_ids(env, &config); - tmp = CanisterPool::load(env, false, &canisters_to_load) - .context("Error collecting canisters for post-install task")?; + tmp = CanisterPool::load(env, false, &canisters_to_load).context(format!( + "Error collecting canisters for {pre_or_post}-install task" + ))?; &tmp } }; @@ -460,27 +475,43 @@ fn run_post_install_tasks( .iter() .map(|can| can.canister_id()) .collect_vec(); - for task in canister.get_post_install() { - run_post_install_task(canister, task, network, pool, &dependencies, env_file)?; + let tasks = if is_pre_install { + canister.get_pre_install() + } else { + canister.get_post_install() + }; + for task in tasks { + run_customized_install_task( + canister, + task, + is_pre_install, + network, + pool, + &dependencies, + env_file, + )?; } Ok(()) } -#[context("Failed to run post-install task {task}")] -fn run_post_install_task( +#[context("Failed to run {}-install task {}", if is_pre_install { "pre" } else { "post" }, task)] +fn run_customized_install_task( canister: &CanisterInfo, task: &str, + is_pre_install: bool, network: &NetworkDescriptor, pool: &CanisterPool, dependencies: &[Principal], env_file: Option<&Path>, ) -> DfxResult { + let pre_or_post = if is_pre_install { "pre" } else { "post" }; let cwd = canister.get_workspace_root(); let words = shell_words::split(task) - .with_context(|| format!("Error interpreting post-install task `{task}`"))?; + .with_context(|| format!("Error interpreting {pre_or_post}-install task `{task}`"))?; let canonicalized = dfx_core::fs::canonicalize(&cwd.join(&words[0])) .or_else(|_| which::which(&words[0])) .map_err(|_| anyhow!("Cannot find command or file {}", &words[0]))?; + let mut command = Command::new(canonicalized); command.args(&words[1..]); let vars = @@ -492,13 +523,14 @@ fn run_post_install_task( .current_dir(cwd) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); + let status = command.status()?; if !status.success() { match status.code() { Some(code) => { - bail!("The post-install task `{task}` failed with exit code {code}") + bail!("The {pre_or_post}-install task `{task}` failed with exit code {code}") } - None => bail!("The post-install task `{task}` was terminated by a signal"), + None => bail!("The {pre_or_post}-install task `{task}` was terminated by a signal"), } } Ok(()) From 79de1d01411b84f54b88db80dfe842b8706e5232 Mon Sep 17 00:00:00 2001 From: mraszyk <31483726+mraszyk@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:53:25 +0100 Subject: [PATCH 3/4] feat: impersonating sender of requests to a local PocketIC instance (#4013) * feat: impersonating sender of requests to a local PocketIC instance * windows * no max_request_time_ms * bump pocket-ic * request-status * cleanup * cleanup * fix * typo * impersonate.bash * dfx canister request-status * smoke * smoke * smoke * smoke * chore: update replica version to d9fe2076f677a08734bed90c67b1c3f4056ed621 * no e2e.yml changes * move impersonate.bash to call.bash * docs * unnecessary DfxResult * harden tests * fix dfx identity new in tests * no changes to request-status command --- CHANGELOG.md | 6 ++ docs/cli-reference/dfx-canister.mdx | 11 ++- e2e/tests-dfx/call.bash | 85 ++++++++++++++++++ src/dfx-core/src/canister/mod.rs | 3 + .../config/model/local_server_descriptor.rs | 33 +++++-- src/dfx-core/src/identity/mod.rs | 1 + src/dfx/Cargo.toml | 4 +- src/dfx/src/commands/canister/call.rs | 87 ++++++++++++++++++- src/dfx/src/commands/canister/delete.rs | 5 ++ .../src/commands/canister/deposit_cycles.rs | 3 + src/dfx/src/commands/canister/status.rs | 12 ++- .../src/commands/canister/update_settings.rs | 14 ++- src/dfx/src/lib/environment.rs | 36 ++++++++ .../operations/canister/create_canister.rs | 3 + src/dfx/src/lib/operations/canister/mod.rs | 55 +++++++++++- 15 files changed, 338 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c54a65832e..9ec00e6d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # UNRELEASED +### feat: impersonating sender of requests to a local PocketIC instance + +`dfx canister call`, `dfx canister status`, and `dfx canister update-settings` take +an additional CLI argument `--impersonate` to specify a principal +on behalf of which requests to a local PocketIC instance are sent. + ### feat: `dfx canister [create|update-settings] --wasm-memory-threshold` This adds support for the WASM memory threshold, used in conjunction with `--wasm-memory-limit`. diff --git a/docs/cli-reference/dfx-canister.mdx b/docs/cli-reference/dfx-canister.mdx index 301d26ec93..7cefb31e7e 100644 --- a/docs/cli-reference/dfx-canister.mdx +++ b/docs/cli-reference/dfx-canister.mdx @@ -145,6 +145,7 @@ You can use the following options with the `dfx canister call` command. | `--argument-file ` | Specifies the file from which to read the argument to pass to the method. Stdin may be referred to as `-`. | | `--async` | Specifies not to wait for the result of the call to be returned by polling the replica. Instead return a response ID. | | `--candid ` | Provide the .did file with which to decode the response. Overrides value from dfx.json for project canisters. | +| `--impersonate ` | Specifies a principal on behalf of which requests to a local PocketIC instance are sent. | | `--output ` | Specifies the output format to use when displaying a method’s return result. The valid values are `idl`, 'json', `pp` and `raw`. The `pp` option is equivalent to `idl`, but is pretty-printed. | | `--query` | Sends a query request instead of an update request. For information about the difference between query and update calls, see [Canisters include both program and state](/docs/current/concepts/canisters-code#canister-state). | | `--random ` | Specifies the config for generating random arguments. | @@ -1005,10 +1006,11 @@ dfx canister status [--all | canister_name] You can use the following arguments with the `dfx canister status` command. -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--all` | Returns status information for all of the canisters configured in the `dfx.json` file. Note that you must specify `--all` or an individual canister name. | -| `canister_name` | Specifies the name of the canister you want to return information for. Note that you must specify either a canister name or the `--all` option. | +| Argument | Description | +|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--all` | Returns status information for all of the canisters configured in the `dfx.json` file. Note that you must specify `--all` or an individual canister name. | +| `--impersonate ` | Specifies a principal on behalf of which requests to a local PocketIC instance are sent. | +| `canister_name` | Specifies the name of the canister you want to return information for. Note that you must specify either a canister name or the `--all` option. | ### Examples @@ -1142,6 +1144,7 @@ You can specify the following options for the `dfx canister update-settings` com | `--add-log-viewer ` | Add a principal to the list of log viewers of the canister. Can be specified more than once to add multiple log viewers. If current log visibility is `public` or `controllers`, it will be changed to the custom allowed viewer list. | | `-c`, `--compute-allocation ` | Specifies the canister's compute allocation. This should be a percent in the range [0..100]. | | `--confirm-very-long-freezing-threshold` | Freezing thresholds above ~1.5 years require this option as confirmation. | +| `--impersonate ` | Specifies a principal on behalf of which requests to a local PocketIC instance are sent. | | `--set-controller ` | Specifies the identity name or the principal of the new controller. Can be specified more than once, indicating the canister will have multiple controllers. If any controllers are set with this parameter, any other controllers will be removed. | | `--set-log-viewer ` | Specifies the the principal of the log viewer of the canister. Can be specified more than once, indicating the canister will have multiple log viewers. If any log viewers are set with this parameter, any other log viewers will be removed. If current log visibility is `public` or `controllers`, it will be changed to the custom allowed viewer list. | | `--memory-allocation ` | Specifies how much memory the canister is allowed to use in total. This should be a value in the range [0..12 GiB]. A setting of 0 means the canister will have access to memory on a “best-effort” basis: It will only be charged for the memory it uses, but at any point in time may stop running if it tries to allocate more memory when there isn’t space available on the subnet. | diff --git a/e2e/tests-dfx/call.bash b/e2e/tests-dfx/call.bash index 8122d77868..e1c18fc82c 100644 --- a/e2e/tests-dfx/call.bash +++ b/e2e/tests-dfx/call.bash @@ -298,3 +298,88 @@ teardown() { assert_command dfx canister call inter2_mo read assert_match '(8 : nat)' } + +function impersonate_sender() { + IDENTITY_PRINCIPAL="${1}" + + dfx_start + assert_command dfx deploy hello_backend + CANISTER_ID="$(dfx canister id hello_backend)" + + # set the management canister as the only controller + assert_command dfx canister update-settings hello_backend --set-controller "${IDENTITY_PRINCIPAL}" --yes + + # updating settings now fails because the default identity does not control the canister anymore + assert_command_fail dfx canister update-settings hello_backend --freezing-threshold 0 + assert_contains "Only controllers of canister $CANISTER_ID can call ic00 method update_settings" + + # updating settings succeeds when impersonating the management canister as the sender + assert_command dfx canister update-settings hello_backend --freezing-threshold 0 --impersonate "${IDENTITY_PRINCIPAL}" + + # test management canister call failure (setting memory allocation to a low value) + assert_command_fail dfx canister update-settings hello_backend --memory-allocation 1 --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Management canister call failed: IC0402: Canister was given 1 B memory allocation but at least" + + # canister status fails because the default identity does not control the canister anymore + assert_command_fail dfx canister status hello_backend + assert_contains "Only controllers of canister $CANISTER_ID can call ic00 method canister_status" + + # canister status succeeds when impersonating the management canister as the sender + assert_command dfx canister status hello_backend --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Controllers: ${IDENTITY_PRINCIPAL}" + assert_contains "Freezing threshold: 0" + + # freeze the canister + assert_command dfx canister update-settings hello_backend --freezing-threshold 9223372036854775808 --confirm-very-long-freezing-threshold --impersonate "${IDENTITY_PRINCIPAL}" + + # test management canister call submission failure + assert_command_fail dfx canister status hello_backend --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Failed to submit management canister call: IC0207: Canister $CANISTER_ID is out of cycles" + + # test update call submission failure + assert_command_fail dfx canister call aaaaa-aa canister_status "(record { canister_id=principal\"$CANISTER_ID\" })" --update --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Failed to submit canister call: IC0207: Canister $CANISTER_ID is out of cycles" + + # test async call submission failure + assert_command_fail dfx canister call aaaaa-aa canister_status "(record { canister_id=principal\"$CANISTER_ID\" })" --async --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Failed to submit canister call: IC0207: Canister $CANISTER_ID is out of cycles" + + # unfreeze the canister + assert_command dfx canister update-settings hello_backend --freezing-threshold 0 --impersonate "${IDENTITY_PRINCIPAL}" + + # test update call failure + assert_command_fail dfx canister call aaaaa-aa delete_canister "(record { canister_id=principal\"$CANISTER_ID\" })" --update --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Canister call failed: IC0510: Canister $CANISTER_ID must be stopped before it is deleted." + + # test update call + assert_command dfx canister call aaaaa-aa start_canister "(record { canister_id=principal\"$CANISTER_ID\" })" --update --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "()" + + # test async call + assert_command dfx canister call aaaaa-aa canister_status "(record { canister_id=principal\"$CANISTER_ID\" })" --async --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "Request ID:" + + # test query call failure + assert_command_fail dfx canister call aaaaa-aa fetch_canister_logs "(record { canister_id=principal\"$CANISTER_ID\" })" --query --impersonate "$CANISTER_ID" + assert_contains "Failed to perform query call: IC0406: Caller $CANISTER_ID is not allowed to query ic00 method fetch_canister_logs" + + # test query call + assert_command dfx canister call aaaaa-aa fetch_canister_logs "(record { canister_id=principal\"$CANISTER_ID\" })" --query --impersonate "${IDENTITY_PRINCIPAL}" + assert_contains "(record { 1_754_302_831 = vec {} })" +} + +@test "impersonate management canister as sender" { + [[ ! "$USE_POCKETIC" ]] && skip "skipped for replica: impersonating sender is only supported for PocketIC" + + impersonate_sender "aaaaa-aa" +} + +@test "impersonate new random identity as sender" { + [[ ! "$USE_POCKETIC" ]] && skip "skipped for replica: impersonating sender is only supported for PocketIC" + + dfx identity new impersonated_identity --storage-mode plaintext + IDENTITY_PRINCIPAL="$(dfx --identity impersonated_identity identity get-principal)" + dfx identity remove impersonated_identity + + impersonate_sender "${IDENTITY_PRINCIPAL}" +} diff --git a/src/dfx-core/src/canister/mod.rs b/src/dfx-core/src/canister/mod.rs index 5e386c338c..e57d7c16a0 100644 --- a/src/dfx-core/src/canister/mod.rs +++ b/src/dfx-core/src/canister/mod.rs @@ -71,6 +71,9 @@ YOU WILL LOSE ALL DATA IN THE CANISTER. .await .map_err(CanisterInstallError::InstallWasmError) } + CallSender::Impersonate(_) => { + unreachable!("Impersonating sender when installing canisters is not supported.") + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; let install_args = CanisterInstall { diff --git a/src/dfx-core/src/config/model/local_server_descriptor.rs b/src/dfx-core/src/config/model/local_server_descriptor.rs index f9eaf562f9..e2a6f9af62 100644 --- a/src/dfx-core/src/config/model/local_server_descriptor.rs +++ b/src/dfx-core/src/config/model/local_server_descriptor.rs @@ -360,7 +360,6 @@ impl LocalServerDescriptor { logger: Option<&Logger>, ) -> Result, NetworkConfigError> { let replica_port_path = self.replica_port_path(); - let pocketic_port_path = self.pocketic_port_path(); match read_port_from(&replica_port_path)? { Some(port) => { if let Some(logger) = logger { @@ -368,15 +367,31 @@ impl LocalServerDescriptor { } Ok(Some(port)) } - None => match read_port_from(&pocketic_port_path)? { - Some(port) => { - if let Some(logger) = logger { - info!(logger, "Found local PocketIC running on port {}", port); - } - Ok(Some(port)) + None => { + let port = self + .get_running_pocketic_port(logger)? + .or(self.replica.port); + Ok(port) + } + } + } + + /// Gets the port of a local PocketIC instance. + /// + /// # Prerequisites + /// - A local PocketIC instance needs to be running, e.g. with `dfx start --pocketic`. + pub fn get_running_pocketic_port( + &self, + logger: Option<&Logger>, + ) -> Result, NetworkConfigError> { + match read_port_from(&self.pocketic_port_path())? { + Some(port) => { + if let Some(logger) = logger { + info!(logger, "Found local PocketIC running on port {}", port); } - None => Ok(self.replica.port), - }, + Ok(Some(port)) + } + None => Ok(None), } } } diff --git a/src/dfx-core/src/identity/mod.rs b/src/dfx-core/src/identity/mod.rs index 8fa5732fc4..580353e1ba 100644 --- a/src/dfx-core/src/identity/mod.rs +++ b/src/dfx-core/src/identity/mod.rs @@ -307,6 +307,7 @@ impl AsRef for Identity { #[derive(Debug, PartialEq, Eq, Copy, Clone)] pub enum CallSender { SelectedId, + Impersonate(Principal), Wallet(Principal), } diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 1a5d37c050..16086c95be 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -86,6 +86,7 @@ os_str_bytes = { version = "6.3.0", features = ["conversions"] } patch = "0.7.0" pem.workspace = true petgraph = "0.6.0" +pocket-ic = { git = "https://github.com/dfinity/ic", rev = "3e24396441e4c7380928d4e8b4ccff7de77d0e7e" } rand = "0.8.5" regex = "1.5.5" reqwest = { workspace = true, features = ["blocking", "json"] } @@ -125,9 +126,6 @@ ci_info = "0.14" [target.'cfg(windows)'.dependencies] junction = "1.0.0" -[target.'cfg(unix)'.dependencies] -pocket-ic = { git = "https://github.com/dfinity/ic", rev = "3e24396441e4c7380928d4e8b4ccff7de77d0e7e" } - [dev-dependencies] env_logger = "0.10" proptest = "1.0" diff --git a/src/dfx/src/commands/canister/call.rs b/src/dfx/src/commands/canister/call.rs index aecf4d9a56..8a8249e19f 100644 --- a/src/dfx/src/commands/canister/call.rs +++ b/src/dfx/src/commands/canister/call.rs @@ -6,6 +6,7 @@ use crate::lib::root_key::fetch_root_key_if_needed; use crate::util::clap::argument_from_cli::ArgumentFromCliPositionalOpt; use crate::util::clap::parsers::cycle_amount_parser; use crate::util::{blob_from_arguments, fetch_remote_did_file, get_candid_type, print_idl_blob}; +use anyhow::bail; use anyhow::{anyhow, Context}; use candid::Principal as CanisterId; use candid::{CandidType, Decode, Deserialize, Principal}; @@ -14,11 +15,14 @@ use clap::Parser; use dfx_core::canister::build_wallet_canister; use dfx_core::identity::CallSender; use ic_agent::agent::CallResponse; +use ic_agent::RequestId; use ic_utils::canister::Argument; use ic_utils::interfaces::management_canister::builders::{CanisterInstall, CanisterSettings}; use ic_utils::interfaces::management_canister::MgmtMethod; use ic_utils::interfaces::wallet::{CallForwarder, CallResult}; use ic_utils::interfaces::WalletCanister; +use pocket_ic::common::rest::RawEffectivePrincipal; +use pocket_ic::WasmResult; use slog::warn; use std::option::Option; use std::path::PathBuf; @@ -83,6 +87,11 @@ pub struct CanisterCallOpts { conflicts_with("random") )] always_assist: bool, + + /// Send request on behalf of the specified principal. + /// This option only works for a local PocketIC instance. + #[arg(long)] + impersonate: Option, } #[derive(Clone, CandidType, Deserialize, Debug)] @@ -206,8 +215,13 @@ pub fn get_effective_canister_id( pub async fn exec( env: &dyn Environment, opts: CanisterCallOpts, - call_sender: &CallSender, + mut call_sender: &CallSender, ) -> DfxResult { + let call_sender_override = opts.impersonate.map(CallSender::Impersonate); + if let Some(ref call_sender_override) = call_sender_override { + call_sender = call_sender_override; + }; + let agent = env.get_agent(); fetch_root_key_if_needed(env).await?; @@ -335,6 +349,29 @@ To figure out the id of your wallet, run 'dfx identity get-wallet (--network ic) .with_arg(arg_value); query_builder.call().await.context("Failed query call.")? } + CallSender::Impersonate(sender) => { + let pocketic = env.get_pocketic(); + if let Some(pocketic) = pocketic { + let res = pocketic + .query_call_with_effective_principal( + canister_id, + RawEffectivePrincipal::CanisterId( + effective_canister_id.as_slice().to_vec(), + ), + *sender, + method_name, + arg_value, + ) + .await + .map_err(|err| anyhow!("Failed to perform query call: {}", err))?; + match res { + WasmResult::Reply(data) => data, + WasmResult::Reject(err) => bail!("Canister rejected: {}", err), + } + } else { + bail!("Impersonating sender is only supported for a local PocketIC instance.") + } + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; do_wallet_call( @@ -361,6 +398,27 @@ To figure out the id of your wallet, run 'dfx identity get-wallet (--network ic) .await .context("Failed update call.")? .map(|(res, _)| res), + CallSender::Impersonate(sender) => { + let pocketic = env.get_pocketic(); + if let Some(pocketic) = pocketic { + let msg_id = pocketic + .submit_call_with_effective_principal( + canister_id, + RawEffectivePrincipal::CanisterId( + effective_canister_id.as_slice().to_vec(), + ), + *sender, + method_name, + arg_value, + ) + .await + .map_err(|err| anyhow!("Failed to submit canister call: {}", err))? + .message_id; + CallResponse::Poll(RequestId::new(msg_id.as_slice().try_into().unwrap())) + } else { + bail!("Impersonating sender is only supported for a local PocketIC instance.") + } + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; let mut args = Argument::default(); @@ -389,6 +447,33 @@ To figure out the id of your wallet, run 'dfx identity get-wallet (--network ic) .with_arg(arg_value) .await .context("Failed update call.")?, + CallSender::Impersonate(sender) => { + let pocketic = env.get_pocketic(); + if let Some(pocketic) = pocketic { + let msg_id = pocketic + .submit_call_with_effective_principal( + canister_id, + RawEffectivePrincipal::CanisterId( + effective_canister_id.as_slice().to_vec(), + ), + *sender, + method_name, + arg_value, + ) + .await + .map_err(|err| anyhow!("Failed to submit canister call: {}", err))?; + let res = pocketic + .await_call_no_ticks(msg_id) + .await + .map_err(|err| anyhow!("Canister call failed: {}", err))?; + match res { + WasmResult::Reply(data) => data, + WasmResult::Reject(err) => bail!("Canister rejected: {}", err), + } + } else { + bail!("Impersonating sender is only supported for a local PocketIC instance.") + } + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; do_wallet_call( diff --git a/src/dfx/src/commands/canister/delete.rs b/src/dfx/src/commands/canister/delete.rs index c99c5df90d..289bf81e3f 100644 --- a/src/dfx/src/commands/canister/delete.rs +++ b/src/dfx/src/commands/canister/delete.rs @@ -127,6 +127,11 @@ async fn delete_canister( CallSender::Wallet(wallet_id) => WithdrawTarget::Canister { canister_id: *wallet_id, }, + CallSender::Impersonate(_) => { + unreachable!( + "Impersonating sender when deleting canisters is not supported." + ) + } CallSender::SelectedId => { let network = env.get_network_descriptor(); let identity_name = env diff --git a/src/dfx/src/commands/canister/deposit_cycles.rs b/src/dfx/src/commands/canister/deposit_cycles.rs index 3a2bb26e0c..5aba4a9ed5 100644 --- a/src/dfx/src/commands/canister/deposit_cycles.rs +++ b/src/dfx/src/commands/canister/deposit_cycles.rs @@ -67,6 +67,9 @@ async fn deposit_cycles( ) .await?; } + CallSender::Impersonate(_) => { + unreachable!("Impersonating sender when depositing cycles is not supported.") + } CallSender::Wallet(_) => { canister::deposit_cycles(env, canister_id, call_sender, cycles).await? } diff --git a/src/dfx/src/commands/canister/status.rs b/src/dfx/src/commands/canister/status.rs index 9218774219..e831e0de46 100644 --- a/src/dfx/src/commands/canister/status.rs +++ b/src/dfx/src/commands/canister/status.rs @@ -16,6 +16,11 @@ pub struct CanisterStatusOpts { /// You must specify either a canister name or the --all flag. canister: Option, + /// Send request on behalf of the specified principal. + /// This option only works for a local PocketIC instance. + #[arg(long)] + impersonate: Option, + /// Returns status information for all of the canisters configured in the dfx.json file. #[arg(long, required_unless_present("canister"))] all: bool, @@ -115,8 +120,13 @@ Log visibility: {log_visibility}", pub async fn exec( env: &dyn Environment, opts: CanisterStatusOpts, - call_sender: &CallSender, + mut call_sender: &CallSender, ) -> DfxResult { + let call_sender_override = opts.impersonate.map(CallSender::Impersonate); + if let Some(ref call_sender_override) = call_sender_override { + call_sender = call_sender_override; + }; + fetch_root_key_if_needed(env).await?; if let Some(canister) = opts.canister.as_deref() { diff --git a/src/dfx/src/commands/canister/update_settings.rs b/src/dfx/src/commands/canister/update_settings.rs index 1d18cb1555..d1ae33fb1a 100644 --- a/src/dfx/src/commands/canister/update_settings.rs +++ b/src/dfx/src/commands/canister/update_settings.rs @@ -17,6 +17,7 @@ use crate::util::clap::parsers::{ use anyhow::{bail, Context}; use byte_unit::Byte; use candid::Principal as CanisterId; +use candid::Principal; use clap::{ArgAction, Parser}; use dfx_core::cli::ask_for_consent; use dfx_core::error::identity::InstantiateIdentityFromNameError::GetIdentityPrincipalFailed; @@ -114,13 +115,23 @@ pub struct UpdateSettingsOpts { /// so this is not recommended outside of CI. #[arg(long, short)] yes: bool, + + /// Send request on behalf of the specified principal. + /// This option only works for a local PocketIC instance. + #[arg(long)] + impersonate: Option, } pub async fn exec( env: &dyn Environment, opts: UpdateSettingsOpts, - call_sender: &CallSender, + mut call_sender: &CallSender, ) -> DfxResult { + let call_sender_override = opts.impersonate.map(CallSender::Impersonate); + if let Some(ref call_sender_override) = call_sender_override { + call_sender = call_sender_override; + }; + // sanity checks if let Some(threshold_in_seconds) = opts.freezing_threshold { if threshold_in_seconds > 50_000_000 /* ~1.5 years */ && !opts.confirm_very_long_freezing_threshold @@ -372,6 +383,7 @@ fn user_is_removing_themselves_as_controller( .get_selected_identity_principal() .context("Selected identity is not instantiated")? .to_string(), + CallSender::Impersonate(sender) => sender.to_string(), CallSender::Wallet(principal) => principal.to_string(), }; let removes_themselves = diff --git a/src/dfx/src/lib/environment.rs b/src/dfx/src/lib/environment.rs index 8e993bbc3e..2b701f233d 100644 --- a/src/dfx/src/lib/environment.rs +++ b/src/dfx/src/lib/environment.rs @@ -12,10 +12,12 @@ use dfx_core::config::model::network_descriptor::{NetworkDescriptor, NetworkType use dfx_core::error::canister_id_store::CanisterIdStoreError; use dfx_core::error::identity::NewIdentityManagerError; use dfx_core::error::load_dfx_config::LoadDfxConfigError; +use dfx_core::error::uri::UriError; use dfx_core::extension::manager::ExtensionManager; use dfx_core::identity::identity_manager::{IdentityManager, InitializeIdentity}; use fn_error_context::context; use ic_agent::{Agent, Identity}; +use pocket_ic::nonblocking::PocketIc; use semver::Version; use slog::{Logger, Record}; use std::borrow::Cow; @@ -23,6 +25,7 @@ use std::cell::RefCell; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; +use url::Url; pub trait Environment { fn get_cache(&self) -> Arc; @@ -44,6 +47,8 @@ pub trait Environment { #[allow(clippy::needless_lifetimes)] fn get_agent<'a>(&'a self) -> &'a Agent; + fn get_pocketic(&self) -> Option<&PocketIc>; + #[allow(clippy::needless_lifetimes)] fn get_network_descriptor<'a>(&'a self) -> &'a NetworkDescriptor; @@ -213,6 +218,10 @@ impl Environment for EnvironmentImpl { unreachable!("Agent only available from an AgentEnvironment"); } + fn get_pocketic(&self) -> Option<&PocketIc> { + unreachable!("PocketIC handle only available from an AgentEnvironment"); + } + fn get_network_descriptor(&self) -> &NetworkDescriptor { // It's not valid to call get_network_descriptor on an EnvironmentImpl. // All of the places that call this have an AgentEnvironment anyway. @@ -267,6 +276,7 @@ impl Environment for EnvironmentImpl { pub struct AgentEnvironment<'a> { backend: &'a dyn Environment, agent: Agent, + pocketic: Option, network_descriptor: NetworkDescriptor, identity_manager: IdentityManager, effective_canister_id: Option, @@ -314,9 +324,27 @@ impl<'a> AgentEnvironment<'a> { None }; + let pocketic = + if let Some(local_server_descriptor) = &network_descriptor.local_server_descriptor { + match local_server_descriptor.get_running_pocketic_port(None)? { + Some(port) => { + let mut socket_addr = local_server_descriptor.bind_address; + socket_addr.set_port(port); + let url = format!("http://{}", socket_addr); + let url = Url::parse(&url) + .map_err(|e| UriError::UrlParseError(url.to_string(), e))?; + Some(create_pocketic(&url)) + } + None => None, + } + } else { + None + }; + Ok(AgentEnvironment { backend, agent: create_agent(logger, url, identity, timeout)?, + pocketic, network_descriptor: network_descriptor.clone(), identity_manager, effective_canister_id, @@ -359,6 +387,10 @@ impl<'a> Environment for AgentEnvironment<'a> { &self.agent } + fn get_pocketic(&self) -> Option<&PocketIc> { + self.pocketic.as_ref() + } + fn get_network_descriptor(&self) -> &NetworkDescriptor { &self.network_descriptor } @@ -422,3 +454,7 @@ pub fn create_agent( .build()?; Ok(agent) } + +pub fn create_pocketic(url: &Url) -> PocketIc { + PocketIc::new_from_existing_instance(url.clone(), 0, None) +} diff --git a/src/dfx/src/lib/operations/canister/create_canister.rs b/src/dfx/src/lib/operations/canister/create_canister.rs index 03c1e3928f..11f135c98e 100644 --- a/src/dfx/src/lib/operations/canister/create_canister.rs +++ b/src/dfx/src/lib/operations/canister/create_canister.rs @@ -173,6 +173,9 @@ The command line value will be used.", .await } } + CallSender::Impersonate(_) => { + unreachable!("Impersonating sender when creating canisters is not supported.") + } CallSender::Wallet(wallet_id) => { create_with_wallet(agent, &wallet_id, with_cycles, settings, subnet_selection).await } diff --git a/src/dfx/src/lib/operations/canister/mod.rs b/src/dfx/src/lib/operations/canister/mod.rs index 9551ddc4fb..57d54490d6 100644 --- a/src/dfx/src/lib/operations/canister/mod.rs +++ b/src/dfx/src/lib/operations/canister/mod.rs @@ -13,11 +13,12 @@ use crate::lib::canister_info::CanisterInfo; use crate::lib::environment::Environment; use crate::lib::error::DfxResult; use crate::lib::ic_attributes::CanisterSettings as DfxCanisterSettings; -use anyhow::{bail, Context}; +use anyhow::{anyhow, bail, Context}; use candid::utils::ArgumentDecoder; use candid::CandidType; use candid::Principal as CanisterId; use candid::Principal; +use candid::{decode_args, encode_args}; use dfx_core::canister::build_wallet_canister; use dfx_core::config::model::dfinity::Config; use dfx_core::identity::CallSender; @@ -29,6 +30,8 @@ use ic_utils::interfaces::management_canister::{ }; use ic_utils::interfaces::ManagementCanister; use ic_utils::Argument; +use pocket_ic::common::rest::RawEffectivePrincipal; +use pocket_ic::WasmResult; use std::collections::HashSet; use std::path::PathBuf; @@ -61,6 +64,32 @@ where .await .context("Update call (without wallet) failed.")? } + CallSender::Impersonate(sender) => { + let pocketic = env.get_pocketic(); + if let Some(pocketic) = pocketic { + let msg_id = pocketic + .submit_call_with_effective_principal( + Principal::management_canister(), + RawEffectivePrincipal::CanisterId(destination_canister.as_slice().to_vec()), + *sender, + method, + encode_args((arg,)).unwrap(), + ) + .await + .map_err(|err| anyhow!("Failed to submit management canister call: {}", err))?; + let res = pocketic + .await_call_no_ticks(msg_id) + .await + .map_err(|err| anyhow!("Management canister call failed: {}", err))?; + match res { + WasmResult::Reply(data) => decode_args(&data) + .context("Could not decode management canister response.")?, + WasmResult::Reject(err) => bail!("Management canister rejected: {}", err), + } + } else { + bail!("Impersonating sender is only supported for a local PocketIC instance.") + } + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; let out: O = wallet @@ -108,6 +137,30 @@ where .await .context("Query call (without wallet) failed.")? } + CallSender::Impersonate(sender) => { + let pocketic = env.get_pocketic(); + if let Some(pocketic) = pocketic { + let res = pocketic + .query_call_with_effective_principal( + Principal::management_canister(), + RawEffectivePrincipal::CanisterId(destination_canister.as_slice().to_vec()), + *sender, + method, + encode_args((arg,)).unwrap(), + ) + .await + .map_err(|err| { + anyhow!("Failed to perform management canister query call: {}", err) + })?; + match res { + WasmResult::Reply(data) => decode_args(&data) + .context("Failed to decode management canister query call response.")?, + WasmResult::Reject(err) => bail!("Management canister rejected: {}", err), + } + } else { + bail!("Impersonating sender is only supported for a local PocketIC instance.") + } + } CallSender::Wallet(wallet_id) => { let wallet = build_wallet_canister(*wallet_id, agent).await?; let out: O = wallet From c03e41a13da2f948ba6bc08ab2d8b9357d4b9401 Mon Sep 17 00:00:00 2001 From: Adam Spofford <93943719+adamspofford-dfinity@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:28:13 -0800 Subject: [PATCH 4/4] fix: add browser warning message (#4048) --- CHANGELOG.md | 6 ++++++ .../react/src/__frontend_name__/index.html | 4 ++++ .../react/src/__frontend_name__/src/index.scss | 4 ++++ .../svelte/src/__frontend_name__/src/app.html | 4 ++++ .../svelte/src/__frontend_name__/src/index.scss | 4 ++++ .../vanilla_js/src/__frontend_name__/index.html | 4 ++++ .../vanilla_js/src/__frontend_name__/src/index.scss | 4 ++++ .../project_templates/vue/src/__frontend_name__/index.html | 4 ++++ .../vue/src/__frontend_name__/src/index.scss | 4 ++++ 9 files changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ec00e6d19..8e76dab1a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # UNRELEASED +### fix: template frontends now have unsupported browser warnings + +DFX's default security headers cause Safari to break when viewing local canisters. Warning messages +have been added to the frontend project templates when the page is broken that indicate to try switching +browsers. + ### feat: impersonating sender of requests to a local PocketIC instance `dfx canister call`, `dfx canister status`, and `dfx canister update-settings` take diff --git a/src/dfx/assets/project_templates/react/src/__frontend_name__/index.html b/src/dfx/assets/project_templates/react/src/__frontend_name__/index.html index a8f14f76b8..0ad7b99b93 100644 --- a/src/dfx/assets/project_templates/react/src/__frontend_name__/index.html +++ b/src/dfx/assets/project_templates/react/src/__frontend_name__/index.html @@ -10,6 +10,10 @@ +
+ ⚠️ If this page appears broken, try reloading. If the problem persists, try switching browsers; Chrome or + Firefox are recommended. +
diff --git a/src/dfx/assets/project_templates/react/src/__frontend_name__/src/index.scss b/src/dfx/assets/project_templates/react/src/__frontend_name__/src/index.scss index c1c320834c..283585fb27 100644 --- a/src/dfx/assets/project_templates/react/src/__frontend_name__/src/index.scss +++ b/src/dfx/assets/project_templates/react/src/__frontend_name__/src/index.scss @@ -35,3 +35,7 @@ button[type="submit"] { #greeting:empty { display: none; } + +.browser-warning { + display: none; +} diff --git a/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/app.html b/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/app.html index dfae06cc73..d4a7d45f50 100644 --- a/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/app.html +++ b/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/app.html @@ -10,6 +10,10 @@ +
+ ⚠️ If this page appears broken, try reloading. If the problem persists, try switching browsers; Chrome or + Firefox are recommended. +
%sveltekit.body%
diff --git a/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/index.scss b/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/index.scss index c1c320834c..283585fb27 100644 --- a/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/index.scss +++ b/src/dfx/assets/project_templates/svelte/src/__frontend_name__/src/index.scss @@ -35,3 +35,7 @@ button[type="submit"] { #greeting:empty { display: none; } + +.browser-warning { + display: none; +} diff --git a/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/index.html b/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/index.html index 6c1db68a54..482508af30 100644 --- a/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/index.html +++ b/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/index.html @@ -9,6 +9,10 @@ +
+ ⚠️ If this page appears broken, try reloading. If the problem persists, try switching browsers; Chrome or + Firefox are recommended. +
diff --git a/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/src/index.scss b/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/src/index.scss index c1c320834c..283585fb27 100644 --- a/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/src/index.scss +++ b/src/dfx/assets/project_templates/vanilla_js/src/__frontend_name__/src/index.scss @@ -35,3 +35,7 @@ button[type="submit"] { #greeting:empty { display: none; } + +.browser-warning { + display: none; +} diff --git a/src/dfx/assets/project_templates/vue/src/__frontend_name__/index.html b/src/dfx/assets/project_templates/vue/src/__frontend_name__/index.html index 6e80f94528..8ba7f24a12 100644 --- a/src/dfx/assets/project_templates/vue/src/__frontend_name__/index.html +++ b/src/dfx/assets/project_templates/vue/src/__frontend_name__/index.html @@ -14,6 +14,10 @@ We're sorry but this application doesn't work properly without JavaScript enabled. Please enable it to continue. +
+ ⚠️ If this page appears broken, try reloading. If the problem persists, try switching browsers; Chrome or + Firefox are recommended. +
diff --git a/src/dfx/assets/project_templates/vue/src/__frontend_name__/src/index.scss b/src/dfx/assets/project_templates/vue/src/__frontend_name__/src/index.scss index c1c320834c..283585fb27 100644 --- a/src/dfx/assets/project_templates/vue/src/__frontend_name__/src/index.scss +++ b/src/dfx/assets/project_templates/vue/src/__frontend_name__/src/index.scss @@ -35,3 +35,7 @@ button[type="submit"] { #greeting:empty { display: none; } + +.browser-warning { + display: none; +}