diff --git a/CHANGELOG.md b/CHANGELOG.md index fad6bf3b98..faf7705a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # UNRELEASED +### feat: Make integration + +* Flags `--no-compile` and `--no-deps` + +* Command `rules` that outputs time-efficient GNU Make rules. + ### feat: `skip_cargo_audit` flag in dfx.json to skip `cargo audit` build step ### fix: `dfx canister install` and `dfx deploy` with `--no-asset-upgrade` no longer hang indefinitely when wasm is not up to date diff --git a/docs/cli-reference/dfx-build.mdx b/docs/cli-reference/dfx-build.mdx index 0e58b8f43d..d8327c499d 100644 --- a/docs/cli-reference/dfx-build.mdx +++ b/docs/cli-reference/dfx-build.mdx @@ -20,9 +20,10 @@ dfx build [flag] [option] [--all | canister_name] You can use the following optional flags with the `dfx build` command. -| Flag | Description | -| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--check` | Builds canisters using a temporary, hard-coded, locally-defined canister identifier for testing that your program compiles without connecting to the IC. | +| Flag | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--check` | Builds canisters using a temporary, hard-coded, locally-defined canister identifier for testing that your program compiles without connecting to the IC. | +| `--no-deps` | Don't build dependencies, only this canister. | ## Options diff --git a/docs/cli-reference/dfx-deploy.mdx b/docs/cli-reference/dfx-deploy.mdx index 6433710058..a270f2c930 100644 --- a/docs/cli-reference/dfx-deploy.mdx +++ b/docs/cli-reference/dfx-deploy.mdx @@ -46,6 +46,7 @@ You can use the following options with the `dfx deploy` command. | `--next-to ` | Create canisters on the same subnet as this canister. | | `--skip-pre-upgrade` | Skip the pre_upgrade hook on upgrade. This requires the upgrade/auto mode. | | `--wasm-memory-persistence ` | Keep or replace the Wasm main memory on upgrade. Possible values: keep, replace. This requires the upgrade/auto mode. | +| `--no-compile` | Don't compile before deploying. | ### Specifies the argument to pass to the init entrypoint diff --git a/docs/cli-reference/dfx-generate.mdx b/docs/cli-reference/dfx-generate.mdx index 3efacc0de5..9d6d8429b9 100644 --- a/docs/cli-reference/dfx-generate.mdx +++ b/docs/cli-reference/dfx-generate.mdx @@ -18,6 +18,14 @@ The `dfx generate` command looks for the configuration under the `declarations` dfx generate [canister_name] ``` +## Options + +You can use the following options with the `dfx deploy` command. + +| Argument | Description | +|----------------|-----------------------------------------------------------| +| `--no-compile` | Don't compile the canister(s) before generating bindings. | + ## Arguments You can specify the following arguments for the `dfx generate` command. diff --git a/docs/cli-reference/dfx-rules.mdx b/docs/cli-reference/dfx-rules.mdx new file mode 100644 index 0000000000..ab14a87231 --- /dev/null +++ b/docs/cli-reference/dfx-rules.mdx @@ -0,0 +1,28 @@ +import { MarkdownChipRow } from "/src/components/Chip/MarkdownChipRow"; + +# dfx rules + + + +**Warning:** This is an experimental feature. + +Use the `dfx rules` command to output GNU Make rules for efficient compilation of canisters. + +The rules, that can be used: +* `canister@NAME` - compile a canister. +* `deploy@NAME` - deploy a canister. +* `generate@NAME` - generate bindings for a canister. + +TODO + +## Basic usage + +``` bash +dfx rules [-o MAKEFILE] +``` + +**Warning:** The MAKEFILE is overwritten by the above command. + +| Argument | Description | +|------------------------------|---------------------------------| +| `-o FILE` or `--output FILE` | Use this file instead of stdout | \ No newline at end of file diff --git a/src/dfx/assets/prepare_assets.rs b/src/dfx/assets/prepare_assets.rs index 95f5856821..5a6dbc793d 100644 --- a/src/dfx/assets/prepare_assets.rs +++ b/src/dfx/assets/prepare_assets.rs @@ -146,13 +146,13 @@ fn write_binary_cache( async fn download_and_check_sha(client: Client, source: Source) -> Bytes { let retry_policy = ExponentialBackoffBuilder::new() .with_initial_interval(Duration::from_secs(1)) - .with_max_interval(Duration::from_secs(16)) - .with_multiplier(2.0) - .with_max_elapsed_time(Some(Duration::from_secs(300))) + .with_max_interval(Duration::from_secs(64)) + .with_multiplier(1.5) + .with_max_elapsed_time(Some(Duration::from_secs(3600))) .build(); let response = retry(retry_policy, || async { - match client.get(&source.url).send().await { + match client.get(&source.url).timeout(Duration::from_secs(3600)).send().await { Ok(response) => Ok(response), Err(err) => Err(backoff::Error::transient(err)), } diff --git a/src/dfx/src/commands/build.rs b/src/dfx/src/commands/build.rs index 449912c9ce..1fcb1c4db2 100644 --- a/src/dfx/src/commands/build.rs +++ b/src/dfx/src/commands/build.rs @@ -25,6 +25,10 @@ pub struct CanisterBuildOpts { #[arg(long)] check: bool, + /// Don't compile the dependencies, only this canister. + #[arg(long)] + no_deps: bool, + /// Output environment variables to a file in dotenv format (without overwriting any user-defined variables, if the file already exists). #[arg(long)] output_env_file: Option, @@ -48,10 +52,21 @@ pub fn exec(env: &dyn Environment, opts: CanisterBuildOpts) -> DfxResult { let build_mode_check = opts.check; - // Option can be None in which case --all was specified - let required_canisters = config - .get_config() - .get_canister_names_with_dependencies(opts.canister_name.as_deref())?; + let required_canisters = if opts.no_deps { + // Option can be None in which case --all was specified + let canister = opts.canister_name.as_deref(); + match canister { + Some(canister) => vec![canister.to_string()], + // inefficient: + None => config + .get_config() + .get_canister_names_with_dependencies(None)? + } + } else { + config + .get_config() + .get_canister_names_with_dependencies(opts.canister_name.as_deref())? + }; let canisters_to_load = add_canisters_with_ids(&required_canisters, &env, &config); let canisters_to_build = required_canisters @@ -90,7 +105,7 @@ pub fn exec(env: &dyn Environment, opts: CanisterBuildOpts) -> DfxResult { .with_build_mode_check(build_mode_check) .with_canisters_to_build(canisters_to_build) .with_env_file(env_file); - runtime.block_on(canister_pool.build_or_fail(&env, logger, &build_config))?; + runtime.block_on(canister_pool.build_or_fail(&env, logger, &build_config, opts.no_deps))?; Ok(()) } diff --git a/src/dfx/src/commands/deploy.rs b/src/dfx/src/commands/deploy.rs index 78751a62de..de2f117ffa 100644 --- a/src/dfx/src/commands/deploy.rs +++ b/src/dfx/src/commands/deploy.rs @@ -103,6 +103,10 @@ pub struct DeployOpts { #[command(flatten)] subnet_selection: SubnetSelectionOpt, + /// Skip compilation before deploying. + #[arg(long)] + no_compile: bool, + /// Always use Candid assist when the argument types are all optional. #[arg( long, @@ -183,6 +187,7 @@ pub fn exec(env: &dyn Environment, opts: DeployOpts) -> DfxResult { opts.no_asset_upgrade, &mut subnet_selection, opts.always_assist, + opts.no_compile, ))?; if matches!(deploy_mode, NormalDeploy | ForceReinstallSingleCanister(_)) { diff --git a/src/dfx/src/commands/generate.rs b/src/dfx/src/commands/generate.rs index bc5debb914..b92907a69f 100644 --- a/src/dfx/src/commands/generate.rs +++ b/src/dfx/src/commands/generate.rs @@ -15,6 +15,10 @@ pub struct GenerateOpts { /// If you do not specify a canister name, generates types for all canisters. canister_name: Option, + /// Don't compile Motoko before generating. + #[arg(long)] + no_compile: bool, + #[command(flatten)] network: NetworkOpt, } @@ -64,7 +68,7 @@ pub fn exec(env: &dyn Environment, opts: GenerateOpts) -> DfxResult { BuildConfig::from_config(&config, env.get_network_descriptor().is_playground())? .with_canisters_to_build(canisters_to_generate); - if build_config + if !opts.no_compile && build_config .canisters_to_build .as_ref() .map(|v| !v.is_empty()) @@ -73,7 +77,7 @@ pub fn exec(env: &dyn Environment, opts: GenerateOpts) -> DfxResult { let canister_pool_build = CanisterPool::load(&env, true, &build_dependees)?; slog::info!(log, "Building canisters before generate for Motoko"); let runtime = Runtime::new().expect("Unable to create a runtime"); - runtime.block_on(canister_pool_build.build_or_fail(&env, log, &build_config))?; + runtime.block_on(canister_pool_build.build_or_fail(&env, log, &build_config, opts.no_compile))?; } for canister in canister_pool_load.canisters_to_build(&generate_config) { diff --git a/src/dfx/src/commands/mod.rs b/src/dfx/src/commands/mod.rs index 6290e4b646..7d4ec5f01a 100644 --- a/src/dfx/src/commands/mod.rs +++ b/src/dfx/src/commands/mod.rs @@ -25,6 +25,7 @@ mod new; mod ping; mod quickstart; mod remote; +mod rules; mod schema; mod start; mod stop; @@ -57,6 +58,7 @@ pub enum DfxCommand { Ping(ping::PingOpts), Quickstart(quickstart::QuickstartOpts), Remote(remote::RemoteOpts), + Rules(rules::RulesOpts), Schema(schema::SchemaOpts), Start(start::StartOpts), Stop(stop::StopOpts), @@ -90,6 +92,7 @@ pub fn exec(env: &dyn Environment, cmd: DfxCommand) -> DfxResult { DfxCommand::Ping(v) => ping::exec(env, v), DfxCommand::Quickstart(v) => quickstart::exec(env, v), DfxCommand::Remote(v) => remote::exec(env, v), + DfxCommand::Rules(v) => rules::exec(env, v), DfxCommand::Schema(v) => schema::exec(v), DfxCommand::Start(v) => start::exec(env, v), DfxCommand::Stop(v) => stop::exec(env, v), diff --git a/src/dfx/src/commands/rules.rs b/src/dfx/src/commands/rules.rs new file mode 100644 index 0000000000..5ba47a2e30 --- /dev/null +++ b/src/dfx/src/commands/rules.rs @@ -0,0 +1,256 @@ +use std::collections::BTreeMap; +use std::fs::OpenOptions; +use std::io::Write; +use std::path::Path; + +use crate::lib::agent::create_anonymous_agent_environment; +use crate::lib::builders::CanisterBuilder; +use crate::lib::environment::Environment; +use crate::lib::error::DfxResult; +use crate::lib::models::canister::{CanisterPool, Import}; +use crate::lib::builders::custom::CustomBuilder; +use crate::lib::network::network_opt::NetworkOpt; +use itertools::Itertools; +use dfx_core::config::model::dfinity::{CanisterTypeProperties, ConfigCanistersCanister}; +use clap::Parser; +use petgraph::visit::EdgeRef; +use petgraph::Graph; +use petgraph::visit::GraphBase; + +/// Output dependencies in Make format +#[derive(Parser)] +pub struct RulesOpts { + /// File to output make rules + #[arg(long, short, value_name = "FILE")] + output: Option, + + #[command(flatten)] + network: NetworkOpt, +} + +// FIXME: It wrongly acts with downloaded canisters (like `internet_identity`). +// This seems to be the cause of double recompilation. +pub fn exec(env1: &dyn Environment, opts: RulesOpts) -> DfxResult { + let env = create_anonymous_agent_environment(env1, opts.network.to_network_name())?; + // let log = env.get_logger(); + + // Read the config. + let config = env.get_config_or_anyhow()?; + + // We load dependencies before creating the file to minimize the time that the file is half-written. + // Load dependencies for Make rules: + let builder = CustomBuilder::new(&env)?; // hackish use of CustomBuilder not intended for this use + let canisters = &config.get_config().canisters.as_ref(); + let canister_names = if let Some(canisters) = canisters { + canisters.keys().map(|k| k.to_string()).collect::>() + } else { + Vec::new() + }; + let pool: CanisterPool = CanisterPool::load( + &env, // if `env1`, fails with "NetworkDescriptor only available from an AgentEnvironment" + false, + &canister_names, + )?; + builder.read_all_dependencies( + &env, + &pool, + )?; + + let mut output_file: Box = match opts.output { + Some(filename) => Box::new(OpenOptions::new().write(true).create(true).truncate(true).open(filename)?), + None => Box::new(std::io::stdout()), + }; + + output_file.write_fmt(format_args!("NETWORK ?= local\n\n"))?; + output_file.write_fmt(format_args!("DEPLOY_FLAGS ?= \n\n"))?; + output_file.write_fmt(format_args!("ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))\n\n"))?; + + let graph0 = env.get_imports().borrow(); + let graph = graph0.graph(); + + match &canisters { + Some(canisters) => { + let canisters: &BTreeMap = canisters; + output_file.write_fmt(format_args!(".PHONY:"))?; + for canister in canisters { + output_file.write_fmt(format_args!(" canister@{}", canister.0))?; + }; + output_file.write_fmt(format_args!("\n\n.PHONY:"))?; + for canister in canisters { + output_file.write_fmt(format_args!(" deploy@{}", canister.0))?; + } + output_file.write_fmt(format_args!("\n\n.PHONY:"))?; + for canister in canisters { + output_file.write_fmt(format_args!(" generate@{}", canister.0))?; + } + output_file.write_fmt(format_args!("\n\n"))?; + for canister in canisters { + // duplicate code + let canister2: std::sync::Arc = pool.get_first_canister_with_name(&canister.0).unwrap(); + if canister2.get_info().is_assets() { + let path1 = format!(".dfx/$(NETWORK)/canisters/{}/assetstorage.wasm.gz", canister.0); + // let path2 = format!(".dfx/$(NETWORK)/canisters/{}/assetstorage.did", canister.0); + output_file.write_fmt(format_args!("canister@{}: \\\n {}\n\n", canister.0, path1))?; + // output_file.write_fmt(format_args!( + // "{} {}:\n\tdfx canister create {}\n\tdfx build --no-deps --network $(NETWORK) {}\n\n", path1, path2, canister.0, canister.0 + // ))?; + } else { + // let path1 = format!(".dfx/$(NETWORK)/canisters/{}/{}.wasm", canister.0, canister.0); + // let path2 = format!(".dfx/$(NETWORK)/canisters/{}/{}.did", canister.0, canister.0); + // TODO: `graph` here is superfluous: + let path = make_target(&pool, graph, *graph0.nodes().get(&Import::Canister(canister.0.clone())).unwrap())?; // TODO: `unwrap`? + output_file.write_fmt(format_args!("canister@{}: \\\n {}\n\n", canister.0, path))?; + if let Some(main) = &canister.1.main { + output_file.write_fmt(format_args!("{}: {}\n\n", path, main.to_str().unwrap()))?; + } + } + }; + for canister in canisters { + let declarations_config_pre = &canister.1.declarations; + // let workspace_root = config.get_path().parent().unwrap(); + // duplicate code: + let output = declarations_config_pre + .output + .clone() + .unwrap_or_else(|| Path::new("src/declarations").join(canister.0)); + let bindings = declarations_config_pre + .bindings + .clone() // probably, inefficient + .unwrap_or_else(|| vec!["js".to_string(), "ts".to_string(), "did".to_string()]); + if !bindings.is_empty() { + let deps = bindings.iter().map(|lang| { + match lang.as_str() { + "did" => vec![format!("{}.did", canister.0)], + "mo" => vec![format!("{}.mo", canister.0)], + "rs" => vec![], // TODO + "js" => vec![format!("{}.did.js", canister.0), "index.js".to_string()], + "ts" => vec![format!("{}.did.d.ts", canister.0), "index.d.ts".to_string()], + _ => panic!("unknown canister type: {}", canister.0.as_str()), + } + }).flatten().map(|path| format!("{}", output.join(path).to_str().unwrap().to_string())).join(" "); // TODO: `unwrap` + if let CanisterTypeProperties::Custom { .. } = &canister.1.type_specific { + // TODO + } else { + output_file.write_fmt(format_args!( + "generate@{}: \\\n {}\n\n", + canister.0, + deps, + ))?; + output_file.write_fmt(format_args!( + "{}: {}\n\t{} {}\n\n", + deps, + format!(".dfx/$(NETWORK)/canisters/{}/{}.did", canister.0, canister.0), + "dfx generate --no-compile --network $(NETWORK)", + canister.0, + ))?; + } + } + }; + } + None => {} + }; + + for edge in graph.edge_references() { + let target_value = graph.node_weight(edge.target()).unwrap(); + if let Import::Lib(_) = target_value { + // Unused, because package manager never update existing files (but create new dirs) + } else { + output_file.write_fmt(format_args!( + "{}: {}\n", + make_target(&pool, graph, edge.source())?, + make_target(&pool, graph, edge.target())?, + ))?; + } + } + for node in graph0.nodes() { + let command = get_build_command(&pool, graph, *node.1); + if let Import::Canister(canister_name) = node.0 { + let canister: std::sync::Arc = pool.get_first_canister_with_name(&canister_name).unwrap(); + if let Some(command) = command { + let target = make_target(&pool, graph, *node.1)?; + if canister.as_ref().get_info().is_assets() { + // We don't support generating dependencies for assets, + // so recompile it every time: + output_file.write_fmt(format_args!(".PHONY: {}\n", target))?; + } + output_file.write_fmt(format_args!("{}:\n\t{}\n\n", target, command))?; + } + output_file.write_fmt(format_args!("\ndeploy-self@{}: canister@{}\n", canister_name, canister_name))?; + let deps = canister.as_ref().get_info().get_dependencies(); + output_file.write_fmt(format_args!( // TODO: Use `canister install` instead. + "\tdfx deploy --no-compile --network $(NETWORK) $(DEPLOY_FLAGS) $(DEPLOY_FLAGS.{}) {}\n\n", canister_name, canister_name + ))?; + // If the canister is assets, add `generate@` dependencies. + if canister.as_ref().get_info().is_assets() { + if !deps.is_empty() { + output_file.write_fmt(format_args!( + "\ncanister@{}: \\\n {}\n", + canister_name, + deps.iter().map(|name| format!("generate@{}", name)).join(" "), + ))?; + } + } + if deps.is_empty() { + output_file.write_fmt(format_args!("deploy@{}: deploy-self@{}\n\n", canister_name, canister_name))?; + } else { + output_file.write_fmt(format_args!( + "deploy@{}: {} \\\n deploy-self@{}\n\n", + canister_name, + deps.iter().map(|name| format!("deploy@{}", name)).join(" "), + canister_name, + ))?; + } + } + } + + Ok(()) +} + +fn make_target(pool: &CanisterPool, graph: &Graph, node_id: as GraphBase>::NodeId) -> DfxResult { + let node_value = graph.node_weight(node_id).unwrap(); + Ok(match node_value { + Import::Canister(canister_name) => { + // duplicate code + let canister: std::sync::Arc = pool.get_first_canister_with_name(&canister_name).unwrap(); + if canister.get_info().is_assets() { + let path1 = format!(".dfx/$(NETWORK)/canisters/{}/assetstorage.wasm.gz", canister_name); + // let path2 = format!(".dfx/$(NETWORK)/canisters/{}/assetstorage.did", canister_name); + path1 + } else if canister.get_info().is_custom() { + // let is_gzip = canister.get_info().get_gzip(); // produces `false`, even if `"wasm"` is compressed. + let is_gzip = // hack + if let CanisterTypeProperties::Custom { wasm, .. } = &canister.get_info().get_type_specific_properties() { + wasm.ends_with(".gz") + } else { + canister.get_info().get_gzip() + }; + let path1 = if is_gzip { + format!(".dfx/$(NETWORK)/canisters/{}/{}.wasm.gz", canister_name, canister_name) + } else { + format!(".dfx/$(NETWORK)/canisters/{}/{}.wasm", canister_name, canister_name) + }; + let path2 = format!(".dfx/$(NETWORK)/canisters/{}/{}.did", canister_name, canister_name); + format!("{} {}", path1, path2) + } else { + let path1 = format!(".dfx/$(NETWORK)/canisters/{}/{}.wasm", canister_name, canister_name); + let path2 = format!(".dfx/$(NETWORK)/canisters/{}/{}.did", canister_name, canister_name); + format!("{} {}", path1, path2) + } + } + Import::Path(path) => format!("{}", path.to_str().unwrap_or("").to_owned()), // TODO: is a hack + Import::Ic(canister_name) => format!("canister@{}", canister_name), + Import::Lib(_path) => "".to_string(), + }) +} + +fn get_build_command(_pool: &CanisterPool, graph: &Graph, node_id: as GraphBase>::NodeId) -> Option { + let node_value = graph.node_weight(node_id).unwrap(); + match node_value { + Import::Canister(canister_name) => { + Some(format!("dfx canister create --network $(NETWORK) {}\n\tdfx build --no-deps --network $(NETWORK) {}", canister_name, canister_name)) + } + Import::Ic(_canister_name) => None, + Import::Path(_path) => None, + Import::Lib(_path) => None, + } +} \ No newline at end of file diff --git a/src/dfx/src/config/cache.rs b/src/dfx/src/config/cache.rs index e54c4a3962..0f5544d382 100644 --- a/src/dfx/src/config/cache.rs +++ b/src/dfx/src/config/cache.rs @@ -1,5 +1,6 @@ use crate::config::dfx_version; use crate::lib::environment::Environment; +use crate::lib::error::DfxResult; use crate::lib::progress_bar::ProgressBar; use crate::util; use dfx_core; @@ -70,9 +71,19 @@ impl VersionCache { &self, env: &dyn Environment, binary_name: &str, - ) -> Result { + ) -> DfxResult { Self::install(env, &self.version_str())?; - let path = binary_command_from_version(&self.version_str(), binary_name)?; + let mut path = binary_command_from_version(&self.version_str(), binary_name)?; + + // We need to ensure that it works well with relative paths: + let config = env.get_config()?; + if let Some(config) = config { + let workspace_root = config.get_path().parent(); + if let Some(workspace_root) = workspace_root { + path.current_dir(workspace_root); + } + } + Ok(path) } } diff --git a/src/dfx/src/lib/builders/assets.rs b/src/dfx/src/lib/builders/assets.rs index 16b0e85f01..aee30a7bee 100644 --- a/src/dfx/src/lib/builders/assets.rs +++ b/src/dfx/src/lib/builders/assets.rs @@ -32,19 +32,23 @@ struct AssetsBuilderExtra { impl AssetsBuilderExtra { #[context("Failed to create AssetBuilderExtra for canister '{}'.", info.get_name())] - fn try_from(info: &CanisterInfo, pool: &CanisterPool) -> DfxResult { - let dependencies = info.get_dependencies() - .iter() - .map(|name| { - pool.get_first_canister_with_name(name) - .map(|c| c.canister_id()) - .map_or_else( - || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), - DfxResult::Ok, - ) - }) - .collect::>>().with_context( || format!("Failed to collect dependencies (canister ids) of canister {}.", info.get_name()))?; - let info = info.as_info::()?; + fn try_from(info: &CanisterInfo, pool: &CanisterPool, no_deps: bool) -> DfxResult { + let dependencies = if no_deps { + Vec::new() + } else { + info.get_dependencies() + .iter() + .map(|name| { + pool.get_first_canister_with_name(name) + .map(|c| c.canister_id()) + .map_or_else( + || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), + DfxResult::Ok, + ) + }) + .collect::>>().with_context( || format!("Failed to collect dependencies (canister ids) of canister {}.", info.get_name()))? + }; + let info = info.as_info::()?; let build = info.get_build_tasks().to_owned(); let workspace = info.get_npm_workspace().map(str::to_owned); @@ -79,8 +83,9 @@ impl CanisterBuilder for AssetsBuilder { _: &dyn Environment, pool: &CanisterPool, info: &CanisterInfo, + no_deps: bool, ) -> DfxResult> { - Ok(AssetsBuilderExtra::try_from(info, pool)?.dependencies) + Ok(AssetsBuilderExtra::try_from(info, pool, no_deps)?.dependencies) } #[context("Failed to build asset canister '{}'.", info.get_name())] @@ -90,6 +95,7 @@ impl CanisterBuilder for AssetsBuilder { _pool: &CanisterPool, info: &CanisterInfo, _config: &BuildConfig, + _no_deps: bool, ) -> DfxResult { let wasm_path = info .get_output_root() @@ -111,12 +117,13 @@ impl CanisterBuilder for AssetsBuilder { pool: &CanisterPool, info: &CanisterInfo, config: &BuildConfig, + no_deps: bool, ) -> DfxResult { let AssetsBuilderExtra { build, dependencies, workspace, - } = AssetsBuilderExtra::try_from(info, pool)?; + } = AssetsBuilderExtra::try_from(info, pool, no_deps)?; let vars = super::get_and_write_environment_variables( info, diff --git a/src/dfx/src/lib/builders/custom.rs b/src/dfx/src/lib/builders/custom.rs index 5895c3ce01..40698ed48d 100644 --- a/src/dfx/src/lib/builders/custom.rs +++ b/src/dfx/src/lib/builders/custom.rs @@ -33,18 +33,22 @@ struct CustomBuilderExtra { impl CustomBuilderExtra { #[context("Failed to create CustomBuilderExtra for canister '{}'.", info.get_name())] - fn try_from(info: &CanisterInfo, pool: &CanisterPool) -> DfxResult { - let dependencies = info.get_dependencies() - .iter() - .map(|name| { - pool.get_first_canister_with_name(name) - .map(|c| c.canister_id()) - .map_or_else( - || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), - DfxResult::Ok, - ) - }) - .collect::>>().with_context( || format!("Failed to collect dependencies (canister ids) of canister {}.", info.get_name()))?; + fn try_from(info: &CanisterInfo, pool: &CanisterPool, no_deps: bool) -> DfxResult { + let dependencies = if no_deps { + Vec::new() + } else { + info.get_dependencies() + .iter() + .map(|name| { + pool.get_first_canister_with_name(name) + .map(|c| c.canister_id()) + .map_or_else( + || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), + DfxResult::Ok, + ) + }) + .collect::>>().with_context( || format!("Failed to collect dependencies (canister ids) of canister {}.", info.get_name()))? + }; let info = info.as_info::()?; let input_wasm_url = info.get_input_wasm_url().to_owned(); let wasm = info.get_output_wasm_path().to_owned(); @@ -87,17 +91,19 @@ impl CanisterBuilder for CustomBuilder { _: &dyn Environment, pool: &CanisterPool, info: &CanisterInfo, + no_deps: bool, ) -> DfxResult> { - Ok(CustomBuilderExtra::try_from(info, pool)?.dependencies) + Ok(CustomBuilderExtra::try_from(info, pool, no_deps)?.dependencies) } #[context("Failed to build custom canister {}.", info.get_name())] fn build( &self, - _: &dyn Environment, + _env: &dyn Environment, pool: &CanisterPool, info: &CanisterInfo, config: &BuildConfig, + no_deps: bool, ) -> DfxResult { let CustomBuilderExtra { input_candid_url: _, @@ -105,7 +111,7 @@ impl CanisterBuilder for CustomBuilder { wasm, build, dependencies, - } = CustomBuilderExtra::try_from(info, pool)?; + } = CustomBuilderExtra::try_from(info, pool, no_deps)?; let canister_id = info.get_canister_id().unwrap(); let vars = super::get_and_write_environment_variables( @@ -155,7 +161,7 @@ pub async fn custom_download(info: &CanisterInfo, pool: &CanisterPool) -> DfxRes wasm, build: _, dependencies: _, - } = CustomBuilderExtra::try_from(info, pool)?; + } = CustomBuilderExtra::try_from(info, pool, true)?; if let Some(url) = input_wasm_url { download_file_to_path(&url, &wasm).await?; diff --git a/src/dfx/src/lib/builders/mod.rs b/src/dfx/src/lib/builders/mod.rs index 32eaf1ca96..c1d0cc6da5 100644 --- a/src/dfx/src/lib/builders/mod.rs +++ b/src/dfx/src/lib/builders/mod.rs @@ -1,10 +1,11 @@ use crate::config::dfx_version_str; +use crate::lib::canister_info::motoko::MotokoCanisterInfo; use crate::lib::canister_info::CanisterInfo; use crate::lib::environment::Environment; use crate::lib::error::{BuildError, DfxError, DfxResult}; -use crate::lib::models::canister::CanisterPool; +use crate::lib::models::canister::{CanisterPool, Import}; use crate::util::command::direct_or_shell_command; -use anyhow::{bail, Context}; +use anyhow::{bail, Context, anyhow}; use candid::Principal as CanisterId; use candid_parser::utils::CandidSource; use dfx_core::config::model::dfinity::{Config, Profile}; @@ -24,7 +25,7 @@ use std::process::Stdio; use std::sync::Arc; mod assets; -mod custom; +pub mod custom; // TODO: Shouldn't be `pub`. mod motoko; mod pull; mod rust; @@ -62,10 +63,25 @@ pub trait CanisterBuilder { _env: &dyn Environment, _pool: &CanisterPool, _info: &CanisterInfo, + _no_deps: bool, ) -> DfxResult> { Ok(Vec::new()) } + fn maybe_get_dependencies( + &self, + env: &dyn Environment, + pool: &CanisterPool, + info: &CanisterInfo, + no_deps: bool, + ) -> DfxResult> { + if no_deps { + Ok(Vec::new()) + } else { + self.get_dependencies(env, pool, info, no_deps) + } + } + fn prebuild( &self, _env: &dyn Environment, @@ -84,6 +100,7 @@ pub trait CanisterBuilder { pool: &CanisterPool, info: &CanisterInfo, config: &BuildConfig, + no_deps: bool, ) -> DfxResult; fn postbuild( @@ -92,6 +109,7 @@ pub trait CanisterBuilder { _pool: &CanisterPool, _info: &CanisterInfo, _config: &BuildConfig, + _no_deps: bool, ) -> DfxResult { Ok(()) } @@ -230,6 +248,125 @@ pub trait CanisterBuilder { Ok(()) } + #[context("Failed to find imports for canister '{}'.", info.get_name())] + fn read_dependencies( + &self, + env: &dyn Environment, + pool: &CanisterPool, + info: &CanisterInfo, + ) -> DfxResult { + #[context("Failed recursive dependency detection at {}.", parent)] + fn read_dependencies_recursive( + env: &dyn Environment, + pool: &CanisterPool, + parent: &Import, + ) -> DfxResult { + if env.get_imports().borrow().nodes().contains_key(parent) { + // The item and its descendants are already in the graph. + return Ok(()); + } + let parent_node_index = env.get_imports().borrow_mut().update_node(parent); + + let file = match parent { + Import::Canister(parent_name) => { + let parent_canister = pool.get_first_canister_with_name(parent_name) + .ok_or_else(|| anyhow!("No such canister {}", parent_name))?; + let parent_canister_info = parent_canister.get_info(); + if parent_canister_info.is_motoko() { + let motoko_info = parent_canister + .get_info() + .as_info::() + .context("Getting Motoko info")?; + Some( + motoko_info + .get_main_path() + .to_path_buf() + // .canonicalize() + // .with_context(|| { + // format!( + // "Canonicalizing Motoko path {}", + // motoko_info.get_main_path().to_string_lossy() + // ) + // })?, + ) + } else { + for child in parent_canister_info.get_dependencies() { + read_dependencies_recursive( + env, + pool, + &Import::Canister(child.clone()), + )?; + + let child_node = Import::Canister(child.clone()); + let child_node_index = + env.get_imports().borrow_mut().update_node(&child_node); + env.get_imports().borrow_mut().update_edge( + parent_node_index, + child_node_index, + (), + ); + } + return Ok(()); + } + } + Import::Path(path) => Some(path.clone()), + _ => None, + }; + if let Some(file) = file { + let mut command = env.get_cache() + .get_binary_command(env, "moc") + .context("Getting binary command \"moc\"")?; + let command = command.arg("--print-deps").arg(file); + let output = command + .output() + .with_context(|| format!("Error executing {:#?}", command))?; + let output = String::from_utf8_lossy(&output.stdout); + + for line in output.lines() { + let child = Import::try_from(line).context("Failed to create MotokoImport.")?; + match &child { + Import::Canister(_) | Import::Path(_) => { + read_dependencies_recursive(env, pool, &child)? + } + _ => {} + } + let child_node_index = env.get_imports().borrow_mut().update_node(&child); + env.get_imports().borrow_mut().update_edge( + parent_node_index, + child_node_index, + (), + ); + } + } + + Ok(()) + } + + read_dependencies_recursive( + env, + pool, + &Import::Canister(info.get_name().to_string()), + )?; + + Ok(()) + } + + #[context("Failed to finread canister dependencues.")] + fn read_all_dependencies( + &self, + env: &dyn Environment, + pool: &CanisterPool, + ) -> DfxResult { + // TODO: several `unwrap`s in this function + // TODO: It should be simpler. + let config = env.get_config()?; + for canister_name in config.as_ref().unwrap().get_config().canisters.as_ref().unwrap().keys() { + let canister = pool.get_first_canister_with_name(&canister_name).unwrap(); // TODO + self.read_dependencies(env, pool, canister.get_info())?; + } + Ok(()) + } + /// Get the path to the provided candid file for the canister. /// No need to guarantee the file exists, as the caller will handle that. fn get_candid_path( diff --git a/src/dfx/src/lib/builders/motoko.rs b/src/dfx/src/lib/builders/motoko.rs index 5f0ce7fbec..91e28958ba 100644 --- a/src/dfx/src/lib/builders/motoko.rs +++ b/src/dfx/src/lib/builders/motoko.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use crate::config::cache::VersionCache; use crate::lib::builders::{ BuildConfig, BuildOutput, CanisterBuilder, IdlBuildOutput, WasmBuildOutput, @@ -7,7 +8,7 @@ use crate::lib::canister_info::CanisterInfo; use crate::lib::environment::Environment; use crate::lib::error::{BuildError, DfxError, DfxResult}; use crate::lib::metadata::names::{CANDID_ARGS, CANDID_SERVICE}; -use crate::lib::models::canister::CanisterPool; +use crate::lib::models::canister::{CanisterPool, Import}; use crate::lib::package_arguments::{self, PackageArguments}; use crate::util::assets::management_idl; use anyhow::Context; @@ -15,9 +16,7 @@ use candid::Principal as CanisterId; use dfx_core::config::model::dfinity::{MetadataVisibility, Profile}; use fn_error_context::context; use slog::{info, o, trace, warn, Logger}; -use std::collections::{BTreeMap, BTreeSet}; -use std::convert::TryFrom; -use std::fmt::Debug; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Output; @@ -40,60 +39,60 @@ impl MotokoBuilder { } } -#[context("Failed to find imports for canister at '{}'.", info.get_main_path().display())] -fn get_imports( - env: &dyn Environment, - cache: &VersionCache, - info: &MotokoCanisterInfo, -) -> DfxResult> { - #[context("Failed recursive dependency detection at {}.", file.display())] - fn get_imports_recursive( - env: &dyn Environment, - cache: &VersionCache, - workspace_root: &Path, - file: &Path, - result: &mut BTreeSet, - ) -> DfxResult { - if result.contains(&MotokoImport::Relative(file.to_path_buf())) { - return Ok(()); - } - - result.insert(MotokoImport::Relative(file.to_path_buf())); - - let mut command = cache.get_binary_command(env, "moc")?; - command.current_dir(workspace_root); - let command = command.arg("--print-deps").arg(file); - let output = command - .output() - .with_context(|| format!("Error executing {:#?}", command))?; - let output = String::from_utf8_lossy(&output.stdout); - - for line in output.lines() { - let import = MotokoImport::try_from(line).context("Failed to create MotokoImport.")?; - match import { - MotokoImport::Relative(path) => { - get_imports_recursive(env, cache, workspace_root, path.as_path(), result)?; - } - _ => { - result.insert(import); - } - } - } - - Ok(()) - } - - let mut result = BTreeSet::new(); - get_imports_recursive( - env, - cache, - info.get_workspace_root(), - info.get_main_path(), - &mut result, - )?; - - Ok(result) -} +// #[context("Failed to find imports for canister at '{}'.", info.get_main_path().display())] +// fn get_imports( +// env: &dyn Environment, +// cache: &VersionCache, +// info: &MotokoCanisterInfo, +// ) -> DfxResult> { +// #[context("Failed recursive dependency detection at {}.", file.display())] +// fn get_imports_recursive( +// env: &dyn Environment, +// cache: &VersionCache, +// workspace_root: &Path, +// file: &Path, +// result: &mut BTreeSet, +// ) -> DfxResult { +// if result.contains(&MotokoImport::Relative(file.to_path_buf())) { +// return Ok(()); +// } + +// result.insert(MotokoImport::Relative(file.to_path_buf())); + +// let mut command = cache.get_binary_command(env, "moc")?; +// command.current_dir(workspace_root); +// let command = command.arg("--print-deps").arg(file); +// let output = command +// .output() +// .with_context(|| format!("Error executing {:#?}", command))?; +// let output = String::from_utf8_lossy(&output.stdout); + +// for line in output.lines() { +// let import = MotokoImport::try_from(line).context("Failed to create MotokoImport.")?; +// match import { +// MotokoImport::Relative(path) => { +// get_imports_recursive(env, cache, workspace_root, path.as_path(), result)?; +// } +// _ => { +// result.insert(import); +// } +// } +// } + +// Ok(()) +// } + +// let mut result = BTreeSet::new(); +// get_imports_recursive( +// env, +// cache, +// info.get_workspace_root(), +// info.get_main_path(), +// &mut result, +// )?; + +// Ok(result) +// } impl CanisterBuilder for MotokoBuilder { #[context("Failed to get dependencies for canister '{}'.", info.get_name())] @@ -102,21 +101,51 @@ impl CanisterBuilder for MotokoBuilder { env: &dyn Environment, pool: &CanisterPool, info: &CanisterInfo, + no_deps: bool, ) -> DfxResult> { - let motoko_info = info.as_info::()?; - let imports = get_imports(env, &self.cache, &motoko_info)?; + if no_deps { + return Ok(Vec::new()) + } - Ok(imports - .iter() - .filter_map(|import| { - if let MotokoImport::Canister(name) = import { - pool.get_first_canister_with_name(name) + self.read_dependencies(env, pool, info)?; + + let imports = env.get_imports().borrow(); + let graph = imports.graph(); + // let space = DfsSpace::new(&graph); + // match petgraph::algo::toposort(graph, Some(&mut space)) { + // TODO: inefficient: + match petgraph::algo::toposort(graph, None) { + Ok(order) => { + let res: Vec<_> = order + .into_iter() + .filter_map(|id| match graph.node_weight(id) { + Some(Import::Canister(name)) => { + pool.get_first_canister_with_name(name.as_str()) // TODO: a little inefficient + } + _ => None, + }) + .map(|canister| canister.canister_id()) + .collect(); + let main_canister_id = info.get_canister_id()?; + if let Some(start_index) = res.iter().position(|&x| x == main_canister_id) { + // Create a slice starting from that index + let slice = &res[start_index+1..]; + Ok(slice.to_vec()) } else { - None + panic!("Programming error"); } - }) - .map(|canister| canister.canister_id()) - .collect()) + } + Err(err) => { + let message = match graph.node_weight(err.node_id()) { + Some(Import::Canister(name)) => name.clone(), + _ => "".to_string(), + }; + return Err(DfxError::new(BuildError::DependencyError(format!( + "Found circular dependency: {}", + message + )))); + } + } } #[context("Failed to build Motoko canister '{}'.", canister_info.get_name())] @@ -126,6 +155,7 @@ impl CanisterBuilder for MotokoBuilder { pool: &CanisterPool, canister_info: &CanisterInfo, config: &BuildConfig, + no_deps: bool, ) -> DfxResult { let motoko_info = canister_info.as_info::()?; let profile = config.profile; @@ -150,15 +180,14 @@ impl CanisterBuilder for MotokoBuilder { .with_context(|| format!("Failed to create {}.", idl_dir_path.to_string_lossy()))?; // If the management canister is being imported, emit the candid file. - if get_imports(env, cache, &motoko_info)? - .contains(&MotokoImport::Ic("aaaaa-aa".to_string())) + if env.get_imports().borrow().nodes().keys().contains(&Import::Canister("aaaaa-aa".to_string())) { let management_idl_path = idl_dir_path.join("aaaaa-aa.did"); dfx_core::fs::write(management_idl_path, management_idl()?)?; } let dependencies = self - .get_dependencies(env, pool, canister_info) + .maybe_get_dependencies(env, pool, canister_info, no_deps) .unwrap_or_default(); super::get_and_write_environment_variables( canister_info, @@ -295,76 +324,6 @@ fn motoko_compile( Ok(()) } -#[derive(Debug, PartialOrd, Ord, PartialEq, Eq)] -enum MotokoImport { - Canister(String), - Ic(String), - Lib(String), - Relative(PathBuf), -} - -impl TryFrom<&str> for MotokoImport { - type Error = DfxError; - - fn try_from(line: &str) -> Result { - let (url, fullpath) = match line.find(' ') { - Some(index) => { - if index >= line.len() - 1 { - return Err(DfxError::new(BuildError::DependencyError(format!( - "Unknown import {}", - line - )))); - } - let (url, fullpath) = line.split_at(index + 1); - (url.trim_end(), Some(fullpath)) - } - None => (line, None), - }; - let import = match url.find(':') { - Some(index) => { - if index >= line.len() - 1 { - return Err(DfxError::new(BuildError::DependencyError(format!( - "Unknown import {}", - url - )))); - } - let (prefix, name) = url.split_at(index + 1); - match prefix { - "canister:" => MotokoImport::Canister(name.to_owned()), - "ic:" => MotokoImport::Ic(name.to_owned()), - "mo:" => MotokoImport::Lib(name.to_owned()), - _ => { - return Err(DfxError::new(BuildError::DependencyError(format!( - "Unknown import {}", - url - )))) - } - } - } - None => match fullpath { - Some(fullpath) => { - let path = PathBuf::from(fullpath); - if !path.is_file() { - return Err(DfxError::new(BuildError::DependencyError(format!( - "Cannot find import file {}", - path.display() - )))); - }; - MotokoImport::Relative(path) - } - None => { - return Err(DfxError::new(BuildError::DependencyError(format!( - "Cannot resolve relative import {}", - url - )))) - } - }, - }; - - Ok(import) - } -} - fn run_command( logger: &slog::Logger, cmd: &mut std::process::Command, diff --git a/src/dfx/src/lib/builders/pull.rs b/src/dfx/src/lib/builders/pull.rs index e63e59e8f0..5f712e048e 100644 --- a/src/dfx/src/lib/builders/pull.rs +++ b/src/dfx/src/lib/builders/pull.rs @@ -33,6 +33,7 @@ impl CanisterBuilder for PullBuilder { _: &dyn Environment, _pool: &CanisterPool, info: &CanisterInfo, + _no_deps: bool, ) -> DfxResult> { Ok(vec![]) } @@ -44,6 +45,7 @@ impl CanisterBuilder for PullBuilder { _pool: &CanisterPool, canister_info: &CanisterInfo, _config: &BuildConfig, + _no_deps: bool, ) -> DfxResult { let pull_info = canister_info.as_info::()?; Ok(BuildOutput { diff --git a/src/dfx/src/lib/builders/rust.rs b/src/dfx/src/lib/builders/rust.rs index 3a5d03051e..df6eede5fa 100644 --- a/src/dfx/src/lib/builders/rust.rs +++ b/src/dfx/src/lib/builders/rust.rs @@ -36,18 +36,23 @@ impl CanisterBuilder for RustBuilder { _: &dyn Environment, pool: &CanisterPool, info: &CanisterInfo, + no_deps: bool, ) -> DfxResult> { - let dependencies = info.get_dependencies() - .iter() - .map(|name| { - pool.get_first_canister_with_name(name) - .map(|c| c.canister_id()) - .map_or_else( - || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), - DfxResult::Ok, - ) - }) - .collect::>>().with_context(|| format!("Failed to collect dependencies (canister ids) for canister {}.", info.get_name()))?; + let dependencies = if no_deps { + info.get_dependencies() + .iter() + .map(|name| { + pool.get_first_canister_with_name(name) + .map(|c| c.canister_id()) + .map_or_else( + || Err(anyhow!("A canister with the name '{}' was not found in the current project.", name.clone())), + DfxResult::Ok, + ) + }) + .collect::>>().with_context(|| format!("Failed to collect dependencies (canister ids) for canister {}.", info.get_name()))? + } else { + Vec::new() + }; Ok(dependencies) } @@ -58,6 +63,7 @@ impl CanisterBuilder for RustBuilder { pool: &CanisterPool, canister_info: &CanisterInfo, config: &BuildConfig, + no_deps: bool, ) -> DfxResult { let rust_info = canister_info.as_info::()?; let package = rust_info.get_package(); @@ -78,7 +84,7 @@ impl CanisterBuilder for RustBuilder { .arg("--locked"); let dependencies = self - .get_dependencies(env, pool, canister_info) + .maybe_get_dependencies(env, pool, canister_info, no_deps) .unwrap_or_default(); let vars = super::get_and_write_environment_variables( canister_info, diff --git a/src/dfx/src/lib/canister_info.rs b/src/dfx/src/lib/canister_info.rs index b217041d91..9ee5b3040e 100644 --- a/src/dfx/src/lib/canister_info.rs +++ b/src/dfx/src/lib/canister_info.rs @@ -124,6 +124,7 @@ impl CanisterInfo { // Fill the default config values if None provided let declarations_config = CanisterDeclarationsConfig { + // duplicate code: output: declarations_config_pre .output .or_else(|| Some(workspace_root.join("src/declarations").join(name))), diff --git a/src/dfx/src/lib/canister_info/motoko.rs b/src/dfx/src/lib/canister_info/motoko.rs index 97b964d850..6cab03e2e2 100644 --- a/src/dfx/src/lib/canister_info/motoko.rs +++ b/src/dfx/src/lib/canister_info/motoko.rs @@ -65,7 +65,7 @@ impl CanisterInfoFactory for MotokoCanisterInfo { let main_path = info .get_main_file() .context("`main` attribute is required on Motoko canisters in dfx.json (and Motoko is the default canister type if not otherwise specified)")?; - let input_path = workspace_root.join(main_path); + let input_path = main_path.to_path_buf(); let output_root = info.get_output_root().to_path_buf(); let output_wasm_path = output_root.join(name).with_extension("wasm"); let output_stable_path = output_wasm_path.with_extension("most"); diff --git a/src/dfx/src/lib/environment.rs b/src/dfx/src/lib/environment.rs index 8626e358ed..9fc3c640e7 100644 --- a/src/dfx/src/lib/environment.rs +++ b/src/dfx/src/lib/environment.rs @@ -26,6 +26,8 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use url::Url; +use super::graph::graph_nodes_map::GraphWithNodesMap; +use super::models::canister::Import; pub trait Environment { fn get_cache(&self) -> VersionCache; @@ -88,6 +90,8 @@ pub trait Environment { self.get_config()?, ) } + + fn get_imports(&self) -> &RefCell>; } pub enum ProjectConfig { @@ -114,6 +118,10 @@ pub struct EnvironmentImpl { effective_canister_id: Option, extension_manager: ExtensionManager, + + /// Graph currently read imports and their children, not necessarily the entire graph of all imports. + /// Invariant: with each node contains all its descendants. + imports: RefCell>, } impl EnvironmentImpl { @@ -131,6 +139,7 @@ impl EnvironmentImpl { identity_override: None, effective_canister_id: None, extension_manager, + imports: RefCell::new(GraphWithNodesMap::new()), spinners: MultiProgress::new(), }) } @@ -279,6 +288,10 @@ impl Environment for EnvironmentImpl { fn get_extension_manager(&self) -> &ExtensionManager { &self.extension_manager } + + fn get_imports(&self) -> &RefCell> { + &self.imports + } } pub struct AgentEnvironment<'a> { @@ -287,6 +300,7 @@ pub struct AgentEnvironment<'a> { pocketic: Option, network_descriptor: NetworkDescriptor, identity_manager: IdentityManager, + imports: RefCell>, effective_canister_id: Option, } @@ -355,6 +369,7 @@ impl<'a> AgentEnvironment<'a> { pocketic, network_descriptor: network_descriptor.clone(), identity_manager, + imports: RefCell::new(GraphWithNodesMap::new()), effective_canister_id, }) } @@ -443,6 +458,10 @@ impl<'a> Environment for AgentEnvironment<'a> { fn get_extension_manager(&self) -> &ExtensionManager { self.backend.get_extension_manager() } + + fn get_imports(&self) -> &RefCell> { + &self.imports + } } #[context("Failed to create agent with url {}.", url)] @@ -498,6 +517,9 @@ pub mod test_env { fn get_identity_override(&self) -> Option<&str> { None } + fn get_imports(&self) -> &RefCell> { + unimplemented!() + } fn get_logger(&self) -> &slog::Logger { unimplemented!() } diff --git a/src/dfx/src/lib/graph/graph_nodes_map.rs b/src/dfx/src/lib/graph/graph_nodes_map.rs new file mode 100644 index 0000000000..c588ea1d3c --- /dev/null +++ b/src/dfx/src/lib/graph/graph_nodes_map.rs @@ -0,0 +1,54 @@ +// TODO: Integrate it into `petgraph` library. + +use std::collections::HashMap; +use std::hash::Hash; + +use petgraph::{ + graph::EdgeIndex, + graph::IndexType, + graph::{DefaultIx, NodeIndex}, + Directed, EdgeType, Graph, +}; + +pub struct GraphWithNodesMap { + graph: Graph, + nodes: HashMap>, +} + +impl GraphWithNodesMap { + pub fn graph(&self) -> &Graph { + &self.graph + } + pub fn nodes(&self) -> &HashMap> { + &self.nodes + } +} + +impl GraphWithNodesMap +where + Ty: EdgeType, + Ix: IndexType, +{ + pub fn update_node(&mut self, weight: &N) -> NodeIndex + where + N: Eq + Hash + Clone, + { + // TODO: Get rid of two `clone`s (apparently, requires data stucture change). + *self + .nodes + .entry(weight.clone()) + .or_insert_with(|| self.graph.add_node(weight.clone())) + } + pub fn update_edge(&mut self, a: NodeIndex, b: NodeIndex, weight: E) -> EdgeIndex { + self.graph.update_edge(a, b, weight) + } +} + +impl GraphWithNodesMap { + pub fn new() -> Self { + Self { + graph: Graph::new(), + nodes: HashMap::new(), + } + } +} diff --git a/src/dfx/src/lib/graph/mod.rs b/src/dfx/src/lib/graph/mod.rs new file mode 100644 index 0000000000..2c9aa8480f --- /dev/null +++ b/src/dfx/src/lib/graph/mod.rs @@ -0,0 +1 @@ +pub mod graph_nodes_map; diff --git a/src/dfx/src/lib/mod.rs b/src/dfx/src/lib/mod.rs index a8d4f0926e..cb4aecdfa9 100644 --- a/src/dfx/src/lib/mod.rs +++ b/src/dfx/src/lib/mod.rs @@ -9,6 +9,7 @@ pub mod diagnosis; pub mod environment; pub mod error; pub mod error_code; +pub mod graph; pub mod ic_attributes; pub mod identity; pub mod info; diff --git a/src/dfx/src/lib/models/canister.rs b/src/dfx/src/lib/models/canister.rs index 0084143514..aa1cb4f01f 100644 --- a/src/dfx/src/lib/models/canister.rs +++ b/src/dfx/src/lib/models/canister.rs @@ -27,8 +27,9 @@ use std::cell::RefCell; use std::collections::{BTreeMap, HashSet}; use std::convert::TryFrom; use std::ffi::OsStr; +use std::fmt::Display; use std::io::Read; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::Arc; @@ -70,8 +71,9 @@ impl Canister { env: &dyn Environment, pool: &CanisterPool, build_config: &BuildConfig, + no_deps: bool, ) -> DfxResult<&BuildOutput> { - let output = self.builder.build(env, pool, &self.info, build_config)?; + let output = self.builder.build(env, pool, &self.info, build_config, no_deps)?; // Ignore the old output, and return a reference. let _ = self.output.replace(Some(output)); @@ -83,8 +85,9 @@ impl Canister { env: &dyn Environment, pool: &CanisterPool, build_config: &BuildConfig, + no_deps: bool, ) -> DfxResult { - self.builder.postbuild(env, pool, &self.info, build_config) + self.builder.postbuild(env, pool, &self.info, build_config, no_deps) } pub fn get_name(&self) -> &str { @@ -448,6 +451,88 @@ fn check_valid_subtype(compiled_idl_path: &Path, specified_idl_path: &Path) -> D Ok(()) } +/// Used mainly for Motoko +#[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub enum Import { + Canister(String), + Ic(String), + Lib(String), // TODO: Unused, because package manager never update existing files (but create new dirs) + Path(PathBuf), +} + +impl Display for Import { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Canister(name) => write!(f, "canister {}", name), + Self::Ic(principal) => write!(f, "principal {}", principal), + Self::Lib(name) => write!(f, "library {}", name), + Self::Path(file) => write!(f, "file {}", file.to_string_lossy()), + } + } +} + +impl TryFrom<&str> for Import { + type Error = DfxError; + + fn try_from(line: &str) -> Result { + let (url, fullpath) = match line.find(' ') { + Some(index) => { + if index >= line.len() - 1 { + return Err(DfxError::new(BuildError::DependencyError(format!( + "Unknown import {}", + line + )))); + } + let (url, fullpath) = line.split_at(index + 1); + (url.trim_end(), Some(fullpath)) + } + None => (line, None), + }; + let import = match url.find(':') { + Some(index) => { + if index >= line.len() - 1 { + return Err(DfxError::new(BuildError::DependencyError(format!( + "Unknown import {}", + url + )))); + } + let (prefix, name) = url.split_at(index + 1); + match prefix { + "canister:" => Import::Canister(name.to_owned()), + "ic:" => Import::Ic(name.to_owned()), + "mo:" => Import::Lib(name.to_owned()), + _ => { + return Err(DfxError::new(BuildError::DependencyError(format!( + "Unknown import {}", + url + )))) + } + } + } + None => match fullpath { + Some(fullpath) => { + let path = PathBuf::from(fullpath); + if !path.is_file() { + return Err(DfxError::new(BuildError::DependencyError(format!( + "Cannot find import file {}", + path.display() + )))); + }; + Import::Path(path) + } + None => { + return Err(DfxError::new(BuildError::DependencyError(format!( + "Cannot resolve relative import {}", + url + )))) + } + }, + }; + + Ok(import) + } +} + /// A canister pool is a list of canisters. pub struct CanisterPool { canisters: Vec>, @@ -548,6 +633,7 @@ impl CanisterPool { &self, env: &dyn Environment, canisters_to_build: Vec<&Canister>, + no_deps: bool, ) -> DfxResult> { let mut graph: DiGraph = DiGraph::new(); @@ -559,12 +645,13 @@ impl CanisterPool { /// /// Returns the index of the canister's graph node. fn add_canister_and_dependencies_to_graph( - env: &dyn Environment, canister_pool: &CanisterPool, + env: &dyn Environment, canister: &Canister, graph: &mut DiGraph, canister_id_to_canister: &BTreeMap, canister_id_to_index: &mut BTreeMap>, + no_deps: bool, ) -> DfxResult { let canister_id = canister.canister_id(); @@ -579,7 +666,7 @@ impl CanisterPool { let deps = canister .builder - .get_dependencies(env, canister_pool, &canister.info)?; + .maybe_get_dependencies(env, canister_pool, &canister.info, no_deps)?; for dependency_id in deps { let dependency = canister_id_to_canister.get(&dependency_id).ok_or_else(|| { @@ -590,12 +677,13 @@ impl CanisterPool { ))) })?; let dependency_index = add_canister_and_dependencies_to_graph( - env, canister_pool, + env, dependency, graph, canister_id_to_canister, canister_id_to_index, + no_deps, )?; graph.add_edge(node_ix, dependency_index, ()); } @@ -611,12 +699,13 @@ impl CanisterPool { .collect::>(); for canister in canisters_to_build { add_canister_and_dependencies_to_graph( - env, self, + env, canister, &mut graph, &canister_id_to_canister, &mut canister_id_to_index, + no_deps, )?; } @@ -709,8 +798,9 @@ impl CanisterPool { env: &dyn Environment, build_config: &BuildConfig, canister: &'a Canister, + no_deps: bool, ) -> DfxResult<&'a BuildOutput> { - canister.build(env, self, build_config) + canister.build(env, self, build_config, no_deps) } fn step_postbuild( @@ -719,6 +809,7 @@ impl CanisterPool { build_config: &BuildConfig, canister: &Canister, build_output: &BuildOutput, + no_deps: bool, ) -> DfxResult<()> { canister.candid_post_process(self.get_logger(), build_config, build_output)?; @@ -726,7 +817,7 @@ impl CanisterPool { build_canister_js(&canister.canister_id(), &canister.info)?; - canister.postbuild(env, self, build_config) + canister.postbuild(env, self, build_config, no_deps) } fn step_postbuild_all( @@ -755,12 +846,13 @@ impl CanisterPool { env: &dyn Environment, log: &Logger, build_config: &BuildConfig, + no_deps: bool, ) -> DfxResult>> { self.step_prebuild_all(log, build_config) .map_err(|e| DfxError::new(BuildError::PreBuildAllStepFailed(Box::new(e))))?; let canisters_to_build = self.canisters_to_build(build_config); - let graph = self.build_dependencies_graph(env, canisters_to_build.clone())?; + let graph = self.build_dependencies_graph(env, canisters_to_build.clone(), no_deps)?; let nodes = petgraph::algo::toposort(&graph, None).map_err(|cycle| { let message = match graph.node_weight(cycle.node_id()) { Some(canister_id) => match self.get_canister_info(canister_id) { @@ -801,7 +893,7 @@ impl CanisterPool { ) }) .and_then(|_| { - self.step_build(env, build_config, canister).map_err(|e| { + self.step_build(env, build_config, canister, no_deps).map_err(|e| { BuildError::BuildStepFailed( *canister_id, canister.get_name().to_string(), @@ -810,7 +902,7 @@ impl CanisterPool { }) }) .and_then(|o| { - self.step_postbuild(env, build_config, canister, o) + self.step_postbuild(env, build_config, canister, o, no_deps) .map_err(|e| { BuildError::PostBuildStepFailed( *canister_id, @@ -838,9 +930,10 @@ impl CanisterPool { env: &dyn Environment, log: &Logger, build_config: &BuildConfig, + no_deps: bool, ) -> DfxResult<()> { self.download().await?; - let outputs = self.build(env, log, build_config)?; + let outputs = self.build(env, log, build_config, no_deps)?; for output in outputs { output.map_err(DfxError::new)?; diff --git a/src/dfx/src/lib/operations/canister/deploy_canisters.rs b/src/dfx/src/lib/operations/canister/deploy_canisters.rs index bdc1c6ab3d..7538965d66 100644 --- a/src/dfx/src/lib/operations/canister/deploy_canisters.rs +++ b/src/dfx/src/lib/operations/canister/deploy_canisters.rs @@ -59,6 +59,7 @@ pub async fn deploy_canisters( no_asset_upgrade: bool, subnet_selection: &mut SubnetSelectionType, always_assist: bool, + no_compile: bool, ) -> DfxResult { let log = env.get_logger(); @@ -77,7 +78,13 @@ pub async fn deploy_canisters( } } - let canisters_to_deploy = canister_with_dependencies(&config, some_canister)?; + // TODO: The following below code is a mess. + + let canisters_to_deploy = if no_compile && some_canister.is_some() { + vec![some_canister.unwrap().to_string()] + } else { + canister_with_dependencies(&config, some_canister)? + }; let canisters_to_build = match deploy_mode { PrepareForProposal(canister_name) | ComputeEvidence(canister_name) => { @@ -99,11 +106,15 @@ pub async fn deploy_canisters( .collect(), }; - let canisters_to_install: Vec = canisters_to_build - .clone() - .into_iter() - .filter(|canister_name| !pull_canisters_in_config.contains_key(canister_name)) - .collect(); + let canisters_to_install: Vec = if no_compile && some_canister.is_some() { + vec![some_canister.unwrap().to_string()] + } else { + canisters_to_build + .clone() + .into_iter() + .filter(|canister_name| !pull_canisters_in_config.contains_key(canister_name)) + .collect() + }; if some_canister.is_some() { info!(log, "Deploying: {}", canisters_to_install.join(" ")); @@ -134,14 +145,23 @@ pub async fn deploy_canisters( let canisters_to_load = all_project_canisters_with_ids(env, &config); - let pool = build_canisters( - env, - &canisters_to_load, - &canisters_to_build, - &config, - env_file.clone(), - ) - .await?; + // TODO: For efficiency, also don't compute canisters order if `no_compile`. + let pool = if no_compile { + CanisterPool::load( + env, // if `env1`, fails with "NetworkDescriptor only available from an AgentEnvironment" + false, + &canisters_to_deploy, + )? + } else { + build_canisters( + env, + &canisters_to_load, + &if no_compile { canisters_to_build } else { Vec::new() }, + &config, + env_file.clone(), + no_compile, + ).await? + }; match deploy_mode { NormalDeploy | ForceReinstallSingleCanister(_) => { @@ -296,6 +316,7 @@ async fn build_canisters( canisters_to_build: &[String], config: &Config, env_file: Option, + no_deps: bool, ) -> DfxResult { let log = env.get_logger(); info!(log, "Building canisters..."); @@ -306,7 +327,7 @@ async fn build_canisters( BuildConfig::from_config(config, env.get_network_descriptor().is_playground())? .with_canisters_to_build(canisters_to_build.into()) .with_env_file(env_file); - canister_pool.build_or_fail(env, log, &build_config).await?; + canister_pool.build_or_fail(env, log, &build_config, no_deps).await?; Ok(canister_pool) }