From 871ab20757f457bda65878d8037f2e597060b6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= Date: Fri, 15 Nov 2024 16:15:43 +0200 Subject: [PATCH 1/6] Add initial support for operating with workflows Workflows are the main objects that provide the platform computation functionality. Workflow tasks convert on-chain into individual tasks, but the overall coordination of execution happens by the rules described in the workflow. This change adds basic CRUD support for workflows. --- Cargo.lock | 4 +- Cargo.toml | 4 +- src/commands/mod.rs | 1 + src/commands/workflows.rs | 97 +++++++++++++++++++++++++++++++++++++++ src/main.rs | 21 +-------- 5 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 src/commands/workflows.rs diff --git a/Cargo.lock b/Cargo.lock index 97d1c41..6ea316b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,7 +901,7 @@ dependencies = [ [[package]] name = "gevulot-rs" version = "0.1.0" -source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=6695dcb0e7214a3ec07cdc2d83bbc4ff26ae2d38#6695dcb0e7214a3ec07cdc2d83bbc4ff26ae2d38" +source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=e6c8f044596e566814076374430de65c281bff57#e6c8f044596e566814076374430de65c281bff57" dependencies = [ "backon", "bip32", @@ -958,7 +958,7 @@ dependencies = [ [[package]] name = "gvltctl" -version = "0.1.1" +version = "0.1.3" dependencies = [ "anyhow", "bip32", diff --git a/Cargo.toml b/Cargo.toml index aa93756..a1d7da0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gvltctl" -version = "0.1.2" +version = "0.1.3" edition = "2021" authors = ["Gevulot Team"] license = "MIT OR Apache-2.0" @@ -9,7 +9,7 @@ description = "Gevulot Control CLI" [dependencies] # TODO: change rev to tag when available -gevulot-rs = { git = "https://github.com/gevulotnetwork/gevulot-rs.git", rev = "28b6c9f39ebe8144cbd8eac746d7ada5b8646268" } +gevulot-rs = { git = "https://github.com/gevulotnetwork/gevulot-rs.git", rev = "e6c8f044596e566814076374430de65c281bff57" } bip32 = "0.5.1" clap = { version = "4", features = ["env", "cargo"] } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 41e92b0..ec0c342 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,5 +2,6 @@ pub mod build; pub mod pins; pub mod tasks; +pub mod workflows; pub mod workers; pub mod sudo; diff --git a/src/commands/workflows.rs b/src/commands/workflows.rs new file mode 100644 index 0000000..e12e9fa --- /dev/null +++ b/src/commands/workflows.rs @@ -0,0 +1,97 @@ +use gevulot_rs::proto::gevulot::gevulot::{workflow_spec::Stage, InputContext, MsgCreateWorkflow, MsgDeleteWorkflow, OutputContext, TaskEnv, TaskSpec, WorkflowSpec}; + +use crate::{connect_to_gevulot, print_object, read_file}; + + + + +pub async fn list_workflows(_sub_m: &clap::ArgMatches) -> Result<(), Box> { + let mut client = crate::connect_to_gevulot(_sub_m).await?; + let workflows = client.workflows.list().await?; + let workflows: Vec = workflows.into_iter().map(Into::into).collect(); + print_object(_sub_m, &workflows)?; + Ok(()) +} + +pub async fn get_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { + if let Some(workflow_id) = _sub_m.get_one::("id") { + let mut client = crate::connect_to_gevulot(_sub_m).await?; + let workflow = client.workflows.get(workflow_id).await?; + let workflow: gevulot_rs::models::Workflow = workflow.into(); + print_object(_sub_m, &workflow)?; + } else { + println!("Workflow ID is required"); + } + Ok(()) +} + +pub async fn create_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { + let workflow: gevulot_rs::models::WorkflowSpec = read_file(_sub_m).await?; + let mut client = connect_to_gevulot(_sub_m).await?; + let me = client + .base_client + .write() + .await + .address + .clone() + .ok_or("No address found, did you set a mnemonic?")?; + + let resp = client + .workflows + .create(MsgCreateWorkflow{ + creator: me, + spec: Some(WorkflowSpec{ + stages: workflow.stages.iter().map(|s| Stage{ + tasks: s.tasks.iter().map(|t| TaskSpec{ + image: t.image.clone(), + command: t.command.clone(), + args: t.args.clone(), + env: t.env.iter().map(|e| TaskEnv{ + name: e.name.clone(), + value: e.value.clone() + }).collect(), + input_contexts: t.input_contexts.iter().map(|ic| InputContext{ + source: ic.source.clone(), + target: ic.target.clone() + }).collect(), + output_contexts: t.output_contexts.iter().map(|oc| OutputContext{ + source: oc.source.clone(), + retention_period: oc.retention_period as u64 + }).collect(), + cpus: t.resources.cpus as u64, + gpus: t.resources.gpus as u64, + memory: t.resources.memory as u64, + time: t.resources.time as u64, + store_stdout: t.store_stdout.unwrap_or(false), + store_stderr: t.store_stderr.unwrap_or(false), + workflow_ref: "".to_string(), + }).collect::>(), + }).collect::>(), + }), + }).await?; + + println!("Created workflow with ID: {}", resp.id); + Ok(()) +} + +pub async fn delete_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { + if let Some(workflow_id) = _sub_m.get_one::("id") { + let mut client = crate::connect_to_gevulot(_sub_m).await?; + let me = client + .base_client + .write() + .await + .address + .clone() + .ok_or("No address found, did you set a mnemonic?")?; + + client.workflows.delete(MsgDeleteWorkflow{ + creator: me, + id: workflow_id.clone(), + }).await?; + println!("ok"); + } else { + println!("Workflow ID is required"); + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 809a9e4..2cd3d9c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ mod commands; #[cfg(target_os = "linux")] use commands::build::*; -use commands::{pins::*, sudo::*, tasks::*, workers::*}; +use commands::{pins::*, sudo::*, tasks::*, workflows::*,workers::*}; /// Main entry point for the Gevulot Control CLI application. /// @@ -718,22 +718,3 @@ async fn generate_completion(_sub_m: &clap::ArgMatches) -> Result<(), Box Result<(), Box> { - println!("Listing all workflows"); - todo!(); -} - -async fn get_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { - println!("Getting a specific workflow"); - todo!(); -} - -async fn create_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { - println!("Creating a new workflow"); - todo!(); -} - -async fn delete_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box> { - println!("Deleting a workflow"); - todo!(); -} From 6c942635d1e0ed0e80cebd0a29d9d0b365c8df98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= Date: Fri, 15 Nov 2024 16:17:38 +0200 Subject: [PATCH 2/6] Construct clients only after arg validation --- src/commands/pins.rs | 2 +- src/commands/tasks.rs | 2 +- src/commands/workers.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/pins.rs b/src/commands/pins.rs index 4f17fff..3120516 100644 --- a/src/commands/pins.rs +++ b/src/commands/pins.rs @@ -30,8 +30,8 @@ pub async fn list_pins(_sub_m: &clap::ArgMatches) -> Result<(), Box Result<(), Box> { - let mut client = connect_to_gevulot(_sub_m).await?; if let Some(pin_cid) = _sub_m.get_one::("cid") { + let mut client = connect_to_gevulot(_sub_m).await?; let pin = client.pins.get(pin_cid).await?; // Convert the pin to the gevulot_rs::models::Pin type let pin: gevulot_rs::models::Pin = pin.into(); diff --git a/src/commands/tasks.rs b/src/commands/tasks.rs index 712313f..a25ce56 100644 --- a/src/commands/tasks.rs +++ b/src/commands/tasks.rs @@ -33,8 +33,8 @@ pub async fn list_tasks(_sub_m: &clap::ArgMatches) -> Result<(), Box Result<(), Box> { - let mut client = crate::connect_to_gevulot(_sub_m).await?; if let Some(task_id) = _sub_m.get_one::("id") { + let mut client = crate::connect_to_gevulot(_sub_m).await?; let task = client.tasks.get(task_id).await?; let task: gevulot_rs::models::Task = task.into(); print_object(_sub_m, &task)?; diff --git a/src/commands/workers.rs b/src/commands/workers.rs index 713edb8..01581e7 100644 --- a/src/commands/workers.rs +++ b/src/commands/workers.rs @@ -31,8 +31,8 @@ pub async fn list_workers(_sub_m: &clap::ArgMatches) -> Result<(), Box if an error occurs. pub async fn get_worker(_sub_m: &clap::ArgMatches) -> Result<(), Box> { - let mut client = connect_to_gevulot(_sub_m).await?; if let Some(worker_id) = _sub_m.get_one::("id") { + let mut client = connect_to_gevulot(_sub_m).await?; let worker = client.workers.get(worker_id).await?; let worker: gevulot_rs::models::Worker = worker.into(); print_object(_sub_m, &worker)?; From e37bbf4fad8842a46df4d78100ebef8fb49f915c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= Date: Fri, 15 Nov 2024 16:44:34 +0200 Subject: [PATCH 3/6] Fix `mia` dependency issue --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/builders/skopeo_builder.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6df037..1d1b27f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,7 +1634,7 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mia-installer" version = "0.2.2" -source = "git+https://github.com/gevulotnetwork/mia.git?tag=mia-installer-0.2.2#f9e39ba0c7c7b092ff798780fff353bfe5919693" +source = "git+https://github.com/gevulotnetwork/mia.git?rev=4434218127c7aa98a74a4abb9fca72eadfc04c40#4434218127c7aa98a74a4abb9fca72eadfc04c40" dependencies = [ "anyhow", "env_logger 0.11.5", diff --git a/Cargo.toml b/Cargo.toml index 2cf33c5..96f07df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ tokio = { version = "1", features = ["full"] } toml = "0.8.19" [target.'cfg(target_os = "linux")'.dependencies] -mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", tag = "mia-installer-0.2.2" } +mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", rev = "4434218127c7aa98a74a4abb9fca72eadfc04c40" } anyhow = "1" log = "0.4.22" diff --git a/src/builders/skopeo_builder.rs b/src/builders/skopeo_builder.rs index 316a727..8dcea99 100644 --- a/src/builders/skopeo_builder.rs +++ b/src/builders/skopeo_builder.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result}; use log::debug; -use gevulot_rs::runtime_config::{self, RuntimeConfig}; +use mia_installer::runtime_config::{self, RuntimeConfig}; use oci_spec::image::{ImageConfiguration, ImageManifest}; use std::io::{self, BufRead, BufReader, Write}; use std::{env, fs, path::Path, process::Command}; From 52297084b3e951ce2d879a2112e379ac6298d21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= Date: Tue, 19 Nov 2024 15:02:37 +0200 Subject: [PATCH 4/6] Use the right struct for reading workflow file --- src/commands/workflows.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/workflows.rs b/src/commands/workflows.rs index e12e9fa..2505518 100644 --- a/src/commands/workflows.rs +++ b/src/commands/workflows.rs @@ -26,7 +26,7 @@ pub async fn get_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box Result<(), Box> { - let workflow: gevulot_rs::models::WorkflowSpec = read_file(_sub_m).await?; + let workflow: gevulot_rs::models::Workflow = read_file(_sub_m).await?; let mut client = connect_to_gevulot(_sub_m).await?; let me = client .base_client @@ -41,7 +41,7 @@ pub async fn create_workflow(_sub_m: &clap::ArgMatches) -> Result<(), Box Date: Wed, 4 Dec 2024 19:23:35 +0200 Subject: [PATCH 5/6] Update gevulot-rs --- Cargo.lock | 12 +++++++----- Cargo.toml | 2 +- src/commands/pins.rs | 22 ++++++++++++++-------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 863b353..6f916d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,10 +1029,11 @@ dependencies = [ [[package]] name = "gevulot-rs" version = "0.1.0" -source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=961a71a9ff933ee5767e7f96597767c809e2c36e#961a71a9ff933ee5767e7f96597767c809e2c36e" +source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=997bd6c5266a98a69253c58b4bd1a35e8d3ec7ae#997bd6c5266a98a69253c58b4bd1a35e8d3ec7ae" dependencies = [ "backon", "bip32", + "const_format", "cosmos-sdk-proto", "cosmrs", "derivative", @@ -1045,6 +1046,7 @@ dependencies = [ "prost-build", "rand 0.8.5", "rand_core 0.6.4", + "semver", "serde", "serde_json", "tendermint", @@ -1057,8 +1059,8 @@ dependencies = [ [[package]] name = "gevulot-rs" -version = "0.1.0" -source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=997bd6c5266a98a69253c58b4bd1a35e8d3ec7ae#997bd6c5266a98a69253c58b4bd1a35e8d3ec7ae" +version = "0.1.3" +source = "git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=0b3c05e34c7680c2dd3ef6caaf27b2f96bca279d#0b3c05e34c7680c2dd3ef6caaf27b2f96bca279d" dependencies = [ "backon", "bip32", @@ -1125,7 +1127,7 @@ dependencies = [ "clap_complete", "cosmrs", "env_logger 0.11.5", - "gevulot-rs 0.1.0 (git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=961a71a9ff933ee5767e7f96597767c809e2c36e)", + "gevulot-rs 0.1.3", "log", "mia-installer", "num_cpus", @@ -1662,7 +1664,7 @@ dependencies = [ "env_logger 0.11.5", "flate2", "fs_extra", - "gevulot-rs 0.1.0 (git+https://github.com/gevulotnetwork/gevulot-rs.git?rev=997bd6c5266a98a69253c58b4bd1a35e8d3ec7ae)", + "gevulot-rs 0.1.0", "log", "octocrab", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index ccfd7a3..fcbaf29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ description = "Gevulot Control CLI" # TODO: change rev to tag when available # NOTE: this revision is aligned with `mia-installer` dependency. # Be careful changing it. -gevulot-rs = { git = "https://github.com/gevulotnetwork/gevulot-rs.git", rev = "961a71a9ff933ee5767e7f96597767c809e2c36e" } +gevulot-rs = { git = "https://github.com/gevulotnetwork/gevulot-rs.git", rev = "0b3c05e34c7680c2dd3ef6caaf27b2f96bca279d" } bip32 = "0.5.1" clap = { version = "4", features = ["env", "cargo"] } diff --git a/src/commands/pins.rs b/src/commands/pins.rs index b9bd6df..ab010d3 100644 --- a/src/commands/pins.rs +++ b/src/commands/pins.rs @@ -85,10 +85,13 @@ pub async fn create_pin(_sub_m: &clap::ArgMatches) -> Result<(), Box Result<(), Box Date: Wed, 4 Dec 2024 19:26:13 +0200 Subject: [PATCH 6/6] Refresh mia-installer --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fcbaf29..950f31f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,8 @@ tokio = { version = "1", features = ["full"] } toml = "0.8.19" [target.'cfg(target_os = "linux")'.dependencies] -mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", tag = "mia-installer-0.2.3" } +mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", rev = "026b84dbe7c3fcc9e0ec644c59a7b7123738104e" } +#mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", tag = "mia-installer-0.2.3" } anyhow = "1" log = "0.4.22"