From 15541ef49dcb53ef579457f7e6bd3498f1635d36 Mon Sep 17 00:00:00 2001 From: Arun Jangra Date: Fri, 26 Jul 2024 01:08:26 +0530 Subject: [PATCH] update : updated with new changes and ci --- .env.example | 3 -- .env.test.example | 33 +++++++++++++ .github/workflows/coverage.yml | 44 ++++++----------- .gitignore | 1 + CHANGELOG.md | 3 ++ crates/orchestrator/src/constants.rs | 1 - .../src/controllers/jobs_controller.rs | 24 --------- crates/orchestrator/src/controllers/mod.rs | 3 -- .../src/data_storage/aws_s3/config.rs | 2 + .../src/data_storage/aws_s3/mod.rs | 20 ++++++-- crates/orchestrator/src/data_storage/mod.rs | 2 + crates/orchestrator/src/queue/job_queue.rs | 4 +- crates/orchestrator/src/routes.rs | 14 +----- crates/orchestrator/src/tests/common/mod.rs | 11 +++-- .../orchestrator/src/tests/controllers/mod.rs | 49 ------------------- .../src/tests/data_storage/mod.rs | 4 +- crates/orchestrator/src/tests/database/mod.rs | 10 ++-- crates/orchestrator/src/tests/mod.rs | 1 - .../src/tests/workers/snos/mod.rs | 2 +- 19 files changed, 90 insertions(+), 141 deletions(-) create mode 100644 .env.test.example delete mode 100644 crates/orchestrator/src/controllers/jobs_controller.rs delete mode 100644 crates/orchestrator/src/tests/controllers/mod.rs diff --git a/.env.example b/.env.example index 37723425..57635205 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,3 @@ AWS_SECRET_ACCESS_KEY= # S3 AWS_S3_BUCKET_NAME= AWS_S3_BUCKET_REGION= - -# Local Stack -AWS_ENDPOINT_URL= \ No newline at end of file diff --git a/.env.test.example b/.env.test.example new file mode 100644 index 00000000..e55b97fc --- /dev/null +++ b/.env.test.example @@ -0,0 +1,33 @@ +HOST= +PORT= +DATABASE_URL= +MADARA_RPC_URL= +DA_LAYER= +SETTLEMENT_LAYER= + +# Ethereum +ETHEREUM_PRIVATE_KEY= +ETHEREUM_RPC_URL= +MEMORY_PAGES_CONTRACT_ADDRESS= +STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS= + + +# Starknet +STARKNET_PUBLIC_KEY= +STARNET_PRIVATE_KEY= +STARKNET_RPC_URL= +STARKNET_CAIRO_CORE_CONTRACT_ADDRESS= + +# MongoDB connection string +MONGODB_CONNECTION_STRING= + +# SQS +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# S3 +AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_REGION= + +# AWS Local Stack URL +AWS_ENDPOINT_URL= diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c97218b0..d5d10b8d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -28,15 +28,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - name: Install dependencies - run: | - pip install awscli-local - # selecting a toolchain either by action or manual `rustup` calls should happen # before the plugin, as the cache uses the current rustc version as its cache key - run: rustup show @@ -49,26 +40,23 @@ jobs: cargo llvm-cov clean --workspace - name: Run llvm-cov - env: - AWS_ACCESS_KEY_ID: "AWS_ACCESS_KEY_ID" - AWS_SECRET_ACCESS_KEY: "AWS_SECRET_ACCESS_KEY" - AWS_S3_BUCKET_NAME: "madara-orchestrator-test-bucket" - AWS_S3_BUCKET_REGION: "us-east-1" - AWS_ENDPOINT_URL: "http://localhost.localstack.cloud:4566" - MADARA_RPC_URL: "http://localhost:3000" - ETHEREUM_RPC_URL: "http://localhost:3001" - MEMORY_PAGES_CONTRACT_ADDRESS: "0x000000000000000000000000000000000001dead" - PRIVATE_KEY: "0xdead" - ETHEREUM_PRIVATE_KEY: "0x000000000000000000000000000000000000000000000000000000000000beef" - STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS: "0x000000000000000000000000000000000002dead" - DA_LAYER: "ethereum" - PROVER_SERVICE: "sharp" - SETTLEMENT_LAYER: "ethereum" - DATA_STORAGE: "s3" - MONGODB_CONNECTION_STRING: "mongodb://localhost:27017" run: | - awslocal s3api create-bucket --bucket madara-orchestrator-test-bucket - awslocal s3 ls + echo 'AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID"' >> .env.test + echo 'AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"' >> .env.test + echo 'AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"' >> .env.test + echo 'AWS_S3_BUCKET_REGION="us-east-1"' >> .env.test + echo 'AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"' >> .env.test + echo 'MADARA_RPC_URL="http://localhost:3000"' >> .env.test + echo 'ETHEREUM_RPC_URL="http://localhost:3001"' >> .env.test + echo 'MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"' >> .env.test + echo 'PRIVATE_KEY="0xdead"' >> .env.test + echo 'ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef"' >> .env.test + echo 'STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"' >> .env.test + echo 'DA_LAYER="ethereum"' >> .env.test + echo 'PROVER_SERVICE="sharp"' >> .env.test + echo 'SETTLEMENT_LAYER="ethereum"' >> .env.test + echo 'DATA_STORAGE="s3"' >> .env.test + echo 'MONGODB_CONNECTION_STRING="mongodb://localhost:27017"' >> .env.test cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1 - name: Upload coverage to codecov.io diff --git a/.gitignore b/.gitignore index cc424bf3..76cd1131 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env .idea .DS_Store +.env.test *.code-workspace .vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 40fd1c8d..70c41f6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Function to calculate the kzg proof of x_0. - Tests for updating the state. - Function to update the state and publish blob on ethereum in state update job. +- Fixtures for testing. ## Changed +- GitHub's coverage CI yml file for localstack and db testing. + ## Removed - `fetch_from_test` argument diff --git a/crates/orchestrator/src/constants.rs b/crates/orchestrator/src/constants.rs index 1fd645a9..9361d764 100644 --- a/crates/orchestrator/src/constants.rs +++ b/crates/orchestrator/src/constants.rs @@ -1,3 +1,2 @@ pub const BLOB_DATA_FILE_NAME: &str = "blob_data.txt"; pub const SNOS_OUTPUT_FILE_NAME: &str = "snos_output.json"; -pub const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue"; diff --git a/crates/orchestrator/src/controllers/jobs_controller.rs b/crates/orchestrator/src/controllers/jobs_controller.rs deleted file mode 100644 index e5f0daf8..00000000 --- a/crates/orchestrator/src/controllers/jobs_controller.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::collections::HashMap; - -use axum::extract::Json; -use serde::Deserialize; - -use crate::controllers::errors::AppError; -use crate::jobs::types::JobType; - -/// Client request to create a job -#[derive(Debug, Deserialize)] -pub struct CreateJobRequest { - /// Job type - pub job_type: JobType, - /// Internal id must be a way to identify the job. For example - /// block_no, transaction_hash etc. The (job_type, internal_id) - /// pair must be unique. - pub internal_id: String, -} - -/// Create a job -pub async fn create_job(Json(payload): Json) -> Result, AppError> { - crate::jobs::create_job(payload.job_type, payload.internal_id, HashMap::new()).await?; - Ok(Json::from(())) -} diff --git a/crates/orchestrator/src/controllers/mod.rs b/crates/orchestrator/src/controllers/mod.rs index aadb38b7..8575ccdc 100644 --- a/crates/orchestrator/src/controllers/mod.rs +++ b/crates/orchestrator/src/controllers/mod.rs @@ -1,5 +1,2 @@ /// Errors mod errors; - -/// Job controllers -pub mod jobs_controller; diff --git a/crates/orchestrator/src/data_storage/aws_s3/config.rs b/crates/orchestrator/src/data_storage/aws_s3/config.rs index d4518d85..06eeaff8 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/config.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/config.rs @@ -13,6 +13,7 @@ pub struct AWSS3Config { /// S3 Bucket region pub s3_bucket_region: String, /// Endpoint url + #[cfg(test)] pub endpoint_url: String, } @@ -25,6 +26,7 @@ impl DataStorageConfig for AWSS3Config { s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"), s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"), s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"), + #[cfg(test)] endpoint_url: get_env_var_or_panic("AWS_ENDPOINT_URL"), } } diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index da673eba..50ae6e3e 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -32,11 +32,15 @@ impl AWSS3 { "loaded_from_custom_env", ); let region = Region::new(config.s3_bucket_region.clone().to_string()); - let conf_builder = Builder::new() - .region(region) - .credentials_provider(credentials) - .endpoint_url(config.endpoint_url.clone()) - .force_path_style(true); + + #[allow(unused_mut)] + let mut conf_builder = Builder::new().region(region).credentials_provider(credentials).force_path_style(true); + + #[cfg(test)] + { + conf_builder = conf_builder.endpoint_url(config.endpoint_url.clone().to_string()); + } + let conf = conf_builder.build(); // Building AWS S3 config @@ -72,4 +76,10 @@ impl DataStorage for AWSS3 { Ok(()) } + + #[cfg(test)] + async fn build_test_bucket(&self, bucket_name: &str) -> Result<()> { + self.client.create_bucket().bucket(bucket_name).send().await?; + Ok(()) + } } diff --git a/crates/orchestrator/src/data_storage/mod.rs b/crates/orchestrator/src/data_storage/mod.rs index f259bf61..b3ff74fd 100644 --- a/crates/orchestrator/src/data_storage/mod.rs +++ b/crates/orchestrator/src/data_storage/mod.rs @@ -17,6 +17,8 @@ use mockall::automock; pub trait DataStorage: Send + Sync { async fn get_data(&self, key: &str) -> Result; async fn put_data(&self, data: Bytes, key: &str) -> Result<()>; + #[cfg(test)] + async fn build_test_bucket(&self, bucket_name: &str) -> Result<()>; } /// **DataStorageConfig** : Trait method to represent the config struct needed for diff --git a/crates/orchestrator/src/queue/job_queue.rs b/crates/orchestrator/src/queue/job_queue.rs index 9432276f..aebba4bf 100644 --- a/crates/orchestrator/src/queue/job_queue.rs +++ b/crates/orchestrator/src/queue/job_queue.rs @@ -12,8 +12,8 @@ use uuid::Uuid; use crate::config::config; use crate::jobs::{process_job, verify_job}; -const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue"; -const JOB_VERIFICATION_QUEUE: &str = "madara_orchestrator_job_verification_queue"; +pub const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue"; +pub const JOB_VERIFICATION_QUEUE: &str = "madara_orchestrator_job_verification_queue"; #[derive(Debug, Serialize, Deserialize)] pub struct JobQueueMessage { diff --git a/crates/orchestrator/src/routes.rs b/crates/orchestrator/src/routes.rs index 39d8f3d4..877e5b88 100644 --- a/crates/orchestrator/src/routes.rs +++ b/crates/orchestrator/src/routes.rs @@ -1,16 +1,10 @@ use axum::http::StatusCode; use axum::response::IntoResponse; -use axum::routing::{get, post}; +use axum::routing::get; use axum::Router; -use crate::controllers::jobs_controller; - pub fn app_router() -> Router { - Router::new() - .route("/health", get(root)) - .nest("/v1/dev", dev_routes()) - .nest("/v1/job", job_routes()) - .fallback(handler_404) + Router::new().route("/health", get(root)).nest("/v1/dev", dev_routes()).fallback(handler_404) } async fn root() -> &'static str { @@ -21,10 +15,6 @@ async fn handler_404() -> impl IntoResponse { (StatusCode::NOT_FOUND, "The requested resource was not found") } -fn job_routes() -> Router { - Router::new().route("/create_job", post(jobs_controller::create_job)) -} - fn dev_routes() -> Router { Router::new() } diff --git a/crates/orchestrator/src/tests/common/mod.rs b/crates/orchestrator/src/tests/common/mod.rs index 4265a707..0a97fa33 100644 --- a/crates/orchestrator/src/tests/common/mod.rs +++ b/crates/orchestrator/src/tests/common/mod.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use ::uuid::Uuid; use constants::*; use da_client_interface::MockDaClient; -use dotenvy::dotenv; use mongodb::Client; use prover_client_interface::MockProverClient; use rstest::*; @@ -17,7 +16,7 @@ use url::Url; use utils::env_utils::get_env_var_or_panic; use utils::settings::default::DefaultSettingsProvider; -use crate::config::{config_force_init, Config}; +use crate::config::{build_storage_client, config_force_init, Config}; use crate::data_storage::MockDataStorage; use crate::database::mongodb::config::MongoDbConfig; use crate::database::mongodb::MongoDb; @@ -84,7 +83,8 @@ pub fn custom_job_item(default_job_item: JobItem, #[default(String::from("0"))] /// For implementation of integration tests #[fixture] pub async fn build_config() -> color_eyre::Result<()> { - dotenv().ok(); + // Getting .env.test variables + dotenvy::from_filename("../.env.test")?; // init starknet client let provider = JsonRpcClient::new(HttpTransport::new( @@ -101,7 +101,10 @@ pub async fn build_config() -> color_eyre::Result<()> { let settings_provider = DefaultSettingsProvider {}; let settlement_client = crate::config::build_settlement_client(&settings_provider).await; let prover_client = crate::config::build_prover_service(&settings_provider); - let storage_client = crate::config::build_storage_client().await; + let storage_client = build_storage_client().await; + + // building a test bucket : + storage_client.build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME")).await?; let config = Config::new(Arc::new(provider), da_client, prover_client, settlement_client, database, queue, storage_client); diff --git a/crates/orchestrator/src/tests/controllers/mod.rs b/crates/orchestrator/src/tests/controllers/mod.rs deleted file mode 100644 index b5849a86..00000000 --- a/crates/orchestrator/src/tests/controllers/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::config::config_force_init; -use crate::constants::JOB_PROCESSING_QUEUE; -use crate::controllers::jobs_controller::{create_job, CreateJobRequest}; -use crate::database::MockDatabase; -use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; -use crate::queue::MockQueueProvider; -use crate::tests::common::init_config; -use axum::Json; -use mockall::predicate::eq; -use rstest::rstest; -use uuid::Uuid; - -#[rstest] -#[tokio::test] -async fn test_create_job_jobs_controller() -> color_eyre::Result<()> { - let mut db = MockDatabase::new(); - let mut queue = MockQueueProvider::new(); - - // mocking db get function (when creating job it should return no job existing) - db.expect_get_last_successful_job_by_type().times(1).with(eq(JobType::SnosRun)).returning(|_| Ok(None)); - // mocking db get function (when creating job to pre-check if job is not existing : worker module) - db.expect_get_job_by_internal_id_and_type().times(1).with(eq("1"), eq(JobType::SnosRun)).returning(|_, _| Ok(None)); - // mocking creation of the job - db.expect_create_job().times(1).withf(move |item| item.internal_id == *"1").returning(move |_| { - Ok(JobItem { - id: Uuid::new_v4(), - internal_id: "1".to_string(), - job_type: JobType::SnosRun, - status: JobStatus::Created, - external_id: ExternalId::Number(0), - metadata: Default::default(), - version: 0, - }) - }); - // mocking sending of the job into the queue after the creation - queue - .expect_send_message_to_queue() - .returning(|_, _, _| Ok(())) - .withf(|queue, _payload, _delay| queue == JOB_PROCESSING_QUEUE); - - let config = init_config(None, Some(db), Some(queue), None, None, None, None).await; - config_force_init(config).await; - - let create_job_request = CreateJobRequest { job_type: JobType::SnosRun, internal_id: "1".to_string() }; - - let _ = create_job(Json::from(create_job_request)).await.unwrap(); - - Ok(()) -} diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs index 8436a808..f007a457 100644 --- a/crates/orchestrator/src/tests/data_storage/mod.rs +++ b/crates/orchestrator/src/tests/data_storage/mod.rs @@ -2,14 +2,14 @@ use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use bytes::Bytes; -use dotenvy::dotenv; use rstest::rstest; use serde_json::json; #[rstest] #[tokio::test] async fn test_put_and_get_data_s3() -> color_eyre::Result<()> { - dotenv().ok(); + dotenvy::from_filename("../.env.test")?; + let config = AWSS3Config::new_from_env(); let s3_client = AWSS3::new(config).await; diff --git a/crates/orchestrator/src/tests/database/mod.rs b/crates/orchestrator/src/tests/database/mod.rs index ae9db9df..5e8a651c 100644 --- a/crates/orchestrator/src/tests/database/mod.rs +++ b/crates/orchestrator/src/tests/database/mod.rs @@ -2,7 +2,6 @@ use crate::config::config; use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; use crate::tests::common::{build_config, drop_database}; use color_eyre::eyre::eyre; -use dotenvy::dotenv; use rstest::*; use uuid::Uuid; @@ -22,7 +21,6 @@ async fn test_database_connection() -> color_eyre::Result<()> { #[rstest] #[tokio::test] async fn test_database_create_job() -> color_eyre::Result<()> { - dotenv().ok(); let init_config = build_config().await.is_ok(); if !init_config { return Err(eyre!("Not able to init config.")); @@ -34,9 +32,9 @@ async fn test_database_create_job() -> color_eyre::Result<()> { let database_client = config.database(); let job_vec = [ - get_random_job_item(JobType::ProofCreation, JobStatus::Created, 1), - get_random_job_item(JobType::ProofCreation, JobStatus::Created, 2), - get_random_job_item(JobType::ProofCreation, JobStatus::Created, 3), + build_job_item(JobType::ProofCreation, JobStatus::Created, 1), + build_job_item(JobType::ProofCreation, JobStatus::Created, 2), + build_job_item(JobType::ProofCreation, JobStatus::Created, 3), ]; database_client.create_job(job_vec[0].clone()).await.unwrap(); @@ -60,7 +58,7 @@ async fn test_database_create_job() -> color_eyre::Result<()> { // Test Util Functions // ========================================== -fn get_random_job_item(job_type: JobType, job_status: JobStatus, internal_id: u64) -> JobItem { +fn build_job_item(job_type: JobType, job_status: JobStatus, internal_id: u64) -> JobItem { JobItem { id: Uuid::new_v4(), internal_id: internal_id.to_string(), diff --git a/crates/orchestrator/src/tests/mod.rs b/crates/orchestrator/src/tests/mod.rs index 348e5c70..83dfc04c 100644 --- a/crates/orchestrator/src/tests/mod.rs +++ b/crates/orchestrator/src/tests/mod.rs @@ -7,6 +7,5 @@ pub mod server; pub mod queue; pub mod common; -mod controllers; mod data_storage; pub mod workers; diff --git a/crates/orchestrator/src/tests/workers/snos/mod.rs b/crates/orchestrator/src/tests/workers/snos/mod.rs index c4e95902..7799ffb2 100644 --- a/crates/orchestrator/src/tests/workers/snos/mod.rs +++ b/crates/orchestrator/src/tests/workers/snos/mod.rs @@ -1,7 +1,7 @@ use crate::config::config_force_init; -use crate::constants::JOB_PROCESSING_QUEUE; use crate::database::MockDatabase; use crate::jobs::types::JobType; +use crate::queue::job_queue::JOB_PROCESSING_QUEUE; use crate::queue::MockQueueProvider; use crate::tests::common::init_config; use crate::tests::workers::utils::get_job_item_mock_by_id;