update : updated with new changes and ci

.env.example

@@ -28,6 +28,3 @@ AWS_SECRET_ACCESS_KEY=
 # S3
 AWS_S3_BUCKET_NAME=
 AWS_S3_BUCKET_REGION=
-
-# Local Stack
-AWS_ENDPOINT_URL=

.env.test.example

@@ -0,0 +1,33 @@
+HOST=
+PORT=
+DATABASE_URL=
+MADARA_RPC_URL=
+DA_LAYER=
+SETTLEMENT_LAYER=
+
+# Ethereum
+ETHEREUM_PRIVATE_KEY=
+ETHEREUM_RPC_URL=
+MEMORY_PAGES_CONTRACT_ADDRESS=
+STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
+
+
+# Starknet
+STARKNET_PUBLIC_KEY=
+STARNET_PRIVATE_KEY=
+STARKNET_RPC_URL=
+STARKNET_CAIRO_CORE_CONTRACT_ADDRESS=
+
+# MongoDB connection string
+MONGODB_CONNECTION_STRING=
+
+# SQS
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# S3
+AWS_S3_BUCKET_NAME=
+AWS_S3_BUCKET_REGION=
+
+# AWS Local Stack URL
+AWS_ENDPOINT_URL=

.github/workflows/coverage.yml

@@ -28,15 +28,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-
-      - name: Install dependencies
-        run: |
-          pip install awscli-local
-
       # selecting a toolchain either by action or manual `rustup` calls should happen
       # before the plugin, as the cache uses the current rustc version as its cache key
       - run: rustup show
@@ -49,26 +40,23 @@ jobs:
           cargo llvm-cov clean --workspace
 
       - name: Run llvm-cov
-        env:
-          AWS_ACCESS_KEY_ID: "AWS_ACCESS_KEY_ID"
-          AWS_SECRET_ACCESS_KEY: "AWS_SECRET_ACCESS_KEY"
-          AWS_S3_BUCKET_NAME: "madara-orchestrator-test-bucket"
-          AWS_S3_BUCKET_REGION: "us-east-1"
-          AWS_ENDPOINT_URL: "http://localhost.localstack.cloud:4566"
-          MADARA_RPC_URL: "http://localhost:3000"
-          ETHEREUM_RPC_URL: "http://localhost:3001"
-          MEMORY_PAGES_CONTRACT_ADDRESS: "0x000000000000000000000000000000000001dead"
-          PRIVATE_KEY: "0xdead"
-          ETHEREUM_PRIVATE_KEY: "0x000000000000000000000000000000000000000000000000000000000000beef"
-          STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS: "0x000000000000000000000000000000000002dead"
-          DA_LAYER: "ethereum"
-          PROVER_SERVICE: "sharp"
-          SETTLEMENT_LAYER: "ethereum"
-          DATA_STORAGE: "s3"
-          MONGODB_CONNECTION_STRING: "mongodb://localhost:27017"
         run: |
-          awslocal s3api create-bucket --bucket madara-orchestrator-test-bucket
-          awslocal s3 ls
+          echo 'AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID"' >> .env.test
+          echo 'AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"' >> .env.test
+          echo 'AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"' >> .env.test
+          echo 'AWS_S3_BUCKET_REGION="us-east-1"' >> .env.test
+          echo 'AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"' >> .env.test
+          echo 'MADARA_RPC_URL="http://localhost:3000"' >> .env.test
+          echo 'ETHEREUM_RPC_URL="http://localhost:3001"' >> .env.test
+          echo 'MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"' >> .env.test
+          echo 'PRIVATE_KEY="0xdead"' >> .env.test
+          echo 'ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef"' >> .env.test
+          echo 'STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"' >> .env.test
+          echo 'DA_LAYER="ethereum"' >> .env.test
+          echo 'PROVER_SERVICE="sharp"' >> .env.test
+          echo 'SETTLEMENT_LAYER="ethereum"' >> .env.test
+          echo 'DATA_STORAGE="s3"' >> .env.test
+          echo 'MONGODB_CONNECTION_STRING="mongodb://localhost:27017"' >> .env.test
           cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1
 
       - name: Upload coverage to codecov.io

.gitignore

@@ -2,6 +2,7 @@
 .env
 .idea
 .DS_Store
+.env.test
 
 *.code-workspace
 .vscode

CHANGELOG.md

@@ -9,9 +9,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Function to calculate the kzg proof of x_0.
 - Tests for updating the state.
 - Function to update the state and publish blob on ethereum in state update job.
+- Fixtures for testing.
 
 ## Changed
 
+- GitHub's coverage CI yml file for localstack and db testing.
+
 ## Removed
 
 - `fetch_from_test` argument

crates/orchestrator/src/constants.rs

@@ -1,3 +1,2 @@
 pub const BLOB_DATA_FILE_NAME: &str = "blob_data.txt";
 pub const SNOS_OUTPUT_FILE_NAME: &str = "snos_output.json";
-pub const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue";

crates/orchestrator/src/controllers/mod.rs

@@ -1,5 +1,2 @@
 /// Errors
 mod errors;
-
-/// Job controllers
-pub mod jobs_controller;

crates/orchestrator/src/data_storage/aws_s3/config.rs

@@ -13,6 +13,7 @@ pub struct AWSS3Config {
     /// S3 Bucket region
     pub s3_bucket_region: String,
     /// Endpoint url
+    #[cfg(test)]
     pub endpoint_url: String,
 }
 
@@ -25,6 +26,7 @@ impl DataStorageConfig for AWSS3Config {
             s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"),
             s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"),
             s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"),
+            #[cfg(test)]
             endpoint_url: get_env_var_or_panic("AWS_ENDPOINT_URL"),
         }
     }

crates/orchestrator/src/data_storage/aws_s3/mod.rs

@@ -32,11 +32,15 @@ impl AWSS3 {
             "loaded_from_custom_env",
         );
         let region = Region::new(config.s3_bucket_region.clone().to_string());
-        let conf_builder = Builder::new()
-            .region(region)
-            .credentials_provider(credentials)
-            .endpoint_url(config.endpoint_url.clone())
-            .force_path_style(true);
+
+        #[allow(unused_mut)]
+        let mut conf_builder = Builder::new().region(region).credentials_provider(credentials).force_path_style(true);
+
+        #[cfg(test)]
+        {
+            conf_builder = conf_builder.endpoint_url(config.endpoint_url.clone().to_string());
+        }
+
         let conf = conf_builder.build();
 
         // Building AWS S3 config
@@ -72,4 +76,10 @@ impl DataStorage for AWSS3 {
 
         Ok(())
     }
+
+    #[cfg(test)]
+    async fn build_test_bucket(&self, bucket_name: &str) -> Result<()> {
+        self.client.create_bucket().bucket(bucket_name).send().await?;
+        Ok(())
+    }
 }

crates/orchestrator/src/data_storage/mod.rs

@@ -17,6 +17,8 @@ use mockall::automock;
 pub trait DataStorage: Send + Sync {
     async fn get_data(&self, key: &str) -> Result<Bytes>;
     async fn put_data(&self, data: Bytes, key: &str) -> Result<()>;
+    #[cfg(test)]
+    async fn build_test_bucket(&self, bucket_name: &str) -> Result<()>;
 }
 
 /// **DataStorageConfig** : Trait method to represent the config struct needed for

crates/orchestrator/src/queue/job_queue.rs

@@ -12,8 +12,8 @@ use uuid::Uuid;
 use crate::config::config;
 use crate::jobs::{process_job, verify_job};
 
-const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue";
-const JOB_VERIFICATION_QUEUE: &str = "madara_orchestrator_job_verification_queue";
+pub const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue";
+pub const JOB_VERIFICATION_QUEUE: &str = "madara_orchestrator_job_verification_queue";
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct JobQueueMessage {

crates/orchestrator/src/routes.rs

@@ -1,16 +1,10 @@
 use axum::http::StatusCode;
 use axum::response::IntoResponse;
-use axum::routing::{get, post};
+use axum::routing::get;
 use axum::Router;
 
-use crate::controllers::jobs_controller;
-
 pub fn app_router() -> Router {
-    Router::new()
-        .route("/health", get(root))
-        .nest("/v1/dev", dev_routes())
-        .nest("/v1/job", job_routes())
-        .fallback(handler_404)
+    Router::new().route("/health", get(root)).nest("/v1/dev", dev_routes()).fallback(handler_404)
 }
 
 async fn root() -> &'static str {
@@ -21,10 +15,6 @@ async fn handler_404() -> impl IntoResponse {
     (StatusCode::NOT_FOUND, "The requested resource was not found")
 }
 
-fn job_routes() -> Router {
-    Router::new().route("/create_job", post(jobs_controller::create_job))
-}
-
 fn dev_routes() -> Router {
     Router::new()
 }

crates/orchestrator/src/tests/common/mod.rs

@@ -6,7 +6,6 @@ use std::sync::Arc;
 use ::uuid::Uuid;
 use constants::*;
 use da_client_interface::MockDaClient;
-use dotenvy::dotenv;
 use mongodb::Client;
 use prover_client_interface::MockProverClient;
 use rstest::*;
@@ -17,7 +16,7 @@ use url::Url;
 use utils::env_utils::get_env_var_or_panic;
 use utils::settings::default::DefaultSettingsProvider;
 
-use crate::config::{config_force_init, Config};
+use crate::config::{build_storage_client, config_force_init, Config};
 use crate::data_storage::MockDataStorage;
 use crate::database::mongodb::config::MongoDbConfig;
 use crate::database::mongodb::MongoDb;
@@ -84,7 +83,8 @@ pub fn custom_job_item(default_job_item: JobItem, #[default(String::from("0"))]
 /// For implementation of integration tests
 #[fixture]
 pub async fn build_config() -> color_eyre::Result<()> {
-    dotenv().ok();
+    // Getting .env.test variables
+    dotenvy::from_filename("../.env.test")?;
 
     // init starknet client
     let provider = JsonRpcClient::new(HttpTransport::new(
@@ -101,7 +101,10 @@ pub async fn build_config() -> color_eyre::Result<()> {
     let settings_provider = DefaultSettingsProvider {};
     let settlement_client = crate::config::build_settlement_client(&settings_provider).await;
     let prover_client = crate::config::build_prover_service(&settings_provider);
-    let storage_client = crate::config::build_storage_client().await;
+    let storage_client = build_storage_client().await;
+
+    // building a test bucket :
+    storage_client.build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME")).await?;
 
     let config =
         Config::new(Arc::new(provider), da_client, prover_client, settlement_client, database, queue, storage_client);

crates/orchestrator/src/tests/data_storage/mod.rs

@@ -2,14 +2,14 @@ use crate::data_storage::aws_s3::config::AWSS3Config;
 use crate::data_storage::aws_s3::AWSS3;
 use crate::data_storage::{DataStorage, DataStorageConfig};
 use bytes::Bytes;
-use dotenvy::dotenv;
 use rstest::rstest;
 use serde_json::json;
 
 #[rstest]
 #[tokio::test]
 async fn test_put_and_get_data_s3() -> color_eyre::Result<()> {
-    dotenv().ok();
+    dotenvy::from_filename("../.env.test")?;
+
     let config = AWSS3Config::new_from_env();
     let s3_client = AWSS3::new(config).await;
 

crates/orchestrator/src/tests/database/mod.rs

@@ -2,7 +2,6 @@ use crate::config::config;
 use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType};
 use crate::tests::common::{build_config, drop_database};
 use color_eyre::eyre::eyre;
-use dotenvy::dotenv;
 use rstest::*;
 use uuid::Uuid;
 
@@ -22,7 +21,6 @@ async fn test_database_connection() -> color_eyre::Result<()> {
 #[rstest]
 #[tokio::test]
 async fn test_database_create_job() -> color_eyre::Result<()> {
-    dotenv().ok();
     let init_config = build_config().await.is_ok();
     if !init_config {
         return Err(eyre!("Not able to init config."));
@@ -34,9 +32,9 @@ async fn test_database_create_job() -> color_eyre::Result<()> {
     let database_client = config.database();
 
     let job_vec = [
-        get_random_job_item(JobType::ProofCreation, JobStatus::Created, 1),
-        get_random_job_item(JobType::ProofCreation, JobStatus::Created, 2),
-        get_random_job_item(JobType::ProofCreation, JobStatus::Created, 3),
+        build_job_item(JobType::ProofCreation, JobStatus::Created, 1),
+        build_job_item(JobType::ProofCreation, JobStatus::Created, 2),
+        build_job_item(JobType::ProofCreation, JobStatus::Created, 3),
     ];
 
     database_client.create_job(job_vec[0].clone()).await.unwrap();
@@ -60,7 +58,7 @@ async fn test_database_create_job() -> color_eyre::Result<()> {
 // Test Util Functions
 // ==========================================
 
-fn get_random_job_item(job_type: JobType, job_status: JobStatus, internal_id: u64) -> JobItem {
+fn build_job_item(job_type: JobType, job_status: JobStatus, internal_id: u64) -> JobItem {
     JobItem {
         id: Uuid::new_v4(),
         internal_id: internal_id.to_string(),