Merge pull request #8 from iosis-tech/feat/compiler

Cargo.toml

@@ -9,7 +9,10 @@ members = [
     "crates/executor",
     "crates/peer",
     "crates/prover",
-    "crates/runner", "crates/tests",
+    "crates/runner",
+    "crates/compiler",
+    "crates/tests",
+
 ]
 exclude = []
 
@@ -63,4 +66,5 @@ sharp-p2p-executor = { path = "crates/executor" }
 sharp-p2p-peer = { path = "crates/peer" }
 sharp-p2p-prover = { path = "crates/prover" }
 sharp-p2p-runner = { path = "crates/runner" }
+sharp-p2p-compiler = { path = "crates/compiler" }
 sharp-p2p-tests = { path = "crates/tests" }

cairo/bootloader/objects.py

@@ -13,83 +13,30 @@
 from starkware.starkware_utils.marshmallow_dataclass_fields import additional_metadata
 from starkware.starkware_utils.validated_dataclass import ValidatedMarshmallowDataclass
 
-
-class TaskSpec(ValidatedMarshmallowDataclass):
-    """
-    Contains task's specification.
-    """
-
-    @abstractmethod
-    def load_task(self) -> "Task":
-        """
-        Returns the corresponding task.
-        """
-
-
 class Task:
     @abstractmethod
     def get_program(self) -> ProgramBase:
         """
         Returns the task's Cairo program.
         """
 
-
-@marshmallow_dataclass.dataclass(frozen=True)
-class RunProgramTask(TaskSpec, Task):
-    TYPE: ClassVar[str] = "RunProgramTask"
-    program: Program
-    program_input: dict
-    use_poseidon: bool
-
-    def get_program(self) -> Program:
-        return self.program
-
-    def load_task(self) -> "Task":
-        return self
-
-
-@marshmallow_dataclass.dataclass(frozen=True)
-class CairoPiePath(TaskSpec):
-    TYPE: ClassVar[str] = "CairoPiePath"
-    path: str
-    use_poseidon: bool
-
-    def load_task(self) -> "CairoPieTask":
-        """
-        Loads the PIE to memory.
-        """
-        return CairoPieTask(cairo_pie=CairoPie.from_file(self.path), use_poseidon=self.use_poseidon)
-
-
-class TaskSchema(OneOfSchema):
-    """
-    Schema for Task/CairoPiePath.
-    OneOfSchema adds a "type" field.
-    """
-
-    type_schemas: Dict[str, Type[marshmallow.Schema]] = {
-        RunProgramTask.TYPE: RunProgramTask.Schema,
-        CairoPiePath.TYPE: CairoPiePath.Schema,
-    }
-
-    def get_obj_type(self, obj):
-        return obj.TYPE
-
-
 @dataclasses.dataclass(frozen=True)
-class CairoPieTask(Task):
+class Job(Task):
+    reward: int
+    num_of_steps: int
     cairo_pie: CairoPie
-    use_poseidon: bool
+    registry_address: bytearray
+    public_key: bytearray
+    signature: bytearray
 
     def get_program(self) -> StrippedProgram:
         return self.cairo_pie.program
 
-
 @marshmallow_dataclass.dataclass(frozen=True)
 class SimpleBootloaderInput(ValidatedMarshmallowDataclass):
-    tasks: List[TaskSpec] = field(
-        metadata=additional_metadata(marshmallow_field=mfields.List(mfields.Nested(TaskSchema)))
-    )
+    identity: bytearray
+    job: Job
+
     fact_topologies_path: Optional[str]
 
     # If true, the bootloader will put all the outputs in a single page, ignoring the

crates/common/Cargo.toml

@@ -10,13 +10,14 @@ license-file.workspace = true
 [dependencies]
 bincode.workspace = true
 cairo-felt.workspace = true
-futures.workspace= true
+futures.workspace = true
 hex.workspace = true
 libp2p.workspace = true
 libsecp256k1.workspace = true
 num-bigint.workspace = true
 serde_json.workspace = true
 serde.workspace = true
+strum.workspace = true
 tempfile.workspace = true
 thiserror.workspace = true
 tokio.workspace = true

crates/common/src/lib.rs

@@ -2,6 +2,7 @@ pub mod hash_macro;
 pub mod job;
 pub mod job_trace;
 pub mod job_witness;
+pub mod layout;
 pub mod network;
 pub mod process;
 pub mod topic;

crates/compiler/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "sharp-p2p-compiler"
+version.workspace = true
+edition.workspace = true
+repository.workspace = true
+license-file.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-process.workspace = true
+cairo-proof-parser.workspace = true
+futures.workspace = true
+hex.workspace = true
+itertools.workspace = true
+libsecp256k1.workspace = true
+rand.workspace = true
+serde_json.workspace = true
+serde.workspace = true
+sharp-p2p-common.workspace = true
+strum.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+zip-extensions.workspace = true

crates/compiler/build.rs

@@ -0,0 +1,16 @@
+use std::process::Command;
+
+fn main() {
+    // Check if cairo-run command is present
+    check_command("cairo-run");
+
+    // Check if cairo-compile command is present
+    check_command("cairo-compile");
+}
+
+fn check_command(cmd: &str) {
+    match Command::new(cmd).arg("--version").output() {
+        Ok(_) => println!("{} command found", cmd),
+        Err(e) => panic!("Failed to execute {} command: {}", cmd, e),
+    }
+}

crates/compiler/src/cairo_compiler/mod.rs

@@ -0,0 +1,113 @@
+use crate::{errors::CompilerControllerError, traits::CompilerController};
+use async_process::Stdio;
+use futures::Future;
+use sharp_p2p_common::layout::Layout;
+use sharp_p2p_common::{job::Job, process::Process};
+use std::path::PathBuf;
+use std::{io::Read, pin::Pin};
+use tempfile::NamedTempFile;
+use tokio::{process::Command, select, sync::mpsc};
+use tracing::debug;
+
+pub mod tests;
+
+pub struct CairoCompiler {}
+
+impl CairoCompiler {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Default for CairoCompiler {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl CompilerController for CairoCompiler {
+    fn run(
+        &self,
+        program_path: PathBuf,
+        program_input_path: PathBuf,
+    ) -> Result<Process<Result<Job, CompilerControllerError>>, CompilerControllerError> {
+        let (terminate_tx, mut terminate_rx) = mpsc::channel::<()>(10);
+        let future: Pin<Box<dyn Future<Output = Result<Job, CompilerControllerError>> + '_>> =
+            Box::pin(async move {
+                let layout: &str = Layout::RecursiveWithPoseidon.into();
+
+                let output = NamedTempFile::new()?;
+
+                let mut task = Command::new("cairo-compile")
+                    .arg(program_path.as_path())
+                    .arg("--output")
+                    .arg(output.path())
+                    .arg("--proof_mode")
+                    .stdout(Stdio::null())
+                    .spawn()?;
+
+                debug!("program {:?} is compiling... ", program_path);
+
+                loop {
+                    select! {
+                        output = task.wait() => {
+                            debug!("{:?}", output);
+                            if !output?.success() {
+                                return Err(CompilerControllerError::TaskTerminated);
+                            }
+                            let output = task.wait_with_output().await?;
+                            debug!("{:?}", output);
+                            break;
+                        }
+                        Some(()) = terminate_rx.recv() => {
+                            task.start_kill()?;
+                        }
+                    }
+                }
+
+                // output
+                let mut cairo_pie = NamedTempFile::new()?;
+
+                let mut task = Command::new("cairo-run")
+                    .arg("--program")
+                    .arg(output.path())
+                    .arg("--layout")
+                    .arg(layout)
+                    .arg("--program_input")
+                    .arg(program_input_path.as_path())
+                    .arg("--cairo_pie_output")
+                    .arg(cairo_pie.path())
+                    .arg("--print_output")
+                    .stdout(Stdio::null())
+                    .spawn()?;
+
+                debug!("program {:?} is generating PIE... ", program_path);
+
+                loop {
+                    select! {
+                        output = task.wait() => {
+                            debug!("{:?}", output);
+                            if !output?.success() {
+                                return Err(CompilerControllerError::TaskTerminated);
+                            }
+                            let output = task.wait_with_output().await?;
+                            debug!("{:?}", output);
+                            break;
+                        }
+                        Some(()) = terminate_rx.recv() => {
+                            task.start_kill()?;
+                        }
+                    }
+                }
+
+                // cairo run had finished
+                let mut cairo_pie_bytes = Vec::new();
+                cairo_pie.read_to_end(&mut cairo_pie_bytes)?;
+
+                // TODO: calculate details
+                Ok(Job { cairo_pie_compressed: cairo_pie_bytes, ..Default::default() })
+            });
+
+        Ok(Process::new(future, terminate_tx))
+    }
+}

crates/compiler/src/cairo_compiler/tests/mod.rs

@@ -0,0 +1,6 @@
+pub mod models;
+
+#[cfg(all(test, feature = "full_test"))]
+pub mod multiple_job;
+#[cfg(test)]
+pub mod single_job;

crates/compiler/src/cairo_compiler/tests/models.rs

@@ -0,0 +1,16 @@
+use std::{env, path::PathBuf};
+
+pub struct TestFixture {
+    pub program_input_path: PathBuf,
+    pub program_path: PathBuf,
+}
+
+pub fn fixture() -> TestFixture {
+    let ws_root =
+        PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR env not present"))
+            .join("../../");
+    let program_path = ws_root.join("crates/tests/cairo/fibonacci.cairo");
+    let program_input_path = ws_root.join("crates/tests/cairo/fibonacci_input.json");
+
+    TestFixture { program_path, program_input_path }
+}

crates/compiler/src/cairo_compiler/tests/multiple_job.rs

@@ -0,0 +1,46 @@
+use crate::{
+    cairo_compiler::{tests::models::fixture, CairoCompiler},
+    traits::CompilerController,
+};
+use futures::{stream::FuturesUnordered, StreamExt};
+
+#[tokio::test]
+async fn run_multiple_jobs() {
+    let fixture1 = fixture();
+    let fixture2 = fixture();
+
+    let compiler = CairoCompiler::new();
+    let mut futures = FuturesUnordered::new();
+
+    let job1 = compiler.run(fixture1.program_path, fixture1.program_input_path).unwrap();
+    let job2 = compiler.run(fixture2.program_path, fixture2.program_input_path).unwrap();
+
+    futures.push(job1);
+    futures.push(job2);
+
+    while let Some(job) = futures.next().await {
+        job.unwrap();
+    }
+}
+
+#[tokio::test]
+async fn abort_multiple_jobs() {
+    let fixture1 = fixture();
+    let fixture2 = fixture();
+
+    let runner = CairoCompiler::new();
+    let mut futures = FuturesUnordered::new();
+
+    let job1 = runner.run(fixture1.program_path, fixture1.program_input_path).unwrap();
+    let job2 = runner.run(fixture2.program_path, fixture2.program_input_path).unwrap();
+
+    job1.abort().await.unwrap();
+    job2.abort().await.unwrap();
+
+    futures.push(job1);
+    futures.push(job2);
+
+    while let Some(job_trace) = futures.next().await {
+        job_trace.unwrap_err();
+    }
+}

crates/compiler/src/cairo_compiler/tests/single_job.rs

@@ -0,0 +1,22 @@
+use crate::{
+    cairo_compiler::{tests::models::fixture, CairoCompiler},
+    traits::CompilerController,
+};
+
+#[tokio::test]
+async fn run_single_job() {
+    let fixture = fixture();
+
+    let compiler = CairoCompiler::new();
+    compiler.run(fixture.program_path, fixture.program_input_path).unwrap().await.unwrap();
+}
+
+#[tokio::test]
+async fn abort_single_jobs() {
+    let fixture = fixture();
+
+    let runner = CairoCompiler::new();
+    let job = runner.run(fixture.program_path, fixture.program_input_path).unwrap();
+    job.abort().await.unwrap();
+    job.await.unwrap_err();
+}