merging with main

.changeset/fluffy-elephants-beg.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+rename `parser/Parser.supportedVersions()` API to `utils/LanguageFacts.supportedVersions()`.

.changeset/gentle-onions-drum.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+expose the `BingingGraph` API to allow querying definitions/references between source files.

.changeset/pink-flowers-draw.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+add a `CompilationBuilder` API to incrementally load and resolve source files and their imports.

.github/workflows/sanctuary.yml

@@ -22,8 +22,41 @@ on:
         default: false
 
 jobs:
-  sanctuary:
+  singleShard:
     runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync)
+    outputs:
+      __SLANG_SANCTUARY_SHARD_RESULTS__0: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__0 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__1: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__1 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__2: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__2 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__3: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__3 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__4: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__4 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__5: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__5 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__6: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__6 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__7: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__7 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__8: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__8 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__9: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__9 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__10: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__10 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__11: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__11 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__12: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__12 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__13: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__13 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__14: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__14 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__15: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__15 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__16: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__16 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__17: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__17 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__18: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__18 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__19: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__19 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__20: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__20 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__21: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__21 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__22: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__22 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__23: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__23 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__24: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__24 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__25: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__25 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__26: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__26 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__27: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__27 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__28: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__28 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__29: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__29 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__30: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__30 }}"
+      __SLANG_SANCTUARY_SHARD_RESULTS__31: "${{ steps.output-shard-results.outputs.__SLANG_SANCTUARY_SHARD_RESULTS__31 }}"
 
     strategy:
       fail-fast: false # Continue running all shards even if some fail.
@@ -45,9 +78,25 @@ jobs:
           - 13
           - 14
           - 15
+          - 16
+          - 17
+          - 18
+          - 19
+          - 20
+          - 21
+          - 22
+          - 23
+          - 24
+          - 25
+          - 26
+          - 27
+          - 28
+          - 29
+          - 30
+          - 31
 
     env:
-      SHARDS_COUNT: 16 # Length of the 'shard_index' array above.
+      SHARDS_COUNT: 32 # Length of the 'shard_index' array above.
 
     steps:
       - name: "Checkout Repository"
@@ -59,4 +108,28 @@ jobs:
       - name: "infra run solidity_testing_sanctuary"
         uses: "./.github/actions/devcontainer/run"
         with:
-          runCmd: "./scripts/bin/infra run --release --bin solidity_testing_sanctuary -- --shards-count ${{ env.SHARDS_COUNT }} --shard-index ${{ matrix.shard_index }} ${{ inputs.check_bindings == true && '--check-bindings' || '' }} ${{ inputs.chain }} ${{ inputs.network }}"
+          runCmd: "./scripts/bin/infra run --release --bin solidity_testing_sanctuary -- test --shards-count ${{ env.SHARDS_COUNT }} --shard-index ${{ matrix.shard_index }} ${{ inputs.check_bindings == true && '--check-bindings' || '' }} ${{ inputs.chain }} ${{ inputs.network }}"
+
+      - name: "Write shard results to output"
+        if: "!cancelled()"
+        id: "output-shard-results"
+        run: 'echo "__SLANG_SANCTUARY_SHARD_RESULTS__${{ matrix.shard_index }}=$(cat target/__SLANG_SANCTUARY_SHARD_RESULTS__.json)" >> "$GITHUB_OUTPUT"'
+
+  combinedResults:
+    runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync)
+    needs: "singleShard"
+    if: "!cancelled()"
+    steps:
+      - name: "Checkout Repository"
+        uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+
+      - name: "Restore Cache"
+        uses: "./.github/actions/cache/restore"
+
+      - name: "Output shards results"
+        run: "echo '${{ toJSON(needs.singleShard.outputs) }}' > __SLANG_SANCTUARY_MATRIX_RESULTS__.json"
+
+      - name: "Show combined results"
+        uses: "./.github/actions/devcontainer/run"
+        with:
+          runCmd: "./scripts/bin/infra run --bin solidity_testing_sanctuary -- show-combined-results __SLANG_SANCTUARY_MATRIX_RESULTS__.json"

crates/codegen/runtime/cargo/crate/Cargo.toml

@@ -12,6 +12,7 @@ description = "Cargo runtime copied over by codegen"
 default = []
 __experimental_bindings_api = ["dep:metaslang_bindings"]
 __private_ariadne_errors = ["dep:ariadne"]
+__private_compilation_api = []
 __private_testing_utils = []
 
 [build-dependencies]

crates/codegen/runtime/cargo/crate/src/extensions/bindings/mod.rs

@@ -0,0 +1,12 @@
+use semver::Version;
+
+use crate::bindings::BindingGraph;
+use crate::parser::ParserInitializationError;
+
+#[allow(clippy::needless_pass_by_value)]
+pub fn add_built_ins(
+    _binding_graph: &mut BindingGraph,
+    _version: Version,
+) -> Result<(), ParserInitializationError> {
+    unreachable!("Built-ins are Solidity-specific")
+}

crates/codegen/runtime/cargo/crate/src/extensions/compilation/mod.rs

@@ -0,0 +1,15 @@
+use crate::cst::Cursor;
+
+pub struct ImportPathsExtractor;
+
+impl ImportPathsExtractor {
+    pub fn new() -> Self {
+        Self
+    }
+
+    #[allow(clippy::unused_self)]
+    #[allow(clippy::needless_pass_by_value)]
+    pub fn extract(&self, _: Cursor) -> Vec<Cursor> {
+        unreachable!("Import paths are Solidity-specific")
+    }
+}

crates/codegen/runtime/cargo/crate/src/extensions/mod.rs

@@ -0,0 +1,8 @@
+#[cfg(all(
+    feature = "__experimental_bindings_api",
+    feature = "__private_compilation_api"
+))]
+pub mod compilation;
+
+#[cfg(feature = "__experimental_bindings_api")]
+pub mod bindings;

crates/codegen/runtime/cargo/crate/src/lib.rs

@@ -2,6 +2,7 @@
 // The final code (generated in output crates) is checked for dead-code anyways.
 #![allow(dead_code, unused_imports)]
 
+mod extensions;
 mod runtime;
 
 pub use runtime::*;

crates/codegen/runtime/cargo/crate/src/runtime/bindings/built_ins.rs.jinja2

@@ -1,7 +1,8 @@
 use semver::Version;
 
+// TODO: This should be moved to the Solidity-specific 'extensions' sub-module.
 #[allow(unused_variables)]
-pub fn get_contents(version: &Version) -> &'static str {
+pub fn get_built_ins_contents(version: &Version) -> &'static str {
     {%- if not rendering_in_stubs -%}
         {%- for version in model.bindings.built_ins_versions %}
             {%- if not loop.first -%}

crates/codegen/runtime/cargo/crate/src/runtime/bindings/mod.rs

@@ -2,31 +2,46 @@
 mod binding_rules;
 
 #[path = "generated/built_ins.rs"]
-mod built_ins;
+pub mod built_ins;
 
-use std::sync::Arc;
+use std::rc::Rc;
 
-use metaslang_bindings::{self, PathResolver};
 use semver::Version;
 
 use crate::cst::KindTypes;
 
-pub type Bindings = metaslang_bindings::Bindings<KindTypes>;
+pub type BindingGraph = metaslang_bindings::BindingGraph<KindTypes>;
 pub type Definition<'a> = metaslang_bindings::Definition<'a, KindTypes>;
 pub type Reference<'a> = metaslang_bindings::Reference<'a, KindTypes>;
+pub type BindingLocation = metaslang_bindings::BindingLocation<KindTypes>;
+pub type UserFileLocation = metaslang_bindings::UserFileLocation<KindTypes>;
+
+pub use metaslang_bindings::{BuiltInLocation, PathResolver};
+
+use crate::parser::ParserInitializationError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum BindingGraphInitializationError {
+    #[error(transparent)]
+    ParserInitialization(#[from] ParserInitializationError),
+}
 
 pub fn create_with_resolver(
     version: Version,
-    resolver: Arc<dyn PathResolver + Sync + Send>,
-) -> Bindings {
-    Bindings::create(version, binding_rules::BINDING_RULES_SOURCE, resolver)
+    resolver: Rc<dyn PathResolver<KindTypes>>,
+) -> Result<BindingGraph, ParserInitializationError> {
+    let mut binding_graph = BindingGraph::create(
+        version.clone(),
+        binding_rules::BINDING_RULES_SOURCE,
+        resolver,
+    );
+
+    crate::extensions::bindings::add_built_ins(&mut binding_graph, version)?;
+
+    Ok(binding_graph)
 }
 
 #[cfg(feature = "__private_testing_utils")]
 pub fn get_binding_rules() -> &'static str {
     binding_rules::BINDING_RULES_SOURCE
 }
-
-pub fn get_built_ins(version: &semver::Version) -> &'static str {
-    built_ins::get_contents(version)
-}

crates/codegen/runtime/cargo/crate/src/runtime/compilation/file.rs

@@ -0,0 +1,45 @@
+use std::collections::BTreeMap;
+
+use metaslang_cst::text_index::TextIndex;
+
+use crate::cst::{Cursor, Node};
+
+#[derive(Clone)]
+pub struct File {
+    id: String,
+    tree: Node,
+
+    resolved_imports: BTreeMap<usize, String>,
+}
+
+impl File {
+    pub(super) fn new(id: String, tree: Node) -> Self {
+        Self {
+            id,
+            tree,
+
+            resolved_imports: BTreeMap::new(),
+        }
+    }
+
+    pub fn id(&self) -> &str {
+        &self.id
+    }
+
+    pub fn tree(&self) -> &Node {
+        &self.tree
+    }
+
+    pub fn create_tree_cursor(&self) -> Cursor {
+        self.tree.clone().cursor_with_offset(TextIndex::ZERO)
+    }
+
+    pub(super) fn resolve_import(&mut self, import_path: &Cursor, destination_file_id: String) {
+        self.resolved_imports
+            .insert(import_path.node().id(), destination_file_id);
+    }
+
+    pub(super) fn resolved_import(&self, import_path: &Cursor) -> Option<&String> {
+        self.resolved_imports.get(&import_path.node().id())
+    }
+}

crates/codegen/runtime/cargo/crate/src/runtime/compilation/internal_builder.rs

@@ -0,0 +1,93 @@
+use std::collections::BTreeMap;
+use std::rc::Rc;
+
+use metaslang_cst::nodes::Node;
+use semver::Version;
+
+use crate::compilation::{CompilationUnit, File};
+use crate::cst::Cursor;
+use crate::extensions::compilation::ImportPathsExtractor;
+use crate::parser::{Parser, ParserInitializationError};
+
+pub struct InternalCompilationBuilder {
+    parser: Parser,
+    imports: ImportPathsExtractor,
+    files: BTreeMap<String, File>,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum CompilationInitializationError {
+    #[error(transparent)]
+    ParserInitialization(#[from] ParserInitializationError),
+}
+
+impl InternalCompilationBuilder {
+    pub fn create(language_version: Version) -> Result<Self, CompilationInitializationError> {
+        let parser = Parser::create(language_version)?;
+
+        Ok(Self {
+            parser,
+            imports: ImportPathsExtractor::new(),
+            files: BTreeMap::new(),
+        })
+    }
+
+    pub fn add_file(&mut self, id: String, contents: &str) -> AddFileResponse {
+        if self.files.contains_key(&id) {
+            // Already added. No need to process it again:
+            return AddFileResponse {
+                import_paths: vec![],
+            };
+        }
+
+        let parse_output = self.parser.parse(Parser::ROOT_KIND, contents);
+
+        let import_paths = self.imports.extract(parse_output.create_tree_cursor());
+
+        let file = File::new(
+            id.clone(),
+            Node::Nonterminal(Rc::clone(parse_output.tree())),
+        );
+        self.files.insert(id, file);
+
+        AddFileResponse { import_paths }
+    }
+
+    pub fn resolve_import(
+        &mut self,
+        source_file_id: &str,
+        import_path: &Cursor,
+        destination_file_id: String,
+    ) -> Result<(), ResolveImportError> {
+        self.files
+            .get_mut(source_file_id)
+            .ok_or_else(|| ResolveImportError::SourceFileNotFound(source_file_id.to_owned()))?
+            .resolve_import(import_path, destination_file_id);
+
+        Ok(())
+    }
+
+    pub fn build(&self) -> CompilationUnit {
+        let language_version = self.parser.language_version().to_owned();
+
+        let files = self
+            .files
+            .iter()
+            .map(|(id, file)| (id.to_owned(), Rc::new(file.to_owned())))
+            .collect();
+
+        CompilationUnit::new(language_version, files)
+    }
+}
+
+pub struct AddFileResponse {
+    pub import_paths: Vec<Cursor>,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum ResolveImportError {
+    #[error(
+        "Source file not found: '{0}'. Make sure to add it first, before resolving its imports."
+    )]
+    SourceFileNotFound(String),
+}

crates/codegen/runtime/cargo/crate/src/runtime/compilation/mod.rs

@@ -0,0 +1,7 @@
+mod file;
+mod internal_builder;
+mod unit;
+
+pub use file::File;
+pub use internal_builder::{AddFileResponse, InternalCompilationBuilder};
+pub use unit::CompilationUnit;