diff --git a/Cargo.lock b/Cargo.lock index 04e05ae4c3..43d8b46227 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,7 +500,7 @@ name = "codegen_testing" version = "0.11.0" dependencies = [ "anyhow", - "codegen_schema", + "codegen_language_definition", "infra_utils", ] @@ -2055,7 +2055,6 @@ name = "solidity_cargo_tests" version = "0.11.0" dependencies = [ "anyhow", - "codegen_schema", "codegen_testing", "infra_utils", "semver", @@ -2128,7 +2127,6 @@ name = "solidity_testing_sanctuary" version = "0.11.0" dependencies = [ "anyhow", - "codegen_schema", "indicatif", "infra_utils", "rayon", diff --git a/crates/codegen/language/definition/src/model/manifest.rs b/crates/codegen/language/definition/src/model/manifest.rs index b3c9ceb7f0..004d221186 100644 --- a/crates/codegen/language/definition/src/model/manifest.rs +++ b/crates/codegen/language/definition/src/model/manifest.rs @@ -1,9 +1,9 @@ -use crate::model::{Identifier, Item, TriviaParser}; +use crate::model::{Field, Identifier, Item, TriviaParser, VersionSpecifier}; use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens}; use indexmap::IndexSet; use semver::Version; use serde::{Deserialize, Serialize}; -use std::rc::Rc; +use std::{collections::BTreeSet, rc::Rc}; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[derive_spanned_type(ParseInputTokens, WriteOutputTokens)] @@ -36,3 +36,103 @@ pub struct Topic { pub items: Vec>, } + +impl Language { + /// Returns every item in the language definition. + pub fn items(&self) -> impl Iterator { + self.sections + .iter() + .flat_map(|section| &*section.topics) + .flat_map(|topic| topic.items.iter().map(AsRef::as_ref)) + } + + /// Returns a flattened iterator over items along with section and topic they belong to. + pub fn items_with_section(&self) -> impl Iterator)> { + self.sections.iter().flat_map(|section| { + section + .topics + .iter() + .flat_map(move |topic| topic.items.iter().map(move |item| (section, topic, item))) + }) + } + + /// Collects all versions that change the language grammar in a breaking way. + pub fn collect_breaking_versions(&self) -> BTreeSet { + let first = self.versions.first().unwrap().clone(); + let mut res = BTreeSet::from_iter([first]); + + let mut add_spec = |spec: &Option| { + let Some(spec) = spec else { + return; + }; + + match spec.clone() { + VersionSpecifier::Never => (), + VersionSpecifier::From { from } => { + res.insert(from); + } + VersionSpecifier::Till { till } => { + res.insert(till); + } + VersionSpecifier::Range { from, till } => { + res.insert(from); + res.insert(till); + } + } + }; + + for item in self.items() { + match item { + Item::Struct { item } => { + add_spec(&item.enabled); + for field in item.fields.values() { + match field { + Field::Required { .. } => (), + Field::Optional { enabled, .. } => add_spec(enabled), + } + } + } + Item::Enum { item } => { + add_spec(&item.enabled); + for variant in &item.variants { + add_spec(&variant.enabled); + } + } + Item::Repeated { item } => add_spec(&item.enabled), + Item::Separated { item } => add_spec(&item.enabled), + Item::Precedence { item } => { + add_spec(&item.enabled); + for prec in &item.precedence_expressions { + for op in &prec.operators { + add_spec(&op.enabled); + for field in op.fields.values() { + match field { + Field::Required { .. } => (), + Field::Optional { enabled, .. } => add_spec(enabled), + } + } + } + } + for prim in &item.primary_expressions { + add_spec(&prim.enabled); + } + } + Item::Keyword { item } => { + for definition in &item.definitions { + add_spec(&definition.enabled); + add_spec(&definition.reserved); + } + } + Item::Token { item } => { + for definition in &item.definitions { + add_spec(&definition.enabled); + } + } + Item::Fragment { item } => add_spec(&item.enabled), + Item::Trivia { .. } => {} + } + } + + res + } +} diff --git a/crates/codegen/testing/Cargo.toml b/crates/codegen/testing/Cargo.toml index 4544a7949b..925f3a0835 100644 --- a/crates/codegen/testing/Cargo.toml +++ b/crates/codegen/testing/Cargo.toml @@ -7,5 +7,5 @@ publish = false [dependencies] anyhow = { workspace = true } -codegen_schema = { workspace = true } +codegen_language_definition = { workspace = true } infra_utils = { workspace = true } diff --git a/crates/codegen/testing/src/cst_output.rs b/crates/codegen/testing/src/cst_output.rs index ce62202bb4..5cd52e77ca 100644 --- a/crates/codegen/testing/src/cst_output.rs +++ b/crates/codegen/testing/src/cst_output.rs @@ -4,14 +4,14 @@ use std::{ }; use anyhow::{bail, Result}; -use codegen_schema::types::LanguageDefinition; +use codegen_language_definition::model::Language; use infra_utils::{ codegen::{Codegen, CodegenReadWrite}, paths::{FileWalker, PathExtensions}, }; pub fn generate_cst_output_tests( - language: &LanguageDefinition, + language: &Language, data_dir: &Path, output_dir: &Path, ) -> Result<()> { @@ -76,7 +76,7 @@ fn collect_parser_tests(data_dir: &Path) -> Result>, @@ -86,7 +86,7 @@ fn generate_mod_file( .map(|parser_name| format!("#[allow(non_snake_case)] mod {parser_name};")) .collect::(); - let version_breaks = language.collect_version_breaks(); + let version_breaks = language.collect_breaking_versions(); let version_breaks_len = version_breaks.len(); let version_breaks_str = version_breaks .iter() diff --git a/crates/codegen/testing/src/lib.rs b/crates/codegen/testing/src/lib.rs index 1d3094eb5b..1ac29db58b 100644 --- a/crates/codegen/testing/src/lib.rs +++ b/crates/codegen/testing/src/lib.rs @@ -3,7 +3,7 @@ mod cst_output; use std::path::Path; use anyhow::Result; -use codegen_schema::types::LanguageDefinition; +use codegen_language_definition::model::Language; use crate::cst_output::generate_cst_output_tests; @@ -11,7 +11,7 @@ pub trait TestingGeneratorExtensions { fn generate_cst_output_tests(&self, snapshots_dir: &Path, output_dir: &Path) -> Result<()>; } -impl TestingGeneratorExtensions for LanguageDefinition { +impl TestingGeneratorExtensions for Language { fn generate_cst_output_tests(&self, data_dir: &Path, output_dir: &Path) -> Result<()> { generate_cst_output_tests(self, data_dir, output_dir) } diff --git a/crates/solidity/inputs/language/src/grammar.rs b/crates/solidity/inputs/language/src/grammar.rs index 764ac6209e..32b3144916 100644 --- a/crates/solidity/inputs/language/src/grammar.rs +++ b/crates/solidity/inputs/language/src/grammar.rs @@ -23,28 +23,20 @@ use codegen_language_definition::model::Identifier; use codegen_language_definition::model::Item; use indexmap::IndexMap; -use crate::definition::SolidityDefinition; - -/// Materializes the DSL v2 model ([`SolidityDefinition`]) into [`Grammar`]. +/// Materializes the DSL v2 model ([`model::Language`]) into [`Grammar`]. pub trait GrammarConstructorDslV2 { - fn from_dsl_v2() -> Rc; + fn from_dsl_v2(lang: &model::Language) -> Grammar; } impl GrammarConstructorDslV2 for Grammar { - fn from_dsl_v2() -> Rc { - let lang = SolidityDefinition::create(); - - let mut items = HashMap::new(); - - for section in lang.sections { - for topic in section.topics { - let ctx = topic.lexical_context; - - for item in topic.items { - items.insert(item.name().clone(), (ctx.clone(), Rc::clone(&item))); - } - } - } + fn from_dsl_v2(lang: &model::Language) -> Grammar { + // Collect language items into a lookup table to speed up resolution + let mut items = HashMap::from_iter(lang.items_with_section().map(|(_, topic, item)| { + ( + item.name().clone(), + (topic.lexical_context.clone(), Rc::clone(item)), + ) + })); // TODO(#638): To minimize regression in the parser migration, we keep the existing DSL v1 model // of SourceUnit being followed by `EndOfFileTrivia`. @@ -95,12 +87,12 @@ impl GrammarConstructorDslV2 for Grammar { let trailing_trivia = Rc::new(NamedTriviaParser { name: "TrailingTrivia", - def: resolve_trivia(lang.trailing_trivia, &mut ctx), + def: resolve_trivia(lang.trailing_trivia.clone(), &mut ctx), }) as Rc; let eof_trivia = Rc::new(NamedTriviaParser { name: "EndOfFileTrivia", - def: resolve_trivia(lang.leading_trivia, &mut ctx), + def: resolve_trivia(lang.leading_trivia.clone(), &mut ctx), }) as Rc; ctx.resolved.insert( @@ -159,9 +151,9 @@ impl GrammarConstructorDslV2 for Grammar { .iter() .map(|(name, elem)| (name.to_string().leak() as &_, elem.clone())); - Rc::new(Grammar { + Grammar { name: lang.name.to_string(), - versions: BTreeSet::from_iter(lang.versions), + versions: BTreeSet::from_iter(lang.versions.clone()), leading_trivia_parser: leading_trivia.clone(), trailing_trivia_parser: trailing_trivia.clone(), elements: HashMap::from_iter( @@ -171,7 +163,7 @@ impl GrammarConstructorDslV2 for Grammar { .map(|elem| (elem.name(), elem.into())), ), ), - }) + } } } diff --git a/crates/solidity/outputs/cargo/build/src/main.rs b/crates/solidity/outputs/cargo/build/src/main.rs index d80be72b76..3d23d1ba09 100644 --- a/crates/solidity/outputs/cargo/build/src/main.rs +++ b/crates/solidity/outputs/cargo/build/src/main.rs @@ -4,7 +4,7 @@ use cargo_emit::rerun_if_changed; use codegen_grammar::Grammar; use codegen_parser_generator::code_generator::CodeGenerator; use infra_utils::{cargo::CargoWorkspace, paths::PathExtensions}; -use solidity_language::GrammarConstructorDslV2; +use solidity_language::{GrammarConstructorDslV2, SolidityDefinition}; // Instead of the soure crate calling codegen APIs directly in the build script, it invokes this binary, which in turn // calls the codegen APIs (and hence why it's emitting `cargo:` directives). @@ -17,7 +17,8 @@ use solidity_language::GrammarConstructorDslV2; fn main() -> Result<()> { // Generate files in the source crate: { - let grammar = Grammar::from_dsl_v2(); + let lang_def = SolidityDefinition::create(); + let grammar = Grammar::from_dsl_v2(&lang_def); let crate_dir = CargoWorkspace::locate_source_crate("slang_solidity")?; diff --git a/crates/solidity/outputs/cargo/tests/Cargo.toml b/crates/solidity/outputs/cargo/tests/Cargo.toml index 3438c481c6..31fc58d253 100644 --- a/crates/solidity/outputs/cargo/tests/Cargo.toml +++ b/crates/solidity/outputs/cargo/tests/Cargo.toml @@ -7,7 +7,6 @@ publish = false [build-dependencies] anyhow = { workspace = true } -codegen_schema = { workspace = true } codegen_testing = { workspace = true } infra_utils = { workspace = true } solidity_language = { workspace = true } diff --git a/crates/solidity/outputs/cargo/tests/build.rs b/crates/solidity/outputs/cargo/tests/build.rs index 2d2cbe14b2..6b8b50426b 100644 --- a/crates/solidity/outputs/cargo/tests/build.rs +++ b/crates/solidity/outputs/cargo/tests/build.rs @@ -1,17 +1,14 @@ use anyhow::Result; -use codegen_schema::types::LanguageDefinition; use codegen_testing::TestingGeneratorExtensions; use infra_utils::cargo::CargoWorkspace; -use solidity_language::SolidityLanguageExtensions; +use solidity_language::SolidityDefinition; fn main() -> Result<()> { - let language = LanguageDefinition::load_solidity()?; + let lang_def = SolidityDefinition::create(); - language.generate_cst_output_tests( + lang_def.generate_cst_output_tests( &CargoWorkspace::locate_source_crate("solidity_testing_snapshots")?.join("cst_output"), &CargoWorkspace::locate_source_crate("solidity_cargo_tests")? .join("src/cst_output/generated"), - )?; - - Ok(()) + ) } diff --git a/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs b/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs index ac8f589f86..814f882d1d 100644 --- a/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs +++ b/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs @@ -86,15 +86,18 @@ mod YulExpression; #[allow(non_snake_case)] mod YulLeaveStatement; -pub const VERSION_BREAKS: [Version; 19] = [ +pub const VERSION_BREAKS: [Version; 22] = [ Version::new(0, 4, 11), + Version::new(0, 4, 14), Version::new(0, 4, 21), Version::new(0, 4, 22), Version::new(0, 5, 0), Version::new(0, 5, 3), + Version::new(0, 5, 10), Version::new(0, 6, 0), Version::new(0, 6, 2), Version::new(0, 6, 5), + Version::new(0, 6, 8), Version::new(0, 6, 11), Version::new(0, 7, 0), Version::new(0, 7, 1), diff --git a/crates/solidity/outputs/npm/build/src/main.rs b/crates/solidity/outputs/npm/build/src/main.rs index 9b44f5cc5a..42c2441eb2 100644 --- a/crates/solidity/outputs/npm/build/src/main.rs +++ b/crates/solidity/outputs/npm/build/src/main.rs @@ -4,7 +4,7 @@ use cargo_emit::rerun_if_changed; use codegen_grammar::Grammar; use codegen_parser_generator::code_generator::CodeGenerator; use infra_utils::{cargo::CargoWorkspace, paths::PathExtensions}; -use solidity_language::GrammarConstructorDslV2; +use solidity_language::{GrammarConstructorDslV2, SolidityDefinition}; // Instead of the soure crate calling codegen APIs directly in the build script, it invokes this binary, which in turn // calls the codegen APIs (and hence why it's emitting `cargo:` directives). @@ -17,7 +17,8 @@ use solidity_language::GrammarConstructorDslV2; fn main() -> Result<()> { // Generate files in the source crate: { - let grammar = Grammar::from_dsl_v2(); + let lang_def = SolidityDefinition::create(); + let grammar = Grammar::from_dsl_v2(&lang_def); let crate_dir = CargoWorkspace::locate_source_crate("solidity_npm_crate")?; diff --git a/crates/solidity/testing/sanctuary/Cargo.toml b/crates/solidity/testing/sanctuary/Cargo.toml index 8dbbed26c6..0c63fcd9b6 100644 --- a/crates/solidity/testing/sanctuary/Cargo.toml +++ b/crates/solidity/testing/sanctuary/Cargo.toml @@ -7,7 +7,6 @@ publish = false [dependencies] anyhow = { workspace = true } -codegen_schema = { workspace = true } infra_utils = { workspace = true } indicatif = { workspace = true } rayon = { workspace = true } diff --git a/crates/solidity/testing/sanctuary/src/main.rs b/crates/solidity/testing/sanctuary/src/main.rs index bd2c73443a..15818dae87 100644 --- a/crates/solidity/testing/sanctuary/src/main.rs +++ b/crates/solidity/testing/sanctuary/src/main.rs @@ -4,12 +4,11 @@ mod reporting; use std::{collections::BTreeSet, path::Path}; use anyhow::Result; -use codegen_schema::types::LanguageDefinition; use infra_utils::paths::PathExtensions; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use semver::Version; use slang_solidity::{kinds::ProductionKind, language::Language}; -use solidity_language::SolidityLanguageExtensions; +use solidity_language::SolidityDefinition; use solidity_testing_utils::version_pragmas::extract_version_pragmas; use crate::{ @@ -18,11 +17,10 @@ use crate::{ }; fn main() { + let versions = SolidityDefinition::create().collect_breaking_versions(); + // Fail the parent process if a child thread panics: std::panic::catch_unwind(|| -> Result<()> { - let language = &LanguageDefinition::load_solidity()?; - let versions = language.collect_version_breaks(); - for dataset in get_all_datasets()? { process_dataset(&dataset, &versions)?; } diff --git a/crates/solidity/testing/solc/src/keywords/mod.rs b/crates/solidity/testing/solc/src/keywords/mod.rs index 3e17cb9735..9afd8a7598 100644 --- a/crates/solidity/testing/solc/src/keywords/mod.rs +++ b/crates/solidity/testing/solc/src/keywords/mod.rs @@ -61,27 +61,23 @@ fn generate_test_cases(language: &Language) -> Vec { let mut test_cases = vec![]; let mut variations = HashSet::new(); - for section in &language.sections { - for topic in §ion.topics { - for item in &topic.items { - let Item::Keyword { item } = item.as_ref() else { - continue; - }; - - if !should_test_item(item.name.as_str()) { - continue; - } + for item in language.items() { + let Item::Keyword { item } = item else { + continue; + }; + + if !should_test_item(item.name.as_str()) { + continue; + } - for definition in &item.definitions { - for variation in definition.value.collect_variations() { - assert!( - variations.insert(format!("{} -> {}", item.identifier, variation)), - "Duplicate variation: {variation}" - ); + for definition in &item.definitions { + for variation in definition.value.collect_variations() { + assert!( + variations.insert(format!("{} -> {}", item.identifier, variation)), + "Duplicate variation: {variation}" + ); - test_cases.push(TestCase::new(language, item, definition, variation)); - } - } + test_cases.push(TestCase::new(language, item, definition, variation)); } } }