From f17c0f80e7ac4d9208fb696cf471c7ca68caafee Mon Sep 17 00:00:00 2001 From: Omar Tawfik <15987992+OmarTawfik@users.noreply.github.com> Date: Mon, 23 Oct 2023 02:06:51 -0700 Subject: [PATCH] introduce Language DSL v2 (#610) ## About DSL v2 This introduces DSL v2 that describes the grammar in terms of AST types, not EBNF rules: - The new model types in [types.rs](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/codegen/language/definition/src/model/types.rs) are much more accurate/restrictive in describing the semantics of different productions, which removes the need to do a lot of validation that we had for earlier grammars. - The DSL inside a Rust macro invocation in [definition.rs](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/solidity/inputs/language/src/definition.rs) are automatically validated on every `cargo check`, are formatted by `rustfmt`, and have definition/references/rename IDE support because of the backing types emitted in [emitter.rs](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/codegen/language/definition/src/compiler/emitter.rs). ## Behind the Scenes The magic happens because of the new `codegen_language_internal_macros` crate, which reads the model types, and generates implementations for three things: 1. `Spanned` which rewrites all fields in `Spanned` types that preserve the input token spans for validatation. 2. `ParseInputTokens` which uses `syn` crate to parse the tokens into the backing Rust types. 3. `WriteOutputTokens` which serializes the grammar `Spanned` types into a Rust expression that generates the definition using the original types (without `Spanned`), so that they can be used by client crates. ## Next Steps We unfortunately now have three sources of truth for the language, that we are manually keeping in sync for now: 1. The YAML grammar ([here](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/solidity/inputs/language/definition/manifest.yml)) is used for produce the HTML spec. 4. The DSL v1 grammar ([here](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/solidity/inputs/language/src/dsl.rs)) is used for produce the parser. 5. The DSL v2 grammar ([here](https://github.com/OmarTawfik-forks/slang/blob/dsl-v2/crates/solidity/inputs/language/src/definition.rs)) introduced in this PR, and not used in anything yet. I will start to delete the YAML grammar, and move `codegen_spec` to use DSL v2, while in parallel starting a discussion about removing DSL v1, as it is a bigger chunk of work that requires coordinating with other ongoing parser/AST work. ## Areas of Improvement Using `serde` to serialize/deserialize was not originally possible, because its data model does not support token spans, which cannot be serialized, recreated, or persisted outside the context of macro invocations. So I moved to use `syn` for parsing for now. It is working well, although it has a few caveats: 1. I needed to implement a parser, along with custom implementations for `Rc`/`IndexMap`/`Box` and other data structures, that `serde` handles by default. 2. The parser is type driven, which means it is strict, and expects fields to be defined exactly in the same order as the backing Rust types. `struct X { a: u8, b: u8 }` has to be declared as `X(a = 1, b = 2)`, not `X(b = 2, a = 1)`. I think I found a solution/workaround to the `serde` data model limitation, that will let me remove all these extra implementations and replace it with a `serde` deserializer, but I will look into this in a later iteration, since it is not blocking us right now. Additionally, validation can still be tightened in a few places, but mostly for keeping the DSL lean/readable, not about correctness, so I will also delay this work for later. --- Cargo.lock | 108 +- Cargo.toml | 16 + crates/codegen/language/definition/Cargo.toml | 21 + .../src/compiler/analysis/definitions.rs | 187 + .../definition/src/compiler/analysis/mod.rs | 82 + .../src/compiler/analysis/reachability.rs | 78 + .../src/compiler/analysis/references.rs | 570 +++ .../definition/src/compiler/emitter.rs | 71 + .../language/definition/src/compiler/mod.rs | 28 + .../definition/src/compiler/versions.rs | 369 ++ .../definition/src/internals/errors.rs | 85 + .../language/definition/src/internals/mod.rs | 9 + .../internals/parse_input_tokens/adapter.rs | 35 + .../internals/parse_input_tokens/helpers.rs | 160 + .../parse_input_tokens/implementations.rs | 190 + .../src/internals/parse_input_tokens/mod.rs | 26 + .../definition/src/internals/spanned/mod.rs | 71 + .../write_output_tokens/implementations.rs | 158 + .../src/internals/write_output_tokens/mod.rs | 5 + crates/codegen/language/definition/src/lib.rs | 6 + .../definition/src/model/identifier.rs | 45 + .../language/definition/src/model/mod.rs | 5 + .../language/definition/src/model/types.rs | 308 ++ .../language/internal_macros/Cargo.toml | 15 + .../internal_macros/src/derive/mod.rs | 7 + .../src/derive/parse_input_tokens.rs | 125 + .../internal_macros/src/derive/spanned.rs | 119 + .../src/derive/write_output_tokens.rs | 74 + .../language/internal_macros/src/lib.rs | 65 + .../language/internal_macros/src/model/mod.rs | 109 + .../language/internal_macros/src/utils/mod.rs | 7 + crates/codegen/language/macros/Cargo.toml | 12 + crates/codegen/language/macros/src/lib.rs | 7 + crates/codegen/language/tests/Cargo.toml | 14 + .../duplicate_expression_name/test.rs | 33 + .../duplicate_expression_name/test.stderr | 5 + .../definitions/duplicate_item_name/test.rs | 22 + .../duplicate_item_name/test.stderr | 5 + .../duplicate_variant_name/test.rs | 26 + .../duplicate_variant_name/test.stderr | 5 + crates/codegen/language/tests/src/fail/mod.rs | 33 + .../fail/parsing/duplicate_map_key/test.rs | 31 + .../parsing/duplicate_map_key/test.stderr | 5 + .../fail/parsing/duplicate_set_entry/test.rs | 21 + .../parsing/duplicate_set_entry/test.stderr | 5 + .../src/fail/parsing/empty_string/test.rs | 21 + .../src/fail/parsing/empty_string/test.stderr | 5 + .../src/fail/parsing/missing_field/test.rs | 15 + .../fail/parsing/missing_field/test.stderr | 5 + .../parsing/non_existing_path_buf/test.rs | 32 + .../parsing/non_existing_path_buf/test.stderr | 5 + .../fail/parsing/unrecognized_field/test.rs | 22 + .../parsing/unrecognized_field/test.stderr | 5 + .../fail/parsing/unrecognized_variant/test.rs | 24 + .../parsing/unrecognized_variant/test.stderr | 5 + .../src/fail/reachability/unreachable/test.rs | 25 + .../fail/reachability/unreachable/test.stderr | 11 + .../fail/reachability/unused_version/test.rs | 27 + .../reachability/unused_version/test.stderr | 5 + .../fail/references/disabled_too_late/test.rs | 40 + .../references/disabled_too_late/test.stderr | 5 + .../fail/references/enabled_too_early/test.rs | 40 + .../references/enabled_too_early/test.stderr | 5 + .../fail/references/invalid_filter/test.rs | 21 + .../references/invalid_filter/test.stderr | 5 + .../fail/references/invalid_version/test.rs | 52 + .../references/invalid_version/test.stderr | 17 + .../fail/references/unknown_reference/test.rs | 30 + .../references/unknown_reference/test.stderr | 5 + .../references/unordered_version_pair/test.rs | 34 + .../unordered_version_pair/test.stderr | 5 + .../fail/references/version_not_found/test.rs | 30 + .../references/version_not_found/test.stderr | 5 + crates/codegen/language/tests/src/lib.rs | 4 + crates/codegen/language/tests/src/pass/mod.rs | 1 + .../language/tests/src/pass/tiny_language.rs | 130 + crates/codegen/schema/Cargo.toml | 1 + .../src/validation/visitors/receivers.rs | 4 +- .../src/validation/visitors/version_set.rs | 298 +- crates/infra/cli/src/commands/run/mod.rs | 1 + crates/infra/cli/src/commands/test/mod.rs | 18 +- crates/solidity/inputs/language/Cargo.toml | 4 +- .../inputs/language/src/definition.rs | 4019 +++++++++++++++++ crates/solidity/inputs/language/src/lib.rs | 3 + 84 files changed, 8269 insertions(+), 93 deletions(-) create mode 100644 crates/codegen/language/definition/Cargo.toml create mode 100644 crates/codegen/language/definition/src/compiler/analysis/definitions.rs create mode 100644 crates/codegen/language/definition/src/compiler/analysis/mod.rs create mode 100644 crates/codegen/language/definition/src/compiler/analysis/reachability.rs create mode 100644 crates/codegen/language/definition/src/compiler/analysis/references.rs create mode 100644 crates/codegen/language/definition/src/compiler/emitter.rs create mode 100644 crates/codegen/language/definition/src/compiler/mod.rs create mode 100644 crates/codegen/language/definition/src/compiler/versions.rs create mode 100644 crates/codegen/language/definition/src/internals/errors.rs create mode 100644 crates/codegen/language/definition/src/internals/mod.rs create mode 100644 crates/codegen/language/definition/src/internals/parse_input_tokens/adapter.rs create mode 100644 crates/codegen/language/definition/src/internals/parse_input_tokens/helpers.rs create mode 100644 crates/codegen/language/definition/src/internals/parse_input_tokens/implementations.rs create mode 100644 crates/codegen/language/definition/src/internals/parse_input_tokens/mod.rs create mode 100644 crates/codegen/language/definition/src/internals/spanned/mod.rs create mode 100644 crates/codegen/language/definition/src/internals/write_output_tokens/implementations.rs create mode 100644 crates/codegen/language/definition/src/internals/write_output_tokens/mod.rs create mode 100644 crates/codegen/language/definition/src/lib.rs create mode 100644 crates/codegen/language/definition/src/model/identifier.rs create mode 100644 crates/codegen/language/definition/src/model/mod.rs create mode 100644 crates/codegen/language/definition/src/model/types.rs create mode 100644 crates/codegen/language/internal_macros/Cargo.toml create mode 100644 crates/codegen/language/internal_macros/src/derive/mod.rs create mode 100644 crates/codegen/language/internal_macros/src/derive/parse_input_tokens.rs create mode 100644 crates/codegen/language/internal_macros/src/derive/spanned.rs create mode 100644 crates/codegen/language/internal_macros/src/derive/write_output_tokens.rs create mode 100644 crates/codegen/language/internal_macros/src/lib.rs create mode 100644 crates/codegen/language/internal_macros/src/model/mod.rs create mode 100644 crates/codegen/language/internal_macros/src/utils/mod.rs create mode 100644 crates/codegen/language/macros/Cargo.toml create mode 100644 crates/codegen/language/macros/src/lib.rs create mode 100644 crates/codegen/language/tests/Cargo.toml create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.rs create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.rs create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.rs create mode 100644 crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/mod.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/empty_string/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/empty_string/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/missing_field/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/missing_field/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.rs create mode 100644 crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/reachability/unreachable/test.rs create mode 100644 crates/codegen/language/tests/src/fail/reachability/unreachable/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/reachability/unused_version/test.rs create mode 100644 crates/codegen/language/tests/src/fail/reachability/unused_version/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/disabled_too_late/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/disabled_too_late/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/enabled_too_early/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/enabled_too_early/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/invalid_filter/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/invalid_filter/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/invalid_version/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/invalid_version/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/unknown_reference/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/unknown_reference/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.stderr create mode 100644 crates/codegen/language/tests/src/fail/references/version_not_found/test.rs create mode 100644 crates/codegen/language/tests/src/fail/references/version_not_found/test.stderr create mode 100644 crates/codegen/language/tests/src/lib.rs create mode 100644 crates/codegen/language/tests/src/pass/mod.rs create mode 100644 crates/codegen/language/tests/src/pass/tiny_language.rs create mode 100644 crates/solidity/inputs/language/src/definition.rs diff --git a/Cargo.lock b/Cargo.lock index e773e75c19..65d8d2c412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,6 +168,15 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "basic-toml" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bfc506e7a2370ec239e1d072507b2a80c833083699d3c6fa176fbb4de8448c6" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -346,7 +355,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -373,6 +382,54 @@ dependencies = [ "strum_macros", ] +[[package]] +name = "codegen_language_definition" +version = "0.10.0" +dependencies = [ + "Inflector", + "codegen_language_internal_macros", + "indexmap 1.9.3", + "infra_utils", + "itertools", + "proc-macro2", + "quote", + "semver", + "serde", + "strum", + "strum_macros", + "syn 2.0.29", + "thiserror", +] + +[[package]] +name = "codegen_language_internal_macros" +version = "0.10.0" +dependencies = [ + "itertools", + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "codegen_language_macros" +version = "0.10.0" +dependencies = [ + "codegen_language_definition", +] + +[[package]] +name = "codegen_language_tests" +version = "0.10.0" +dependencies = [ + "codegen_language_definition", + "codegen_language_macros", + "infra_utils", + "rayon", + "semver", + "trybuild", +] + [[package]] name = "codegen_parser_generator" version = "0.10.0" @@ -406,6 +463,7 @@ dependencies = [ "anyhow", "indexmap 1.9.3", "infra_utils", + "itertools", "schemars", "semver", "serde", @@ -539,7 +597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" dependencies = [ "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -657,6 +715,12 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.13" @@ -1152,7 +1216,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1454,7 +1518,7 @@ checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1598,6 +1662,8 @@ dependencies = [ "bson", "cargo-emit", "codegen_grammar", + "codegen_language_definition", + "codegen_language_macros", "codegen_schema", "infra_utils", "once_cell", @@ -1723,9 +1789,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2", "quote", @@ -1761,6 +1827,15 @@ dependencies = [ "unic-segment", ] +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.2.6" @@ -1788,7 +1863,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1877,6 +1952,21 @@ dependencies = [ "winnow", ] +[[package]] +name = "trybuild" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196a58260a906cedb9bf6d8034b6379d0c11f552416960452f267402ceeddff1" +dependencies = [ + "basic-toml", + "glob", + "once_cell", + "serde", + "serde_derive", + "serde_json", + "termcolor", +] + [[package]] name = "typenum" version = "1.16.0" @@ -2057,7 +2147,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -2079,7 +2169,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index ab6d4e5287..ae176f5a4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ resolver = "2" members = [ "crates/codegen/ebnf", "crates/codegen/grammar", + "crates/codegen/language/definition", + "crates/codegen/language/internal_macros", + "crates/codegen/language/macros", + "crates/codegen/language/tests", "crates/codegen/parser/generator", "crates/codegen/parser/runtime", "crates/codegen/schema", @@ -35,6 +39,10 @@ members = [ # codegen_ebnf = { path = "crates/codegen/ebnf" } codegen_grammar = { path = "crates/codegen/grammar" } +codegen_language_definition = { path = "crates/codegen/language/definition" } +codegen_language_internal_macros = { path = "crates/codegen/language/internal_macros" } +codegen_language_macros = { path = "crates/codegen/language/macros" } +codegen_language_tests = { path = "crates/codegen/language/tests" } codegen_parser_generator = { path = "crates/codegen/parser/generator" } codegen_parser_runtime = { path = "crates/codegen/parser/runtime" } codegen_schema = { path = "crates/codegen/schema" } @@ -86,9 +94,17 @@ serde_yaml = { version = "0.9.19" } similar-asserts = { version = "1.4.2" } strum = { version = "0.24.0" } strum_macros = { version = "0.24.0" } +syn = { version = "2.0.29", features = [ + "fold", + "full", + "extra-traits", + "parsing", + "printing", +] } tera = { version = "1.19.0" } terminal_size = { version = "0.2.6" } thiserror = { version = "1.0.40" } +trybuild = { version = "1.0.85" } toml = { version = "0.7.6" } typed-arena = { version = "2.0.2" } url = { version = "2.3.1" } diff --git a/crates/codegen/language/definition/Cargo.toml b/crates/codegen/language/definition/Cargo.toml new file mode 100644 index 0000000000..498d2b0c02 --- /dev/null +++ b/crates/codegen/language/definition/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "codegen_language_definition" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +[dependencies] +codegen_language_internal_macros = { workspace = true } +indexmap = { workspace = true } +Inflector = { workspace = true } +infra_utils = { workspace = true } +itertools = { workspace = true } +proc-macro2 = { workspace = true } +quote = { workspace = true } +semver = { workspace = true } +serde = { workspace = true } +strum = { workspace = true } +strum_macros = { workspace = true } +syn = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/codegen/language/definition/src/compiler/analysis/definitions.rs b/crates/codegen/language/definition/src/compiler/analysis/definitions.rs new file mode 100644 index 0000000000..af9057a7c3 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/analysis/definitions.rs @@ -0,0 +1,187 @@ +use crate::{ + compiler::{ + analysis::{Analysis, ItemMetadata}, + versions::{VersionRange, VersionSet}, + }, + internals::Spanned, + spanned::Item, + Identifier, +}; +use semver::Version; +use std::{collections::HashSet, fmt::Debug}; + +pub fn analyze_definitions(analysis: &mut Analysis) { + collect_top_level_items(analysis); + + check_enum_items(analysis); + check_precedence_items(analysis); +} + +fn collect_top_level_items(analysis: &mut Analysis) { + for item in analysis.language.clone().items() { + let name = get_item_name(item); + let defined_in = calculate_defined_in(analysis, item); + + if analysis.metadata.contains_key(&**name) { + analysis.errors.add(name, &Errors::ExistingItem(name)); + } + + analysis.metadata.insert( + (**name).to_owned(), + ItemMetadata { + name: name.clone(), + kind: item.into(), + + defined_in, + used_in: VersionSet::empty(), + + referenced_from: Vec::new(), + referenced_items: Vec::new(), + }, + ); + } +} + +fn check_enum_items(analysis: &mut Analysis) { + for item in analysis.language.clone().items() { + let Item::Enum { item } = item else { + continue; + }; + + let mut variants = HashSet::new(); + + for variant in &item.variants { + let name = &variant.name; + if !variants.insert(&**name) { + analysis.errors.add(name, &Errors::ExistingVariant(name)); + } + } + } +} + +fn check_precedence_items(analysis: &mut Analysis) { + // Make sure that all expressions have unique names across the entire language, since they produce their own kinds. + // However, they cannot be referenced from anywhere else, so no need to add them to top-level definitions. + let mut all_expressions = HashSet::new(); + + for item in analysis.language.clone().items() { + let Item::Precedence { item } = item else { + continue; + }; + + // Additionally, make sure that both precedence and primary expressions under + // the same precedence item are unique, as they will produce enum variants. + let mut current_expressions = HashSet::new(); + + for precedence_expression in &item.precedence_expressions { + let name = &precedence_expression.name; + if analysis.metadata.contains_key(&**name) { + analysis.errors.add(name, &Errors::ExistingItem(name)); + } + + if !all_expressions.insert(name) { + analysis.errors.add(name, &Errors::ExistingExpression(name)); + } + + current_expressions.insert(name); + } + + for primary_expression in &item.primary_expressions { + let expression = &primary_expression.expression; + + if !current_expressions.insert(expression) { + analysis + .errors + .add(expression, &Errors::ExistingExpression(expression)); + } + } + } +} + +fn get_item_name(item: &Item) -> &Spanned { + match item { + Item::Struct { item } => &item.name, + Item::Enum { item } => &item.name, + Item::Repeated { item } => &item.name, + Item::Separated { item } => &item.name, + Item::Precedence { item } => &item.name, + Item::Trivia { item } => &item.name, + Item::Keyword { item } => &item.name, + Item::Token { item } => &item.name, + Item::Fragment { item } => &item.name, + } +} + +fn calculate_defined_in(analysis: &mut Analysis, item: &Item) -> VersionSet { + return match item { + Item::Struct { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Enum { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Repeated { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Separated { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Precedence { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Trivia { item: _ } => { + VersionSet::from_range(calculate_enablement(analysis, &None, &None)) + } + Item::Keyword { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + Item::Token { item } => { + VersionSet::from_ranges(item.definitions.iter().map(|definition| { + calculate_enablement(analysis, &definition.enabled_in, &definition.disabled_in) + })) + } + Item::Fragment { item } => VersionSet::from_range(calculate_enablement( + analysis, + &item.enabled_in, + &item.disabled_in, + )), + }; +} + +fn calculate_enablement( + analysis: &mut Analysis, + enabled_in: &Option>, + disabled_in: &Option>, +) -> VersionRange { + let enabled_in = match enabled_in { + Some(enabled_in) => enabled_in, + None => &analysis.language.versions[0], + }; + + return match disabled_in { + Some(disabled_in) => VersionRange::between(enabled_in, disabled_in), + None => VersionRange::starting_from(enabled_in), + }; +} + +#[derive(thiserror::Error, Debug)] +enum Errors<'err> { + #[error("An item with the name '{0}' already exists.")] + ExistingItem(&'err Identifier), + #[error("A variant with the name '{0}' already exists.")] + ExistingVariant(&'err Identifier), + #[error("An expression with the name '{0}' already exists.")] + ExistingExpression(&'err Identifier), +} diff --git a/crates/codegen/language/definition/src/compiler/analysis/mod.rs b/crates/codegen/language/definition/src/compiler/analysis/mod.rs new file mode 100644 index 0000000000..b6ba19f464 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/analysis/mod.rs @@ -0,0 +1,82 @@ +mod definitions; +mod reachability; +mod references; + +use crate::{ + compiler::{ + analysis::{ + definitions::analyze_definitions, reachability::analyze_reachability, + references::analyze_references, + }, + versions::VersionSet, + }, + internals::{ErrorsCollection, ParseOutput, Spanned}, + spanned::{Item, ItemKind, Language}, + Identifier, +}; +use indexmap::IndexMap; +use proc_macro2::Span; +use std::rc::Rc; + +pub struct Analysis { + pub errors: ErrorsCollection, + pub language: Rc, + pub metadata: IndexMap, +} + +pub struct ItemMetadata { + pub name: Spanned, + pub kind: ItemKind, + + pub defined_in: VersionSet, + pub used_in: VersionSet, + + pub referenced_from: Vec, + pub referenced_items: Vec, +} + +impl Analysis { + pub fn analyze(parse_output: ParseOutput) -> Self { + let ParseOutput { language, errors } = parse_output; + + let mut analysis = Self { + errors, + language: language.into(), + metadata: IndexMap::new(), + }; + + // Early return if there are already errors, to prevent producing noise from later analysis: + if analysis.errors.has_errors() { + return analysis; + } + + analyze_definitions(&mut analysis); + + // Early return if there are already errors, to prevent producing noise from later analysis: + if analysis.errors.has_errors() { + return analysis; + } + + analyze_references(&mut analysis); + + // Early return if there are already errors, to prevent producing noise from later analysis: + if analysis.errors.has_errors() { + return analysis; + } + + analyze_reachability(&mut analysis); + + return analysis; + } +} + +impl Language { + fn items(&self) -> impl Iterator { + return self + .sections + .iter() + .flat_map(|section| §ion.topics) + .flat_map(|topic| &topic.items) + .map(|item| &***item); + } +} diff --git a/crates/codegen/language/definition/src/compiler/analysis/reachability.rs b/crates/codegen/language/definition/src/compiler/analysis/reachability.rs new file mode 100644 index 0000000000..d021b7fdf0 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/analysis/reachability.rs @@ -0,0 +1,78 @@ +use crate::{ + compiler::{analysis::Analysis, versions::VersionSet}, + spanned::TriviaParser, + Identifier, +}; +use std::collections::HashSet; + +pub fn analyze_reachability(analysis: &mut Analysis) { + check_unreachabable_items(analysis); + check_unused_versions(analysis); +} + +fn check_unused_versions(analysis: &mut Analysis) { + for (name, metadata) in &analysis.metadata { + if name == &*analysis.language.root_item { + continue; + } + + let unused_in = metadata.defined_in.difference(&metadata.used_in); + + if !unused_in.is_empty() { + analysis + .errors + .add(&metadata.name, &Errors::UnusedVersion(name, &unused_in)); + } + } +} + +fn check_unreachabable_items(analysis: &mut Analysis) { + let language = analysis.language.clone(); + + let mut queue = vec![&*language.root_item]; + + collect_trivia(&language.leading_trivia, &mut queue); + collect_trivia(&language.trailing_trivia, &mut queue); + + let mut visited = queue.iter().cloned().collect::>(); + + while let Some(name) = queue.pop() { + for referenced_item in &analysis.metadata[name].referenced_items { + if visited.insert(referenced_item) { + queue.push(referenced_item); + } + } + } + + for metadata in analysis.metadata.values() { + if !metadata.defined_in.is_empty() && !visited.contains(&*metadata.name) { + analysis + .errors + .add(&metadata.name, &Errors::Unreachable(&*metadata.name)); + } + } +} + +fn collect_trivia<'l>(parser: &'l TriviaParser, acc: &mut Vec<&'l Identifier>) { + match parser { + TriviaParser::Sequence { parsers } | TriviaParser::Choice { parsers } => { + for parser in parsers { + collect_trivia(parser, acc); + } + } + TriviaParser::ZeroOrMore { parser } | TriviaParser::Optional { parser } => { + collect_trivia(parser, acc); + } + TriviaParser::Trivia { trivia } => { + acc.push(trivia); + } + }; +} + +#[derive(thiserror::Error, Debug)] +enum Errors<'err> { + #[error("Item '{0}' is not used in versions: {1}")] + UnusedVersion(&'err Identifier, &'err VersionSet), + #[error("Item '{0}' is not reachable from grammar root.")] + Unreachable(&'err Identifier), +} diff --git a/crates/codegen/language/definition/src/compiler/analysis/references.rs b/crates/codegen/language/definition/src/compiler/analysis/references.rs new file mode 100644 index 0000000000..e67f871181 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/analysis/references.rs @@ -0,0 +1,570 @@ +use crate::{ + compiler::{ + analysis::Analysis, + versions::{VersionRange, VersionSet}, + }, + internals::Spanned, + spanned::{ + EnumItem, EnumVariant, Field, FieldKind, FragmentItem, Item, ItemKind, KeywordItem, + PrecedenceExpression, PrecedenceItem, PrecedenceOperator, PrimaryExpression, RepeatedItem, + Scanner, SeparatedItem, StructItem, TokenDefinition, TokenItem, TriviaItem, TriviaParser, + }, + Identifier, +}; +use indexmap::IndexMap; +use itertools::Itertools; +use semver::Version; +use std::fmt::Debug; +use strum::IntoEnumIterator; +use strum_macros::Display; + +pub fn analyze_references(analysis: &mut Analysis) { + let language = analysis.language.clone(); + let enablement = VersionRange::starting_from(&language.versions[0]); + + check_reference( + analysis, + None, + &language.root_item, + &enablement, + ReferenceFilter::NonTerminals, + ); + + check_trivia_parser(analysis, &language.leading_trivia, &enablement); + check_trivia_parser(analysis, &language.trailing_trivia, &enablement); + + for item in language.items() { + check_item(analysis, item, &enablement); + } +} + +fn check_item(analysis: &mut Analysis, item: &Item, enablement: &VersionRange) { + match item { + Item::Struct { item } => { + check_struct(analysis, item, &enablement); + } + Item::Enum { item } => { + check_enum(analysis, item, &enablement); + } + Item::Repeated { item } => { + check_repeated(analysis, item, &enablement); + } + Item::Separated { item } => { + check_separated(analysis, item, &enablement); + } + Item::Precedence { item } => { + check_precedence(analysis, item, &enablement); + } + Item::Trivia { item } => { + check_trivia(analysis, item, &enablement); + } + Item::Keyword { item } => { + check_keyword(analysis, item, &enablement); + } + Item::Token { item } => { + check_token(analysis, item, &enablement); + } + Item::Fragment { item } => { + check_fragment(analysis, item, &enablement); + } + } +} + +fn check_struct(analysis: &mut Analysis, item: &StructItem, enablement: &VersionRange) { + let StructItem { + name, + enabled_in, + disabled_in, + error_recovery: _, + fields, + } = item; + + let enablement = update_enablement(analysis, enablement, &enabled_in, &disabled_in); + + check_fields(analysis, Some(name), fields, &enablement); +} + +fn check_enum(analysis: &mut Analysis, item: &EnumItem, enablement: &VersionRange) { + let EnumItem { + name, + enabled_in, + disabled_in, + default_variant, + variants, + } = item; + + if !variants + .iter() + .any(|variant| variant.name == *default_variant) + { + analysis + .errors + .add(default_variant, &Errors::VariantDoesNotExist); + } + + let enablement = update_enablement(analysis, enablement, &enabled_in, &disabled_in); + + for variant in variants { + let EnumVariant { + name: _, + enabled_in, + disabled_in, + error_recovery: _, + fields, + } = &**variant; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_fields(analysis, Some(name), &fields, &enablement); + } +} + +fn check_repeated(analysis: &mut Analysis, item: &RepeatedItem, enablement: &VersionRange) { + let RepeatedItem { + name, + repeated, + allow_empty: _, + enabled_in, + disabled_in, + } = item; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_reference( + analysis, + Some(name), + repeated, + &enablement, + ReferenceFilter::Nodes, + ); +} + +fn check_separated(analysis: &mut Analysis, item: &SeparatedItem, enablement: &VersionRange) { + let SeparatedItem { + name, + separated, + separator, + allow_empty: _, + enabled_in, + disabled_in, + } = item; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_reference( + analysis, + Some(name), + separated, + &enablement, + ReferenceFilter::Nodes, + ); + check_reference( + analysis, + Some(name), + separator, + &enablement, + ReferenceFilter::Tokens, + ); +} + +fn check_precedence(analysis: &mut Analysis, item: &PrecedenceItem, enablement: &VersionRange) { + let PrecedenceItem { + name, + enabled_in, + disabled_in, + default_primary_expression, + precedence_expressions, + primary_expressions, + } = item; + + if !primary_expressions + .iter() + .any(|primary_expression| primary_expression.expression == *default_primary_expression) + { + analysis.errors.add( + default_primary_expression, + &Errors::PrimaryExpressionDoesNotExist, + ); + } + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + for precedence_expression in precedence_expressions { + let PrecedenceExpression { name: _, operators } = &**precedence_expression; + + for operator in operators { + let PrecedenceOperator { + model: _, + enabled_in, + disabled_in, + error_recovery: _, + fields, + } = &**operator; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_fields(analysis, Some(name), &fields, &enablement); + } + } + + for primary_expression in primary_expressions { + let PrimaryExpression { + expression, + enabled_in, + disabled_in, + } = &**primary_expression; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_reference( + analysis, + Some(name), + &expression, + &enablement, + ReferenceFilter::Nodes, + ); + } +} + +fn check_fields( + analysis: &mut Analysis, + source: Option<&Identifier>, + fields: &IndexMap, Spanned>, + enablement: &VersionRange, +) { + for field in fields.values() { + match &**field { + Field::Required { kind } => { + check_field_kind(analysis, source, &kind, &enablement); + } + Field::Optional { + kind, + enabled_in, + disabled_in, + } => { + let enablement = + update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_field_kind(analysis, source, &kind, &enablement); + } + }; + } +} + +fn check_field_kind( + analysis: &mut Analysis, + source: Option<&Identifier>, + kind: &FieldKind, + enablement: &VersionRange, +) { + match kind { + FieldKind::NonTerminal { item } => { + check_reference( + analysis, + source, + item, + &enablement, + ReferenceFilter::NonTerminals, + ); + } + FieldKind::Terminal { items } => { + for item in items { + check_reference( + analysis, + source, + item, + &enablement, + ReferenceFilter::Terminals, + ); + } + } + }; +} + +fn check_trivia_parser(analysis: &mut Analysis, parser: &TriviaParser, enablement: &VersionRange) { + match parser { + TriviaParser::Sequence { parsers } | TriviaParser::Choice { parsers } => { + for parser in parsers { + check_trivia_parser(analysis, parser, &enablement); + } + } + TriviaParser::ZeroOrMore { parser } | TriviaParser::Optional { parser } => { + check_trivia_parser(analysis, parser, &enablement); + } + TriviaParser::Trivia { trivia } => { + check_reference(analysis, None, trivia, &enablement, ReferenceFilter::Trivia); + } + }; +} + +fn check_trivia(analysis: &mut Analysis, item: &TriviaItem, enablement: &VersionRange) { + let TriviaItem { name, scanner } = item; + + check_scanner(analysis, Some(name), &scanner, &enablement); +} + +fn check_keyword(analysis: &mut Analysis, item: &KeywordItem, enablement: &VersionRange) { + let KeywordItem { + name, + identifier, + enabled_in, + disabled_in, + reserved_in, + unreserved_in, + value: _, + } = item; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_reference( + analysis, + Some(name), + identifier, + &enablement, + ReferenceFilter::Tokens, + ); + + check_version_pair(analysis, reserved_in, unreserved_in); +} + +fn check_token(analysis: &mut Analysis, item: &TokenItem, enablement: &VersionRange) { + let TokenItem { name, definitions } = item; + + for definition in definitions { + let TokenDefinition { + enabled_in, + disabled_in, + scanner, + } = &**definition; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_scanner(analysis, Some(name), &scanner, &enablement); + } +} + +fn check_fragment(analysis: &mut Analysis, item: &FragmentItem, enablement: &VersionRange) { + let FragmentItem { + name, + enabled_in, + disabled_in, + scanner, + } = item; + + let enablement = update_enablement(analysis, &enablement, &enabled_in, &disabled_in); + + check_scanner(analysis, Some(name), scanner, &enablement); +} + +fn check_scanner( + analysis: &mut Analysis, + source: Option<&Identifier>, + scanner: &Scanner, + enablement: &VersionRange, +) { + match scanner { + Scanner::Sequence { scanners } | Scanner::Choice { scanners } => { + for scanner in scanners { + check_scanner(analysis, source, scanner, &enablement); + } + } + Scanner::Optional { scanner } + | Scanner::ZeroOrMore { scanner } + | Scanner::OneOrMore { scanner } => { + check_scanner(analysis, source, scanner, &enablement); + } + Scanner::Not { chars: _ } + | Scanner::Range { + inclusive_start: _, + inclusive_end: _, + } + | Scanner::Atom { atom: _ } => { + // Nothing to check for now. + } + Scanner::TrailingContext { + scanner, + not_followed_by, + } => { + check_scanner(analysis, source, scanner, &enablement); + check_scanner(analysis, source, not_followed_by, &enablement); + } + Scanner::Fragment { reference } => { + check_reference( + analysis, + source, + reference, + enablement, + ReferenceFilter::Fragments, + ); + } + }; +} + +fn update_enablement( + analysis: &mut Analysis, + existing: &VersionRange, + enabled_in: &Option>, + disabled_in: &Option>, +) -> VersionRange { + check_version_pair(analysis, enabled_in, disabled_in); + + let enabled_in = match enabled_in { + None => &existing.inclusive_start, + Some(enabled_in) => { + if **enabled_in <= existing.inclusive_start { + analysis.errors.add( + enabled_in, + &Errors::EnabledTooEarly(&existing.inclusive_start), + ); + } + &**enabled_in + } + }; + + let disabled_in = match disabled_in { + None => &existing.exclusive_end, + Some(disabled_in) => { + if **disabled_in >= existing.exclusive_end { + analysis.errors.add( + disabled_in, + &Errors::DisabledTooLate(&existing.exclusive_end), + ); + } + &**disabled_in + } + }; + + return VersionRange::between(enabled_in, disabled_in); +} + +fn check_version_pair( + analysis: &mut Analysis, + first: &Option>, + second: &Option>, +) { + for version in [first, second].into_iter().flatten() { + if !analysis.language.versions.contains(version) { + analysis + .errors + .add(version, &Errors::VersionNotFound(version)); + } + } + + if let (Some(first), Some(second)) = (first, second) { + if first >= second { + analysis + .errors + .add(first, &Errors::UnorderedVersionPair(first, second)); + } + } +} + +fn check_reference( + analysis: &mut Analysis, + source: Option<&Identifier>, + reference: &Spanned, + enablement: &VersionRange, + filter: ReferenceFilter, +) { + let target = match analysis.metadata.get_mut(&**reference) { + Some(target) => target, + None => { + analysis + .errors + .add(reference, &Errors::UnknownReference(reference)); + return; + } + }; + + let defined_in = VersionSet::from_range(enablement.to_owned()); + let not_defined_in = defined_in.difference(&target.defined_in); + + if !not_defined_in.is_empty() { + analysis.errors.add( + reference, + &Errors::InvalidReferenceVersion(reference, ¬_defined_in), + ); + } + + if !filter.apply(&target.kind) { + let expected = &ItemKind::iter() + .filter(|kind| filter.apply(kind)) + .collect_vec()[..]; + + analysis.errors.add( + reference, + &Errors::InvalidReferenceFilter(reference, &target.kind, expected), + ); + } + + target.used_in.add(enablement); + + target.referenced_from.push(reference.span()); + + if let Some(source) = source { + analysis.metadata[source] + .referenced_items + .push((**reference).to_owned()); + } +} + +#[derive(Debug, Display, PartialEq, Eq)] +enum ReferenceFilter { + Nodes, + + NonTerminals, + Terminals, + + Trivia, + Tokens, + + Fragments, +} + +impl ReferenceFilter { + fn apply(&self, target_kind: &ItemKind) -> bool { + match target_kind { + ItemKind::Struct + | ItemKind::Enum + | ItemKind::Repeated + | ItemKind::Separated + | ItemKind::Precedence => { + return matches!(self, Self::Nodes | Self::NonTerminals); + } + ItemKind::Trivia => { + return matches!(self, Self::Nodes | Self::Trivia); + } + ItemKind::Keyword => { + return matches!(self, Self::Nodes | Self::Terminals); + } + ItemKind::Token => { + return matches!(self, Self::Nodes | Self::Terminals | Self::Tokens); + } + ItemKind::Fragment => { + return matches!(self, Self::Fragments); + } + }; + } +} + +#[derive(thiserror::Error, Debug)] +enum Errors<'err> { + #[error("Variant does not exist.")] + VariantDoesNotExist, + #[error("Primary expression does not exist.")] + PrimaryExpressionDoesNotExist, + #[error("Version '{0}' does not exist in the language definition.")] + VersionNotFound(&'err Version), + #[error("Version '{0}' must be less than corresponding version '{1}'.")] + UnorderedVersionPair(&'err Version, &'err Version), + #[error("Parent scope is enabled in '{0}'.")] + EnabledTooEarly(&'err Version), + #[error("Parent scope is disabled in '{0}'.")] + DisabledTooLate(&'err Version), + #[error("Reference to unknown item '{0}'.")] + UnknownReference(&'err Identifier), + #[error("Reference '{0}' is not defined in versions '{1}'.")] + InvalidReferenceVersion(&'err Identifier, &'err VersionSet), + #[error("Reference '{0}' of kind '{1}' is not valid. Expected: {2:?}.")] + InvalidReferenceFilter(&'err Identifier, &'err ItemKind, &'err [ItemKind]), +} diff --git a/crates/codegen/language/definition/src/compiler/emitter.rs b/crates/codegen/language/definition/src/compiler/emitter.rs new file mode 100644 index 0000000000..da16579ef0 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/emitter.rs @@ -0,0 +1,71 @@ +use crate::{compiler::analysis::Analysis, internals::WriteOutputTokens}; +use inflector::Inflector; +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; + +pub struct LanguageEmitter; + +impl LanguageEmitter { + pub fn emit(analysis: &Analysis) -> TokenStream { + let language_name = analysis.language.name.to_string(); + let container_mod = format_ident!("{}", language_name.to_snake_case()); + + let definition_struct = format_ident!("{}Definition", language_name); + let definition = analysis.language.write_output_tokens(); + + let errors = analysis.errors.to_compile_errors(); + + let intellisense_types = Self::emit_intellisense_types(analysis); + + return quote! { + mod #container_mod { + // Main entry-point to create language definitions: + pub struct #definition_struct; + + impl #definition_struct { + pub fn create() -> codegen_language_definition::Language { + return #definition; + } + } + + // Any validation errors, as `std::compile_error!()` macro invocations: + #errors + + // `intellisense_types` are hidden from the user. Only used by Rust Analyzer: + mod intellisense { + #intellisense_types + } + } + }; + } + + fn emit_intellisense_types(analysis: &Analysis) -> TokenStream { + let mut result = TokenStream::new(); + + if analysis.errors.has_errors() { + // No need to generate intellisense types if there are errors. + // They need to be fixed first, so that we don't get noise in the IDE. + return result; + } + + for metadata in analysis.metadata.values() { + let name = &metadata.name; + let type_name = format_ident!("{name}", span = name.span()); + + result.extend(quote! { + struct #type_name ; + }); + + for (i, reference) in metadata.referenced_from.iter().enumerate() { + let reference_name = format_ident!("{name}Reference{i}"); + let reference_value = format_ident!("{name}", span = *reference); + + result.extend(quote! { + type #reference_name = #reference_value; + }); + } + } + + return result; + } +} diff --git a/crates/codegen/language/definition/src/compiler/mod.rs b/crates/codegen/language/definition/src/compiler/mod.rs new file mode 100644 index 0000000000..f9cb2728ba --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/mod.rs @@ -0,0 +1,28 @@ +mod analysis; +mod emitter; +mod versions; + +use crate::{ + compiler::{analysis::Analysis, emitter::LanguageEmitter}, + internals::ParseAdapter, +}; +use proc_macro2::TokenStream; + +pub struct LanguageCompiler; + +impl LanguageCompiler { + pub fn compile(input: TokenStream) -> TokenStream { + let parse_output = match ParseAdapter::parse(input) { + // Parsed all input successfully, with potentionally recoverable errors: + Ok(parse_output) => parse_output, + // Non-recoverable parsing error. Let's quit early: + Err(error) => return error.to_compile_error(), + }; + + let analysis = Analysis::analyze(parse_output); + + let output = LanguageEmitter::emit(&analysis); + + return output; + } +} diff --git a/crates/codegen/language/definition/src/compiler/versions.rs b/crates/codegen/language/definition/src/compiler/versions.rs new file mode 100644 index 0000000000..d73ec00ea6 --- /dev/null +++ b/crates/codegen/language/definition/src/compiler/versions.rs @@ -0,0 +1,369 @@ +use itertools::Itertools; +use semver::Version; +use std::{ + cmp::{max, min}, + fmt::Display, + iter::once, +}; + +const MAX_VERSION: Version = Version::new(u64::MAX, u64::MAX, u64::MAX); + +#[derive(Clone, Debug)] +pub struct VersionRange { + pub inclusive_start: Version, + pub exclusive_end: Version, +} + +impl VersionRange { + pub fn starting_from(inclusive_start: &Version) -> Self { + return Self { + inclusive_start: inclusive_start.to_owned(), + exclusive_end: MAX_VERSION, + }; + } + + pub fn between(inclusive_start: &Version, exclusive_end: &Version) -> Self { + if inclusive_start <= exclusive_end { + return Self { + inclusive_start: inclusive_start.to_owned(), + exclusive_end: exclusive_end.to_owned(), + }; + } else { + // This is an invalid state, which we still produce errors for. + // However, for validation to continue, let's correct the order: + return Self { + inclusive_start: exclusive_end.to_owned(), + exclusive_end: inclusive_start.to_owned(), + }; + } + } + + pub fn is_empty(&self) -> bool { + return self.inclusive_start == self.exclusive_end; + } +} + +impl Display for VersionRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.inclusive_start == MAX_VERSION { + "MAX".fmt(f)?; + } else { + self.inclusive_start.fmt(f)?; + } + + "..".fmt(f)?; + + if self.exclusive_end == MAX_VERSION { + "MAX".fmt(f)?; + } else { + self.exclusive_end.fmt(f)?; + } + + return Ok(()); + } +} + +#[derive(Debug)] +pub struct VersionSet { + ranges: Vec, +} + +impl VersionSet { + pub fn empty() -> Self { + return Self { ranges: vec![] }; + } + + pub fn from_range(range: VersionRange) -> Self { + let mut instance = Self::empty(); + + instance.add(&range); + + return instance; + } + + pub fn from_ranges(ranges: impl IntoIterator) -> Self { + let mut instance = Self::empty(); + + for range in ranges.into_iter() { + instance.add(&range); + } + + return instance; + } + + pub fn is_empty(&self) -> bool { + return self.ranges.is_empty(); + } + + pub fn add(&mut self, range: &VersionRange) { + if range.is_empty() { + return; + } + + let mut result = vec![]; + + // Iterate on both (existing and new) in-order, combining them if they overlap: + let mut input_iter = self + .ranges + .iter() + .chain(once(range)) + .sorted_by_key(|range| &range.inclusive_start); + + let mut current = input_iter.next().unwrap().to_owned(); + + for next in input_iter { + if current.exclusive_end < next.inclusive_start { + // current fully exists before next: + result.push(current); + current = next.to_owned(); + } else { + // current and next overlap, combine them: + current = VersionRange::between( + min(¤t.inclusive_start, &next.inclusive_start), + max(¤t.exclusive_end, &next.exclusive_end), + ); + } + } + + result.push(current); + self.ranges = result; + } + + pub fn difference(&self, other: &Self) -> Self { + let mut result = vec![]; + + let mut first_iter = self.ranges.iter().cloned().peekable(); + let mut second_iter = other.ranges.iter().peekable(); + + loop { + let (Some(first), Some(second)) = (first_iter.peek_mut(), second_iter.peek()) else { + break; + }; + + if first.exclusive_end <= second.inclusive_start { + // first fully exists before second: take it, and advance first: + result.push(first_iter.next().unwrap()); + continue; + } + + if second.exclusive_end <= first.inclusive_start { + // second fully exists before first: advance second: + second_iter.next(); + continue; + } + + // first and second overlap: + + if first.inclusive_start < second.inclusive_start { + // take part of first that exists before second: + result.push(VersionRange::between( + &first.inclusive_start, + &second.inclusive_start, + )); + } + + if first.exclusive_end <= second.exclusive_end { + // first ends before second: advance first, as it's been fully processed: + first_iter.next(); + continue; + } + + // keep part of first that exists after second: + first.inclusive_start = second.exclusive_end.to_owned(); + + // advance second, as it's been fully processed: + second_iter.next(); + } + + // Take anything remaining in first: + result.extend(first_iter); + + return Self { ranges: result }; + } +} + +impl Display for VersionSet { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.is_empty() { + return "{EMPTY}".fmt(f); + } + + for (i, range) in self.ranges.iter().enumerate() { + if i > 0 { + " | ".fmt(f)?; + } + + range.fmt(f)?; + } + + return Ok(()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_ranges() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(3, 0, 0)), + VersionRange::between(&Version::new(4, 0, 0), &Version::new(5, 0, 0)), + ]); + + assert_eq!(set.to_string(), "1.0.0..2.0.0 | 4.0.0..5.0.0"); + } + + #[test] + fn connected_ranges_in_order() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(2, 0, 0), &Version::new(3, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + ]); + + assert_eq!(set.to_string(), "1.0.0..4.0.0"); + } + + #[test] + fn connected_ranges_out_of_order() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + VersionRange::between(&Version::new(2, 0, 0), &Version::new(3, 0, 0)), + ]); + + assert_eq!(set.to_string(), "1.0.0..4.0.0"); + } + + #[test] + fn disconnected_ranges_in_order() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + VersionRange::between(&Version::new(5, 0, 0), &Version::new(6, 0, 0)), + ]); + + assert_eq!( + set.to_string(), + "1.0.0..2.0.0 | 3.0.0..4.0.0 | 5.0.0..6.0.0" + ); + } + + #[test] + fn disconnected_ranges_out_of_order() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(5, 0, 0), &Version::new(6, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + ]); + + assert_eq!( + set.to_string(), + "1.0.0..2.0.0 | 3.0.0..4.0.0 | 5.0.0..6.0.0" + ); + } + + #[test] + fn overlap_with_multiple() { + let set = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + VersionRange::between(&Version::new(5, 0, 0), &Version::new(6, 0, 0)), + VersionRange::between(&Version::new(1, 0, 0), &Version::new(5, 0, 0)), + ]); + + assert_eq!(set.to_string(), "1.0.0..6.0.0"); + } + + #[test] + fn difference_between_same_sets_is_empty() { + let first = VersionSet::from_range(VersionRange::between( + &Version::new(1, 0, 0), + &Version::new(2, 0, 0), + )); + + let second = VersionSet::from_range(VersionRange::between( + &Version::new(1, 0, 0), + &Version::new(2, 0, 0), + )); + + assert!(first.difference(&second).is_empty()); + } + + #[test] + fn difference_between_connected_sets() { + let first = VersionSet::from_range(VersionRange::between( + &Version::new(1, 0, 0), + &Version::new(5, 0, 0), + )); + + let second = VersionSet::from_range(VersionRange::between( + &Version::new(3, 0, 0), + &Version::new(8, 0, 0), + )); + + assert_eq!(first.difference(&second).to_string(), "1.0.0..3.0.0"); + + assert_eq!(second.difference(&first).to_string(), "5.0.0..8.0.0"); + } + + #[test] + fn difference_between_disconnected_sets() { + let first = VersionSet::from_range(VersionRange::between( + &Version::new(1, 0, 0), + &Version::new(4, 0, 0), + )); + + let second = VersionSet::from_range(VersionRange::between( + &Version::new(6, 0, 0), + &Version::new(10, 0, 0), + )); + + assert_eq!(first.difference(&second).to_string(), "1.0.0..4.0.0"); + + assert_eq!(second.difference(&first).to_string(), "6.0.0..10.0.0"); + } + + #[test] + fn difference_between_contained_sets() { + let first = VersionSet::from_range(VersionRange::between( + &Version::new(1, 0, 0), + &Version::new(8, 0, 0), + )); + + let second = VersionSet::from_range(VersionRange::between( + &Version::new(3, 0, 0), + &Version::new(5, 0, 0), + )); + + assert_eq!( + first.difference(&second).to_string(), + "1.0.0..3.0.0 | 5.0.0..8.0.0" + ); + + assert!(second.difference(&first).is_empty()); + } + + #[test] + fn difference_between_multiple_contained_sets() { + let first = VersionSet::from_ranges([ + VersionRange::between(&Version::new(1, 0, 0), &Version::new(2, 0, 0)), + VersionRange::between(&Version::new(3, 0, 0), &Version::new(4, 0, 0)), + VersionRange::between(&Version::new(5, 0, 0), &Version::new(6, 0, 0)), + ]); + + let second = VersionSet::from_range(VersionRange::between( + &Version::new(0, 0, 0), + &Version::new(7, 0, 0), + )); + + assert!(first.difference(&second).is_empty()); + + assert_eq!( + second.difference(&first).to_string(), + "0.0.0..1.0.0 | 2.0.0..3.0.0 | 4.0.0..5.0.0 | 6.0.0..7.0.0" + ); + } +} diff --git a/crates/codegen/language/definition/src/internals/errors.rs b/crates/codegen/language/definition/src/internals/errors.rs new file mode 100644 index 0000000000..8d8eedcb19 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/errors.rs @@ -0,0 +1,85 @@ +use proc_macro2::{Span, TokenStream}; +use std::fmt::Display; + +pub type Result = std::result::Result; + +/// Our own proxy for [syn::Error] since the latter does not expose the underlying sub-errors. +#[derive(Debug)] +pub struct Error { + message: String, + span: Span, +} + +impl Error { + pub fn fatal(has_span: &impl HasSpan, message: &impl Display) -> Result { + return Err(Error { + message: message.to_string(), + span: has_span.span(), + }); + } + + pub fn from_syn(error: syn::Error) -> Self { + return Self { + message: error.to_string(), + span: error.span(), + }; + } + + pub fn to_syn(&self) -> syn::Error { + return syn::Error::new(self.span, &self.message); + } + + pub fn to_compile_error(&self) -> TokenStream { + return syn::Error::new(self.span, &self.message).to_compile_error(); + } +} + +#[derive(Debug)] +pub struct ErrorsCollection { + errors: Vec, +} + +impl ErrorsCollection { + pub fn new() -> Self { + return Self { errors: vec![] }; + } + + pub fn has_errors(&self) -> bool { + return !self.errors.is_empty(); + } + + pub fn add(&mut self, has_span: &impl HasSpan, message: &impl Display) { + self.errors.push(Error { + message: message.to_string(), + span: has_span.span(), + }); + } + + pub fn push(&mut self, error: Error) { + self.errors.push(error); + } + + pub fn to_compile_errors(&self) -> TokenStream { + return self + .errors + .iter() + .map(|error| error.to_compile_error()) + .collect(); + } +} + +pub trait HasSpan { + fn span(&self) -> Span; +} + +impl HasSpan for crate::internals::Spanned { + fn span(&self) -> Span { + return self.span(); + } +} + +impl HasSpan for T { + fn span(&self) -> Span { + return self.span(); + } +} diff --git a/crates/codegen/language/definition/src/internals/mod.rs b/crates/codegen/language/definition/src/internals/mod.rs new file mode 100644 index 0000000000..91bd33c394 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/mod.rs @@ -0,0 +1,9 @@ +mod errors; +mod parse_input_tokens; +mod spanned; +mod write_output_tokens; + +pub use errors::*; +pub use parse_input_tokens::*; +pub use spanned::*; +pub use write_output_tokens::*; diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/adapter.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/adapter.rs new file mode 100644 index 0000000000..6e7e8ec426 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/parse_input_tokens/adapter.rs @@ -0,0 +1,35 @@ +use crate::{ + internals::{Error, ErrorsCollection, ParseInputTokens, Result}, + spanned::Language, +}; +use proc_macro2::TokenStream; +use syn::parse::{Parse, ParseStream}; + +pub struct ParseAdapter; + +impl ParseAdapter { + pub fn parse(input: TokenStream) -> Result { + return syn::parse2(input).map_err(|error| Error::from_syn(error)); + } +} + +pub struct ParseOutput { + pub language: Language, + pub errors: ErrorsCollection, +} + +/// A wrapper around [syn::parse::Parse] to convert to/from our own error types. +impl Parse for ParseOutput { + fn parse(input: ParseStream) -> syn::Result { + let mut errors = ErrorsCollection::new(); + + match Language::parse_named_value(input, &mut errors) { + Ok(language) => { + return Ok(Self { language, errors }); + } + Err(error) => { + return Err(error.to_syn()); + } + }; + } +} diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/helpers.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/helpers.rs new file mode 100644 index 0000000000..f781556a36 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/parse_input_tokens/helpers.rs @@ -0,0 +1,160 @@ +use crate::internals::{Error, ErrorsCollection, ParseInputTokens, Result, Spanned}; +use indexmap::IndexMap; +use proc_macro2::{extra::DelimSpan, Delimiter, Ident, TokenStream}; +use std::fmt::Debug; +use syn::{braced, bracketed, parenthesized, parse::ParseStream, Token}; + +pub struct ParseHelpers; + +impl ParseHelpers { + pub fn syn(input: ParseStream) -> Result { + return input.parse::().map_err(|error| { + return Error::from_syn(error); + }); + } + + pub fn delimited( + delimiter: Delimiter, + input: ParseStream, + inner_parser: impl FnOnce(DelimSpan, ParseStream) -> Result, + ) -> Result { + return delimited(delimiter, input, inner_parser).map_err(|error| { + return Error::from_syn(error); + }); + } + + pub fn sequence( + input: ParseStream, + errors: &mut ErrorsCollection, + ) -> Result> { + match Self::delimited(Delimiter::Bracket, input, |_, inner_input| { + let mut result = Vec::new(); + + while !inner_input.is_empty() { + result.push(ParseInputTokens::parse_named_value(&inner_input, errors)?); + + if !inner_input.is_empty() { + let comma = Self::syn::(&inner_input)?; + + if inner_input.is_empty() { + errors.add(&comma, &Errors::TrailingComma); + } + } + } + + return Ok(result); + }) { + Ok(value) => { + return Ok(value); + } + Err(error) => { + errors.push(error); + return Ok(vec![]); + } + }; + } + + pub fn map( + input: ParseStream, + errors: &mut ErrorsCollection, + ) -> Result, Spanned>> { + match Self::delimited(Delimiter::Parenthesis, input, |_, inner_input| { + let mut result = IndexMap::new(); + + while !inner_input.is_empty() { + let key = ParseInputTokens::parse_named_value(&inner_input, errors)?; + + Self::syn::(&inner_input)?; + + let value = ParseInputTokens::parse_named_value(&inner_input, errors)?; + + if !inner_input.is_empty() { + let comma = Self::syn::(&inner_input)?; + + if inner_input.is_empty() { + errors.add(&comma, &Errors::TrailingComma); + } + } + + if result.contains_key(&key) { + errors.add(&key, &Errors::DuplicateMapKey); + } else { + result.insert(key, value); + } + } + + return Ok(result); + }) { + Ok(value) => { + return Ok(value); + } + Err(error) => { + errors.push(error); + return Ok(IndexMap::new()); + } + } + } + + pub fn field( + name: &str, + input: ParseStream, + errors: &mut ErrorsCollection, + ) -> Result { + let span = input.span(); + match Self::syn::(&input) { + Ok(key) if key.to_string() == name => key, + _ => { + return Error::fatal(&span, &Errors::ExpectedField(name)); + } + }; + + Self::syn::(&input)?; + + let value = ParseInputTokens::parse_named_value(&input, errors)?; + + if !input.is_empty() { + let comma = Self::syn::(&input)?; + + if input.is_empty() { + errors.add(&comma, &Errors::TrailingComma); + } + } + + return Ok(value); + } +} + +/// A wrapper to convert error types in callsites, since the macros below requires returning [syn::Result]. +fn delimited( + delimiter: Delimiter, + input: ParseStream, + inner_parser: impl FnOnce(DelimSpan, ParseStream) -> Result, +) -> syn::Result { + let inner_input; + let span = match delimiter { + Delimiter::Brace => braced!(inner_input in input).span, + Delimiter::Bracket => bracketed!(inner_input in input).span, + Delimiter::Parenthesis => parenthesized!(inner_input in input).span, + Delimiter::None => { + return Err(syn::Error::new(input.span(), "Expected a delimited block.")); + } + }; + + return inner_parser(span, &inner_input).map_err(|error| { + // consume the rest of 'inner_input' so that we don't end up + // reporting an extra/unnecessary "unexpected token" error: + inner_input.parse::().ok(); + + return error.to_syn(); + }); +} + +#[derive(thiserror::Error, Debug)] +enum Errors<'err> { + #[error("Expected field: '{0}'.")] + ExpectedField(&'err str), + #[error("Duplicate map key.")] + DuplicateMapKey, + #[error("Trailing commas are not allowed.")] + TrailingComma, +} diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/implementations.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/implementations.rs new file mode 100644 index 0000000000..ccc6ba2e33 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/parse_input_tokens/implementations.rs @@ -0,0 +1,190 @@ +use crate::{ + internals::{ + parse_input_tokens::ParseHelpers, Error, ErrorsCollection, ParseInputTokens, Result, + Spanned, + }, + Identifier, +}; +use indexmap::{IndexMap, IndexSet}; +use infra_utils::paths::PathExtensions; +use proc_macro2::Ident; +use semver::Version; +use std::{ + fmt::Debug, + path::{Path, PathBuf}, + rc::Rc, +}; +use syn::{parse::ParseStream, LitBool, LitChar, LitStr}; + +impl ParseInputTokens for bool { + fn parse_value(input: ParseStream, _: &mut ErrorsCollection) -> Result { + let literal = ParseHelpers::syn::(input)?; + + return Ok(literal.value()); + } +} + +impl ParseInputTokens for Box { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let value = T::parse_value(input, errors)?; + + return Ok(value.into()); + } +} + +impl ParseInputTokens for char { + fn parse_value(input: ParseStream, _: &mut ErrorsCollection) -> Result { + let literal = ParseHelpers::syn::(input)?; + + return Ok(literal.value()); + } +} + +impl ParseInputTokens for Identifier { + fn parse_value(input: ParseStream, _: &mut ErrorsCollection) -> Result { + let value = ParseHelpers::syn::(input)?.to_string(); + + return Ok(value.into()); + } +} + +impl ParseInputTokens + for IndexMap, Spanned> +{ + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + return ParseHelpers::map(input, errors); + } +} + +impl ParseInputTokens for IndexSet> { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let sequence: Vec> = ParseHelpers::sequence(input, errors)?; + + let mut set = Self::new(); + + for item in sequence { + if set.contains(&item) { + errors.add(&item, &Errors::DuplicateEntry); + } + + set.insert(item); + } + + return Ok(set); + } +} + +impl ParseInputTokens for Option { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + if input.is_empty() { + return Ok(None); + } else { + return Ok(Some(T::parse_value(input, errors)?)); + } + } + + fn parse_field(name: &str, input: ParseStream, errors: &mut ErrorsCollection) -> Result { + match ParseHelpers::syn::(&input.fork()) { + Ok(key) if key.to_string() == name => { + return Ok(Some(ParseHelpers::field(name, input, errors)?)); + } + _ => { + return Ok(None); + } + }; + } +} + +impl ParseInputTokens for PathBuf { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let literal = &ParseHelpers::syn::(input)?; + let value = literal.value(); + let full_path = Path::repo_path(value); + + if !full_path.exists() { + errors.add(literal, &Errors::PathNotFound(&full_path)); + } + + return Ok(full_path); + } +} + +impl ParseInputTokens for Rc { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let value = T::parse_value(input, errors)?; + + return Ok(value.into()); + } +} + +impl ParseInputTokens for Spanned { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let span = input.span(); + let value = T::parse_value(input, errors)?; + + return Ok(Self::new(span, value)); + } + + fn parse_named_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let span = input.span(); + let value = ParseInputTokens::parse_named_value(input, errors)?; + + return Ok(Self::new(span, value)); + } +} + +impl ParseInputTokens for String { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let literal = ParseHelpers::syn::(input)?; + let value = literal.value(); + + if value.is_empty() { + errors.add(&literal, &Errors::EmptyString); + } + + return Ok(value); + } +} + +impl ParseInputTokens for usize { + fn parse_value(input: ParseStream, _: &mut ErrorsCollection) -> Result { + let literal = ParseHelpers::syn::(input)?; + + return literal + .base10_parse::() + .map_err(|error| Error::from_syn(error)); + } +} + +impl ParseInputTokens for Vec { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + return ParseHelpers::sequence(input, errors); + } +} + +impl ParseInputTokens for Version { + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + let literal = ParseHelpers::syn::(input)?; + + match Self::parse(&literal.value()) { + Ok(version) => { + return Ok(version); + } + Err(error) => { + errors.add(&literal, &error); + + return Ok(Version::new(0, 0, 0)); + } + }; + } +} + +#[derive(thiserror::Error, Debug)] +enum Errors<'err> { + #[error("Set entries must be unique.")] + DuplicateEntry, + #[error("Expected a non-empty string.")] + EmptyString, + #[error("Path not found: {0}")] + PathNotFound(&'err PathBuf), +} diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/mod.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/mod.rs new file mode 100644 index 0000000000..a16bab8ce6 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/parse_input_tokens/mod.rs @@ -0,0 +1,26 @@ +mod adapter; +mod helpers; +mod implementations; + +pub use adapter::*; +pub use helpers::*; + +use crate::internals::{ErrorsCollection, Result}; +use syn::parse::ParseStream; + +pub trait ParseInputTokens: Sized { + /// Main parser entrypoint, and should be implemented by all types. + fn parse_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result; + + /// Allows named types (structs) to parse the type name before its body. + /// By default, it will parse the value directly if not overriden. + fn parse_named_value(input: ParseStream, errors: &mut ErrorsCollection) -> Result { + return Self::parse_value(input, errors); + } + + /// Allows implementations (like 'Option') to modify the parsing logic, + /// by checking if the field exists before attempting to parse it. + fn parse_field(name: &str, input: ParseStream, errors: &mut ErrorsCollection) -> Result { + return ParseHelpers::field(name, input, errors); + } +} diff --git a/crates/codegen/language/definition/src/internals/spanned/mod.rs b/crates/codegen/language/definition/src/internals/spanned/mod.rs new file mode 100644 index 0000000000..9d71ba9aca --- /dev/null +++ b/crates/codegen/language/definition/src/internals/spanned/mod.rs @@ -0,0 +1,71 @@ +use proc_macro2::Span; +use serde::Serialize; +use std::cmp::Ordering; + +#[derive(Clone, Debug)] +pub struct Spanned { + span: Span, + value: T, +} + +impl Spanned { + pub fn new(span: Span, value: T) -> Self { + return Self { span, value }; + } + + pub fn span(&self) -> Span { + return self.span; + } +} + +impl std::ops::Deref for Spanned { + type Target = T; + + fn deref(&self) -> &Self::Target { + return &self.value; + } +} + +impl std::fmt::Display for Spanned +where + T: std::fmt::Display, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return self.value.fmt(f); + } +} + +impl Eq for Spanned {} + +impl std::hash::Hash for Spanned { + fn hash(&self, state: &mut H) { + return self.value.hash(state); + } +} + +impl Ord for Spanned { + fn cmp(&self, other: &Self) -> Ordering { + return self.value.cmp(&other.value); + } +} + +impl PartialEq for Spanned { + fn eq(&self, other: &Self) -> bool { + return self.value == other.value; + } +} + +impl PartialOrd for Spanned { + fn partial_cmp(&self, other: &Self) -> Option { + return self.value.partial_cmp(&other.value); + } +} + +impl Serialize for Spanned { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + return self.value.serialize(serializer); + } +} diff --git a/crates/codegen/language/definition/src/internals/write_output_tokens/implementations.rs b/crates/codegen/language/definition/src/internals/write_output_tokens/implementations.rs new file mode 100644 index 0000000000..1b80f41992 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/write_output_tokens/implementations.rs @@ -0,0 +1,158 @@ +use crate::{ + internals::{Spanned, WriteOutputTokens}, + Identifier, +}; +use indexmap::{IndexMap, IndexSet}; +use proc_macro2::{Literal, TokenStream}; +use quote::{format_ident, quote}; +use semver::Version; +use std::{ops::Deref, path::PathBuf, rc::Rc}; + +impl WriteOutputTokens for bool { + fn write_output_tokens(&self) -> TokenStream { + let value = format_ident!("{self}"); + + return quote! { + #value + }; + } +} + +impl WriteOutputTokens for Box { + fn write_output_tokens(&self) -> TokenStream { + let value = self.deref().write_output_tokens(); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for char { + fn write_output_tokens(&self) -> TokenStream { + let value = Literal::character(*self); + + return quote! { + #value + }; + } +} + +impl WriteOutputTokens for Identifier { + fn write_output_tokens(&self) -> TokenStream { + let value = Literal::string(&self); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for IndexMap { + fn write_output_tokens(&self) -> TokenStream { + let keys = self.keys().map(|key| key.write_output_tokens()); + let values = self.values().map(|value| value.write_output_tokens()); + + return quote! { + [ #( (#keys, #values) ),* ].into() + }; + } +} + +impl WriteOutputTokens for IndexSet { + fn write_output_tokens(&self) -> TokenStream { + let items = self.iter().map(|item| item.write_output_tokens()); + + return quote! { + [ #( #items ),* ].into() + }; + } +} + +impl WriteOutputTokens for Option { + fn write_output_tokens(&self) -> TokenStream { + match self { + Some(value) => { + let value = value.write_output_tokens(); + + return quote! { + Some(#value) + }; + } + None => { + return quote! { + None + }; + } + }; + } +} + +impl WriteOutputTokens for PathBuf { + fn write_output_tokens(&self) -> TokenStream { + let value = Literal::string(self.to_str().unwrap()); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for Rc { + fn write_output_tokens(&self) -> TokenStream { + let value = self.deref().write_output_tokens(); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for Spanned { + fn write_output_tokens(&self) -> TokenStream { + // 'Spanned' is removed from macro output, leaving only the inner value: + return self.deref().write_output_tokens(); + } +} + +impl WriteOutputTokens for String { + fn write_output_tokens(&self) -> TokenStream { + let value = Literal::string(self.as_str()); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for usize { + fn write_output_tokens(&self) -> TokenStream { + let value = Literal::usize_suffixed(*self); + + return quote! { + #value.into() + }; + } +} + +impl WriteOutputTokens for Vec { + fn write_output_tokens(&self) -> TokenStream { + let items = self.iter().map(|item| item.write_output_tokens()); + + return quote! { + [ #( #items ),* ].into() + }; + } +} + +impl WriteOutputTokens for Version { + fn write_output_tokens(&self) -> TokenStream { + let major = Literal::u64_unsuffixed(self.major); + let minor = Literal::u64_unsuffixed(self.minor); + let patch = Literal::u64_unsuffixed(self.patch); + + return quote! { + semver::Version::new(#major, #minor, #patch) + }; + } +} diff --git a/crates/codegen/language/definition/src/internals/write_output_tokens/mod.rs b/crates/codegen/language/definition/src/internals/write_output_tokens/mod.rs new file mode 100644 index 0000000000..cfed029127 --- /dev/null +++ b/crates/codegen/language/definition/src/internals/write_output_tokens/mod.rs @@ -0,0 +1,5 @@ +mod implementations; + +pub trait WriteOutputTokens { + fn write_output_tokens(&self) -> proc_macro2::TokenStream; +} diff --git a/crates/codegen/language/definition/src/lib.rs b/crates/codegen/language/definition/src/lib.rs new file mode 100644 index 0000000000..fefdfe5ca2 --- /dev/null +++ b/crates/codegen/language/definition/src/lib.rs @@ -0,0 +1,6 @@ +mod compiler; +mod internals; +mod model; + +pub use compiler::*; +pub use model::*; diff --git a/crates/codegen/language/definition/src/model/identifier.rs b/crates/codegen/language/definition/src/model/identifier.rs new file mode 100644 index 0000000000..39af49ab9d --- /dev/null +++ b/crates/codegen/language/definition/src/model/identifier.rs @@ -0,0 +1,45 @@ +use serde::Serialize; +use std::ops::Deref; + +/// A wrapper type to make sure the DSL token is written as an identifier instead of a string literal. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Identifier { + value: String, +} + +impl Deref for Identifier { + type Target = String; + + fn deref(&self) -> &Self::Target { + return &self.value; + } +} + +impl std::fmt::Display for Identifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return self.value.fmt(f); + } +} + +impl From for Identifier { + fn from(value: String) -> Self { + return Self { value }; + } +} + +impl From<&str> for Identifier { + fn from(value: &str) -> Self { + return Self { + value: value.to_owned(), + }; + } +} + +impl Serialize for Identifier { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + return self.value.serialize(serializer); + } +} diff --git a/crates/codegen/language/definition/src/model/mod.rs b/crates/codegen/language/definition/src/model/mod.rs new file mode 100644 index 0000000000..28e382ee37 --- /dev/null +++ b/crates/codegen/language/definition/src/model/mod.rs @@ -0,0 +1,5 @@ +mod identifier; +mod types; + +pub use identifier::*; +pub use types::*; diff --git a/crates/codegen/language/definition/src/model/types.rs b/crates/codegen/language/definition/src/model/types.rs new file mode 100644 index 0000000000..ddd1136efd --- /dev/null +++ b/crates/codegen/language/definition/src/model/types.rs @@ -0,0 +1,308 @@ +pub use self::wrapper::*; + +/// We want to generate all Spanned types in a single module. +/// Unfortunately, module-level (inner) attribute macros are not supported yet: +/// This is why we put the attribute on a wrapper module containing all of them, then re-export its inner contents. +/// More information: https://github.com/rust-lang/rust/issues/54726 +#[codegen_language_internal_macros::derive_internals] +mod wrapper { + use crate::Identifier; + use indexmap::{IndexMap, IndexSet}; + use semver::Version; + use serde::Serialize; + use std::{path::PathBuf, rc::Rc}; + use strum_macros::{Display, EnumDiscriminants, EnumIter}; + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct Language { + pub name: Identifier, + + pub root_item: Identifier, + + pub leading_trivia: TriviaParser, + pub trailing_trivia: TriviaParser, + + pub versions: IndexSet, + + pub sections: Vec
, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct Section { + pub title: String, + pub topics: Vec, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct Topic { + pub title: String, + pub notes_file: Option, + pub lexical_context: Option, + + pub items: Vec>, + } + + #[derive(Debug, Eq, PartialEq, EnumDiscriminants, Serialize)] + #[strum_discriminants(name(ItemKind))] + #[strum_discriminants(derive(Display, EnumIter))] + pub enum Item { + Struct { item: StructItem }, + Enum { item: EnumItem }, + + Repeated { item: RepeatedItem }, + Separated { item: SeparatedItem }, + + Precedence { item: PrecedenceItem }, + + Trivia { item: TriviaItem }, + Keyword { item: KeywordItem }, + Token { item: TokenItem }, + + Fragment { item: FragmentItem }, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct StructItem { + pub name: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub error_recovery: Option, + pub fields: IndexMap, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct EnumItem { + pub name: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub default_variant: Identifier, + pub variants: Vec, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct EnumVariant { + pub name: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub error_recovery: Option, + pub fields: IndexMap, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct RepeatedItem { + pub name: Identifier, + pub repeated: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub allow_empty: Option, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct SeparatedItem { + pub name: Identifier, + pub separated: Identifier, + pub separator: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub allow_empty: Option, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct PrecedenceItem { + pub name: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub precedence_expressions: Vec, + + pub default_primary_expression: Identifier, + pub primary_expressions: Vec, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct PrecedenceExpression { + pub name: Identifier, + + pub operators: Vec, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct PrecedenceOperator { + pub model: OperatorModel, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub error_recovery: Option, + pub fields: IndexMap, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum OperatorModel { + Prefix, + Postfix, + BinaryLeftAssociative, + BinaryRightAssociative, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct PrimaryExpression { + pub expression: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct FieldsErrorRecovery { + pub terminator: Option, + pub delimiters: Option, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct FieldDelimiters { + pub open: Identifier, + pub close: Identifier, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum Field { + Required { + kind: FieldKind, + }, + Optional { + kind: FieldKind, + + enabled_in: Option, + disabled_in: Option, + }, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum FieldKind { + NonTerminal { item: Identifier }, + Terminal { items: IndexSet }, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum TriviaParser { + Sequence { parsers: Vec }, + Choice { parsers: Vec }, + + ZeroOrMore { parser: Box }, + Optional { parser: Box }, + + Trivia { trivia: Identifier }, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct TriviaItem { + pub name: Identifier, + + pub scanner: Scanner, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct KeywordItem { + pub name: Identifier, + pub identifier: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub reserved_in: Option, + pub unreserved_in: Option, + + pub value: KeywordValue, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum KeywordValue { + Sequence { + values: Vec, + }, + Optional { + value: Box, + }, + Range { + inclusive_start: usize, + inclusive_end: usize, + increment: usize, + }, + Atom { + atom: String, + }, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct TokenItem { + pub name: Identifier, + + pub definitions: Vec, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct TokenDefinition { + pub enabled_in: Option, + pub disabled_in: Option, + + pub scanner: Scanner, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub struct FragmentItem { + pub name: Identifier, + + pub enabled_in: Option, + pub disabled_in: Option, + + pub scanner: Scanner, + } + + #[derive(Debug, Eq, PartialEq, Serialize)] + pub enum Scanner { + Sequence { + scanners: Vec, + }, + Choice { + scanners: Vec, + }, + Optional { + scanner: Box, + }, + ZeroOrMore { + scanner: Box, + }, + OneOrMore { + scanner: Box, + }, + Not { + chars: IndexSet, + }, + Range { + inclusive_start: char, + inclusive_end: char, + }, + Atom { + atom: String, + }, + TrailingContext { + scanner: Box, + not_followed_by: Box, + }, + Fragment { + reference: Identifier, + }, + } +} diff --git a/crates/codegen/language/internal_macros/Cargo.toml b/crates/codegen/language/internal_macros/Cargo.toml new file mode 100644 index 0000000000..502d50c468 --- /dev/null +++ b/crates/codegen/language/internal_macros/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "codegen_language_internal_macros" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +[lib] +proc-macro = true + +[dependencies] +itertools = { workspace = true } +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } diff --git a/crates/codegen/language/internal_macros/src/derive/mod.rs b/crates/codegen/language/internal_macros/src/derive/mod.rs new file mode 100644 index 0000000000..2ab8f9098d --- /dev/null +++ b/crates/codegen/language/internal_macros/src/derive/mod.rs @@ -0,0 +1,7 @@ +mod parse_input_tokens; +mod spanned; +mod write_output_tokens; + +pub use parse_input_tokens::*; +pub use spanned::*; +pub use write_output_tokens::*; diff --git a/crates/codegen/language/internal_macros/src/derive/parse_input_tokens.rs b/crates/codegen/language/internal_macros/src/derive/parse_input_tokens.rs new file mode 100644 index 0000000000..718668ba0f --- /dev/null +++ b/crates/codegen/language/internal_macros/src/derive/parse_input_tokens.rs @@ -0,0 +1,125 @@ +use crate::model::{Field, Item, Variant}; +use itertools::Itertools; +use proc_macro2::{Ident, Literal, TokenStream}; +use quote::quote; + +pub fn parse_input_tokens(item: &Item) -> TokenStream { + return match item { + Item::Struct { name, fields, .. } => derive_struct(name, fields), + Item::Enum { name, variants, .. } => derive_enum(name, variants), + }; +} + +fn derive_struct(name: &Ident, fields: &Vec) -> TokenStream { + let name_string = Literal::string(&name.to_string()); + let unexpected_type_error = Literal::string(&format!("Expected type: {name}")); + + let fields_return = derive_fields_return(quote!(Self), fields); + + return quote! { + impl crate::internals::ParseInputTokens for #name { + fn parse_value( + input: syn::parse::ParseStream, + errors: &mut crate::internals::ErrorsCollection, + ) -> crate::internals::Result { + #fields_return + } + + fn parse_named_value( + input: syn::parse::ParseStream, + errors: &mut crate::internals::ErrorsCollection, + ) -> crate::internals::Result { + let name = crate::internals::ParseHelpers::syn::(input)?; + if name.to_string() != #name_string { + return crate::internals::Error::fatal(&name, &#unexpected_type_error); + } + + return crate::internals::ParseHelpers::delimited( + proc_macro2::Delimiter::Parenthesis, + input, + |_, input| Self::parse_value(&input, errors) + ); + } + } + }; +} + +fn derive_enum(name: &Ident, variants: &Vec) -> TokenStream { + let match_arms = variants.iter().map(|variant| { + let variant_id = &variant.name; + let variant_name = variant_id.to_string(); + + if let Some(fields) = &variant.fields { + let fields_return = derive_fields_return(quote!(Self::#variant_id), &fields); + + return quote! { + #variant_name => { + return crate::internals::ParseHelpers::delimited( + proc_macro2::Delimiter::Parenthesis, + input, + |_, input| { #fields_return } + ); + } + }; + } else { + return quote! { + #variant_name => { + return Ok(Self::#variant_id); + } + }; + } + }); + + let unknown_variant_error = Literal::string(&format!( + "Expected a variant: {}", + variants + .iter() + .map(|variant| format!("'{}'", variant.name.to_string())) + .collect_vec() + .join(" or ") + )); + + return quote! { + impl crate::internals::ParseInputTokens for #name { + fn parse_value( + input: syn::parse::ParseStream, + errors: &mut crate::internals::ErrorsCollection, + ) -> crate::internals::Result { + let variant = crate::internals::ParseHelpers::syn::(input)?; + match variant.to_string().as_str() { + #( #match_arms )* + + _ => { + return crate::internals::Error::fatal(&variant, &#unknown_variant_error); + } + }; + } + } + }; +} + +fn derive_fields_return(type_name: TokenStream, fields: &Vec) -> TokenStream { + // When there is only one field, we omit the `key = ` part. + // This way, we can just write `Foo(Bar)` instead of `Foo(key = Bar)`. + let assignments = if let [single_field] = &fields[..] { + let name = &single_field.name; + quote!( + #name: crate::internals::ParseInputTokens::parse_value(&input, errors)? + ) + } else { + fields.iter().map(|field| { + let name = &field.name; + let name_string = name.to_string(); + + return quote!( + #name: crate::internals::ParseInputTokens::parse_field(#name_string, &input, errors)?, + ); + }).collect() + }; + + return quote! { + return Ok(#type_name { + #assignments + }); + }; +} diff --git a/crates/codegen/language/internal_macros/src/derive/spanned.rs b/crates/codegen/language/internal_macros/src/derive/spanned.rs new file mode 100644 index 0000000000..011d25dc74 --- /dev/null +++ b/crates/codegen/language/internal_macros/src/derive/spanned.rs @@ -0,0 +1,119 @@ +use crate::model::{Field, Item, Variant}; +use proc_macro2::TokenStream; +use quote::quote; +use syn::{fold::Fold, parse_quote}; + +pub fn spanned(item: &Item) -> TokenStream { + match item { + Item::Struct { + name, + fields, + attributes, + } => { + let fields = fields.iter().map(derive_field); + + return quote! { + #( #attributes )* + pub struct #name { + #( pub #fields ),* + } + }; + } + Item::Enum { + name, + variants, + attributes, + } => { + let variants = variants.iter().map(derive_variant); + + return quote! { + #( #attributes )* + pub enum #name { + #(#variants),* + } + }; + } + }; +} + +fn derive_variant(variant: &Variant) -> TokenStream { + let name = &variant.name; + + match &variant.fields { + None => { + return quote! { + #name + }; + } + Some(fields) => { + let fields = fields.iter().map(derive_field); + + return quote! { + #name { + #( #fields ),* + } + }; + } + }; +} + +fn derive_field(field: &Field) -> TokenStream { + let name = &field.name; + let tipe = &field.tipe; + + let tipe = match RewriteGenericArgs::rewrite(tipe.clone()) { + Some(tipe) => tipe, + None => { + // If there are no generic args, wrap the top-level type itself: + parse_quote! { + crate::internals::Spanned<#tipe> + } + } + }; + + return quote! { + #name: #tipe + }; +} + +struct RewriteGenericArgs { + found_generic_args: bool, +} + +impl RewriteGenericArgs { + fn rewrite(input: syn::Type) -> Option { + let mut instance = RewriteGenericArgs { + found_generic_args: false, + }; + + let result = instance.fold_type(input); + + return if instance.found_generic_args { + Some(result) + } else { + None + }; + } +} + +impl Fold for RewriteGenericArgs { + fn fold_generic_argument(&mut self, input: syn::GenericArgument) -> syn::GenericArgument { + if let syn::GenericArgument::Type(inner) = input { + self.found_generic_args = true; + + let result = match RewriteGenericArgs::rewrite(inner.clone()) { + Some(inner) => inner, + None => { + // If the inner type was not already wrapped, wrap this one instead: + parse_quote! { + crate::internals::Spanned<#inner> + } + } + }; + + return syn::GenericArgument::Type(result); + } else { + return syn::fold::fold_generic_argument(self, input); + } + } +} diff --git a/crates/codegen/language/internal_macros/src/derive/write_output_tokens.rs b/crates/codegen/language/internal_macros/src/derive/write_output_tokens.rs new file mode 100644 index 0000000000..44ba09bb09 --- /dev/null +++ b/crates/codegen/language/internal_macros/src/derive/write_output_tokens.rs @@ -0,0 +1,74 @@ +use crate::model::{Field, Item, Variant}; +use itertools::Itertools; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; + +pub fn write_output_tokens(item: &Item) -> TokenStream { + let (name, implementation) = match item { + Item::Struct { name, fields, .. } => (name, derive_struct(name, fields)), + Item::Enum { name, variants, .. } => (name, derive_enum(name, variants)), + }; + + return quote! { + impl crate::internals::WriteOutputTokens for #name { + fn write_output_tokens(&self) -> proc_macro2::TokenStream { + #implementation + } + } + }; +} + +fn derive_struct(name: &Ident, fields: &Vec) -> TokenStream { + let keys = fields.iter().map(|field| &field.name).collect_vec(); + + return quote! { + #( let #keys = self.#keys.write_output_tokens(); )* + + return quote::quote! { + codegen_language_definition::#name { + #( #keys: ##keys ),* + } + }; + }; +} + +fn derive_enum(name: &Ident, variants: &Vec) -> TokenStream { + let match_arms = variants.iter().map(|variant| { + let variant_name = &variant.name; + + match &variant.fields { + None => { + return quote! { + Self::#variant_name => { + return quote::quote! { + codegen_language_definition::#name::#variant_name + }; + } + }; + } + Some(fields) => { + let keys = fields.iter().map(|field| &field.name).collect_vec(); + + return quote! { + Self::#variant_name { + #( #keys ),* + } => { + #( let #keys = #keys.write_output_tokens(); )* + + return quote::quote! { + codegen_language_definition::#name::#variant_name { + #( #keys: ##keys ),* + } + }; + } + }; + } + }; + }); + + return quote! { + match self { + #( #match_arms )* + }; + }; +} diff --git a/crates/codegen/language/internal_macros/src/lib.rs b/crates/codegen/language/internal_macros/src/lib.rs new file mode 100644 index 0000000000..d72f3c534c --- /dev/null +++ b/crates/codegen/language/internal_macros/src/lib.rs @@ -0,0 +1,65 @@ +mod derive; +mod model; +mod utils; + +use crate::{model::Model, utils::error}; +use quote::{quote, ToTokens}; +use std::borrow::BorrowMut; +use syn::Result; + +#[proc_macro_attribute] +pub fn derive_internals( + config: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + return match derive_internals_aux(config, input) { + Ok(output) => output.into(), + Err(error) => error.to_compile_error().into(), + }; +} + +fn derive_internals_aux( + config: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> Result { + if !config.is_empty() { + return error( + proc_macro2::TokenStream::from(config), + "Attribute does not support configuration.", + ); + } + + let mut input_mod = syn::parse::(input)?; + + let input_items = match input_mod.content.borrow_mut() { + Some((_, items)) => items, + None => return error(input_mod, "Expected a module containing items within."), + }; + + // Use [syn::Item::Verbatim] so that we don't end up parsing the generated items: + let output = run_derivers(input_items)?; + input_items.push(syn::Item::Verbatim(output)); + + return Ok(input_mod.into_token_stream().into()); +} + +fn run_derivers(input_items: &Vec) -> Result { + let model = Model::from_syn(input_items)?; + + let spanned = model.items().map(derive::spanned); + let parse_input_tokens = model.items().map(derive::parse_input_tokens); + let write_output_tokens = model.items().map(derive::write_output_tokens); + + // Group macro-generated items in a 'spanned' sub-module. + // It is visible only to the definition crate, and never leaked outside the context of the macro. + // Only the parent types are returned to the user via the generated definition. + return Ok(quote! { + pub(crate) mod spanned { + pub use super::*; + + #(#spanned)* + #(#parse_input_tokens)* + #(#write_output_tokens)* + } + }); +} diff --git a/crates/codegen/language/internal_macros/src/model/mod.rs b/crates/codegen/language/internal_macros/src/model/mod.rs new file mode 100644 index 0000000000..b942cc1926 --- /dev/null +++ b/crates/codegen/language/internal_macros/src/model/mod.rs @@ -0,0 +1,109 @@ +use crate::utils::error; +use itertools::Itertools; +use syn::Result; + +pub struct Model { + items: Vec, +} + +impl Model { + pub fn items(&self) -> impl Iterator { + return self.items.iter(); + } + + pub fn from_syn(input: &Vec) -> Result { + let items = input + .iter() + .filter_map(|item| match &item { + syn::Item::Struct(struct_item) => Some(Item::from_syn_struct(struct_item)), + syn::Item::Enum(enum_item) => Some(Item::from_syn_enum(enum_item)), + _ => None, + }) + .try_collect()?; + + return Ok(Model { items }); + } +} + +pub enum Item { + Struct { + name: syn::Ident, + attributes: Vec, + fields: Vec, + }, + Enum { + name: syn::Ident, + attributes: Vec, + variants: Vec, + }, +} + +impl Item { + fn from_syn_struct(input: &syn::ItemStruct) -> Result { + let syn::Fields::Named(fields) = &input.fields else { + return error(&input.fields, "Only named fields are supported."); + }; + + return Ok(Item::Struct { + name: input.ident.clone(), + attributes: input.attrs.clone(), + fields: Field::from_syn(fields), + }); + } + + fn from_syn_enum(input: &syn::ItemEnum) -> Result { + return Ok(Item::Enum { + name: input.ident.clone(), + attributes: input.attrs.clone(), + variants: input + .variants + .iter() + .map(|variant| Variant::from_syn(variant)) + .try_collect()?, + }); + } +} + +pub struct Variant { + pub name: syn::Ident, + pub fields: Option>, +} + +impl Variant { + fn from_syn(input: &syn::Variant) -> Result { + let name = input.ident.clone(); + + match &input.fields { + syn::Fields::Named(fields) => { + return Ok(Variant { + name, + fields: Some(Field::from_syn(fields)), + }); + } + syn::Fields::Unit => { + return Ok(Variant { name, fields: None }); + } + syn::Fields::Unnamed(fields) => { + return error(fields, "Unnamed fields are not supported."); + } + }; + } +} + +pub struct Field { + pub name: syn::Ident, + pub tipe: syn::Type, +} + +impl Field { + fn from_syn(input: &syn::FieldsNamed) -> Vec { + return input + .named + .iter() + .map(|field| Self { + name: field.ident.clone().unwrap(), + tipe: field.ty.clone(), + }) + .collect(); + } +} diff --git a/crates/codegen/language/internal_macros/src/utils/mod.rs b/crates/codegen/language/internal_macros/src/utils/mod.rs new file mode 100644 index 0000000000..e84b56fe0a --- /dev/null +++ b/crates/codegen/language/internal_macros/src/utils/mod.rs @@ -0,0 +1,7 @@ +use quote::ToTokens; +use std::fmt::Display; +use syn::{Error, Result}; + +pub fn error(spanned: impl ToTokens, message: impl Display) -> Result { + return Err(Error::new_spanned(spanned, message)); +} diff --git a/crates/codegen/language/macros/Cargo.toml b/crates/codegen/language/macros/Cargo.toml new file mode 100644 index 0000000000..c4178c76ea --- /dev/null +++ b/crates/codegen/language/macros/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "codegen_language_macros" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +[lib] +proc-macro = true + +[dependencies] +codegen_language_definition = { workspace = true } diff --git a/crates/codegen/language/macros/src/lib.rs b/crates/codegen/language/macros/src/lib.rs new file mode 100644 index 0000000000..c1da3bc699 --- /dev/null +++ b/crates/codegen/language/macros/src/lib.rs @@ -0,0 +1,7 @@ +use codegen_language_definition::LanguageCompiler; +use proc_macro::TokenStream; + +#[proc_macro] +pub fn compile(input: TokenStream) -> TokenStream { + return LanguageCompiler::compile(input.into()).into(); +} diff --git a/crates/codegen/language/tests/Cargo.toml b/crates/codegen/language/tests/Cargo.toml new file mode 100644 index 0000000000..2233243727 --- /dev/null +++ b/crates/codegen/language/tests/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "codegen_language_tests" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +[dev-dependencies] +codegen_language_definition = { workspace = true } +codegen_language_macros = { workspace = true } +infra_utils = { workspace = true } +rayon = { workspace = true } +semver = { workspace = true } +trybuild = { workspace = true } diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.rs b/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.rs new file mode 100644 index 0000000000..84ad827f93 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.rs @@ -0,0 +1,33 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Precedence( + name = Bar, + precedence_expressions = [ + PrecedenceExpression(name = Expression1, operators = []), + PrecedenceExpression(name = Expression2, operators = []), + PrecedenceExpression(name = Expression1, operators = []) + ], + default_primary_expression = Baz, + primary_expressions = [PrimaryExpression(expression = Baz)] + ), + Token( + name = Baz, + definitions = [TokenDefinition(scanner = Atom("baz"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.stderr b/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.stderr new file mode 100644 index 0000000000..6d6b1ea841 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.stderr @@ -0,0 +1,5 @@ +error: An expression with the name 'Expression1' already exists. + --> src/fail/definitions/duplicate_expression_name/test.rs:19:53 + | +19 | PrecedenceExpression(name = Expression1, operators = []) + | ^^^^^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.rs b/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.rs new file mode 100644 index 0000000000..5518052483 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.rs @@ -0,0 +1,22 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar1, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct(name = Bar1, fields = (field = Required(NonTerminal(Bar2)))), + Struct(name = Bar1, fields = (field = Required(NonTerminal(Bar2)))), + Struct(name = Bar2, fields = (field = Required(NonTerminal(Bar1)))) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.stderr b/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.stderr new file mode 100644 index 0000000000..42cc9efcdd --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.stderr @@ -0,0 +1,5 @@ +error: An item with the name 'Bar1' already exists. + --> src/fail/definitions/duplicate_item_name/test.rs:15:31 + | +15 | Struct(name = Bar1, fields = (field = Required(NonTerminal(Bar2)))), + | ^^^^ diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.rs b/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.rs new file mode 100644 index 0000000000..ce70cdeb50 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.rs @@ -0,0 +1,26 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [Enum( + name = Bar, + default_variant = Variant1, + variants = [ + EnumVariant(name = Variant1, fields = ()), + EnumVariant(name = Variant2, fields = ()), + EnumVariant(name = Variant1, fields = ()) + ] + )] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.stderr b/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.stderr new file mode 100644 index 0000000000..c6cb660e92 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.stderr @@ -0,0 +1,5 @@ +error: A variant with the name 'Variant1' already exists. + --> src/fail/definitions/duplicate_variant_name/test.rs:19:40 + | +19 | EnumVariant(name = Variant1, fields = ()) + | ^^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/mod.rs b/crates/codegen/language/tests/src/fail/mod.rs new file mode 100644 index 0000000000..cd8accb7bc --- /dev/null +++ b/crates/codegen/language/tests/src/fail/mod.rs @@ -0,0 +1,33 @@ +/// These test cases are not actually part of this compilation, so they are not imported via 'mod' declarations here. +/// Instead, they are individually picked up during runtime, and each is compiled separately, checking any errors they produce. +/// For more info: https://crates.io/crates/trybuild +use infra_utils::{cargo::CargoWorkspace, github::GitHub, paths::FileWalker}; +use rayon::prelude::*; + +#[test] +fn run_trybuild() { + if GitHub::is_running_in_ci() { + // This instructs 'trybuild' to fail the tests if any snapshots are out of date. + // Snapshots are written to a 'wip' directory (git-ignored). + std::env::set_var("TRYBUILD", "wip"); + } else { + // This instructs 'trybuild' to overwrite (or generate new) snapshots next to test files that generate them. + // This is useful for local development, where we always want to see the updates as we edit the tests. + std::env::set_var("TRYBUILD", "overwrite"); + } + + let crate_dir = &CargoWorkspace::locate_source_crate("codegen_language_tests").unwrap(); + + let tests = FileWalker::from_directory(crate_dir) + .find(["src/fail/**/test.rs"]) + .unwrap() + .into_iter() + .collect::>(); + + assert!(!tests.is_empty(), "No tests found."); + + tests.par_iter().for_each(|test| { + let relative_path = test.strip_prefix(crate_dir).unwrap(); + trybuild::TestCases::new().compile_fail(relative_path); + }); +} diff --git a/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.rs b/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.rs new file mode 100644 index 0000000000..17e2c3d9fa --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.rs @@ -0,0 +1,31 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = Bar, + fields = ( + field_1 = Required(Terminal([Baz])), + field_1 = Required(Terminal([Baz])), + field_3 = Required(Terminal([Baz])) + ) + ), + Token( + name = Baz, + definitions = [TokenDefinition(scanner = Atom("baz"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.stderr b/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.stderr new file mode 100644 index 0000000000..b023aa033e --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.stderr @@ -0,0 +1,5 @@ +error: Duplicate map key. + --> src/fail/parsing/duplicate_map_key/test.rs:18:25 + | +18 | field_1 = Required(Terminal([Baz])), + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.rs b/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.rs new file mode 100644 index 0000000000..7892610f3f --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.rs @@ -0,0 +1,21 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "1.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [Token( + name = Bar, + definitions = [TokenDefinition(scanner = Atom("bar"))] + )] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.stderr b/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.stderr new file mode 100644 index 0000000000..4b2794bd20 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.stderr @@ -0,0 +1,5 @@ +error: Set entries must be unique. + --> src/fail/parsing/duplicate_set_entry/test.rs:8:26 + | +8 | versions = ["1.0.0", "1.0.0", "3.0.0"], + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/empty_string/test.rs b/crates/codegen/language/tests/src/fail/parsing/empty_string/test.rs new file mode 100644 index 0000000000..e703d6681e --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/empty_string/test.rs @@ -0,0 +1,21 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [Token( + name = Bar, + definitions = [TokenDefinition(scanner = Atom(""))] + )] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/empty_string/test.stderr b/crates/codegen/language/tests/src/fail/parsing/empty_string/test.stderr new file mode 100644 index 0000000000..1795a5d709 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/empty_string/test.stderr @@ -0,0 +1,5 @@ +error: Expected a non-empty string. + --> src/fail/parsing/empty_string/test.rs:15:63 + | +15 | definitions = [TokenDefinition(scanner = Atom(""))] + | ^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/missing_field/test.rs b/crates/codegen/language/tests/src/fail/parsing/missing_field/test.rs new file mode 100644 index 0000000000..cf5ac987d2 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/missing_field/test.rs @@ -0,0 +1,15 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"], + sections = [Section( + // title = "Section One" + topics = [] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/missing_field/test.stderr b/crates/codegen/language/tests/src/fail/parsing/missing_field/test.stderr new file mode 100644 index 0000000000..056d89494f --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/missing_field/test.stderr @@ -0,0 +1,5 @@ +error: Expected field: 'title'. + --> src/fail/parsing/missing_field/test.rs:11:9 + | +11 | topics = [] + | ^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.rs b/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.rs new file mode 100644 index 0000000000..e3a471ef6e --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.rs @@ -0,0 +1,32 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar1, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [ + Topic( + title = "Topic One", + notes_file = "README.md", // should be correct + items = [Struct( + name = Bar1, + fields = (field = Required(NonTerminal(Bar2))) + )] + ), + Topic( + title = "Topic Two", + notes_file = "foo/bar/non-existing-file.md", + items = [Struct( + name = Bar2, + fields = (field = Required(NonTerminal(Bar1))) + )] + ) + ] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.stderr b/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.stderr new file mode 100644 index 0000000000..0a9b096a45 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/non_existing_path_buf/test.stderr @@ -0,0 +1,5 @@ +error: Path not found: $WORKSPACE/foo/bar/non-existing-file.md + --> src/fail/parsing/non_existing_path_buf/test.rs:22:30 + | +22 | notes_file = "foo/bar/non-existing-file.md", + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.rs b/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.rs new file mode 100644 index 0000000000..3fcec0ba75 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.rs @@ -0,0 +1,22 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [Token( + name = Bar, + definitions = [TokenDefinition(scanner = Atom("bar"))] + )], + unrecognized_field = true + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.stderr b/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.stderr new file mode 100644 index 0000000000..baa992355d --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.stderr @@ -0,0 +1,5 @@ +error: unexpected token + --> src/fail/parsing/unrecognized_field/test.rs:17:13 + | +17 | unrecognized_field = true + | ^^^^^^^^^^^^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.rs b/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.rs new file mode 100644 index 0000000000..ff0f51200a --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.rs @@ -0,0 +1,24 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar1, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [ + Topic( + title = "Topic One", + items = [ + Struct(name = Bar1, fields = (field = Required(NonTerminal(Bar2)))), + Struct(name = Bar2, fields = (field = Required(NonTerminal(Bar1)))) + ] + ), + Topic(title = "Topic Two", items = [Unrecognized(true)]) + ] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.stderr b/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.stderr new file mode 100644 index 0000000000..136647fc86 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.stderr @@ -0,0 +1,5 @@ +error: Expected a variant: 'Struct' or 'Enum' or 'Repeated' or 'Separated' or 'Precedence' or 'Trivia' or 'Keyword' or 'Token' or 'Fragment' + --> src/fail/parsing/unrecognized_variant/test.rs:19:49 + | +19 | Topic(title = "Topic Two", items = [Unrecognized(true)]) + | ^^^^^^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/reachability/unreachable/test.rs b/crates/codegen/language/tests/src/fail/reachability/unreachable/test.rs new file mode 100644 index 0000000000..dd50c2800f --- /dev/null +++ b/crates/codegen/language/tests/src/fail/reachability/unreachable/test.rs @@ -0,0 +1,25 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar1, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + // reachable (from root) + Struct(name = Bar1, fields = (field = Required(NonTerminal(Bar2)))), + Struct(name = Bar2, fields = (field = Required(NonTerminal(Bar1)))), + // not-reachable + Struct(name = Baz1, fields = (field = Required(NonTerminal(Baz2)))), + Struct(name = Baz2, fields = (field = Required(NonTerminal(Baz1)))) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/reachability/unreachable/test.stderr b/crates/codegen/language/tests/src/fail/reachability/unreachable/test.stderr new file mode 100644 index 0000000000..3eb13eb69a --- /dev/null +++ b/crates/codegen/language/tests/src/fail/reachability/unreachable/test.stderr @@ -0,0 +1,11 @@ +error: Item 'Baz1' is not reachable from grammar root. + --> src/fail/reachability/unreachable/test.rs:18:31 + | +18 | Struct(name = Baz1, fields = (field = Required(NonTerminal(Baz2)))), + | ^^^^ + +error: Item 'Baz2' is not reachable from grammar root. + --> src/fail/reachability/unreachable/test.rs:19:31 + | +19 | Struct(name = Baz2, fields = (field = Required(NonTerminal(Baz1)))) + | ^^^^ diff --git a/crates/codegen/language/tests/src/fail/reachability/unused_version/test.rs b/crates/codegen/language/tests/src/fail/reachability/unused_version/test.rs new file mode 100644 index 0000000000..0eb7224885 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/reachability/unused_version/test.rs @@ -0,0 +1,27 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = Bar, + fields = (field_1 = Optional(kind = Terminal([Baz]), enabled_in = "2.0.0")) + ), + Token( + name = Baz, + definitions = [TokenDefinition(scanner = Atom("baz"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/reachability/unused_version/test.stderr b/crates/codegen/language/tests/src/fail/reachability/unused_version/test.stderr new file mode 100644 index 0000000000..b9ce59a916 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/reachability/unused_version/test.stderr @@ -0,0 +1,5 @@ +error: Item 'Baz' is not used in versions: 1.0.0..2.0.0 + --> src/fail/reachability/unused_version/test.rs:19:28 + | +19 | name = Baz, + | ^^^ diff --git a/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.rs b/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.rs new file mode 100644 index 0000000000..c9fcc60259 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.rs @@ -0,0 +1,40 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = One, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = One, + fields = ( + field_1 = Optional( + kind = NonTerminal(Two), + enabled_in = "2.0.0", + disabled_in = "3.0.0" + ), + field_2 = Optional(kind = Terminal([Three])) + ) + ), + Struct( + name = Two, + enabled_in = "2.0.0", + disabled_in = "3.0.0", + fields = (field_1 = Optional(kind = Terminal([Three]), disabled_in = "4.0.0")) + ), + Token( + name = Three, + definitions = [TokenDefinition(scanner = Atom("three"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.stderr b/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.stderr new file mode 100644 index 0000000000..99bd3977ca --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/disabled_too_late/test.stderr @@ -0,0 +1,5 @@ +error: Parent scope is disabled in '3.0.0'. + --> src/fail/references/disabled_too_late/test.rs:29:90 + | +29 | fields = (field_1 = Optional(kind = Terminal([Three]), disabled_in = "4.0.0")) + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.rs b/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.rs new file mode 100644 index 0000000000..e4e161c3e0 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.rs @@ -0,0 +1,40 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = One, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = One, + fields = ( + field_1 = Optional( + kind = NonTerminal(Two), + enabled_in = "2.0.0", + disabled_in = "3.0.0" + ), + field_2 = Optional(kind = Terminal([Three])) + ) + ), + Struct( + name = Two, + enabled_in = "2.0.0", + disabled_in = "3.0.0", + fields = (field_1 = Optional(kind = Terminal([Three]), enabled_in = "1.0.0")) + ), + Token( + name = Three, + definitions = [TokenDefinition(scanner = Atom("three"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.stderr b/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.stderr new file mode 100644 index 0000000000..5eb1c309be --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/enabled_too_early/test.stderr @@ -0,0 +1,5 @@ +error: Parent scope is enabled in '2.0.0'. + --> src/fail/references/enabled_too_early/test.rs:29:89 + | +29 | fields = (field_1 = Optional(kind = Terminal([Three]), enabled_in = "1.0.0")) + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/references/invalid_filter/test.rs b/crates/codegen/language/tests/src/fail/references/invalid_filter/test.rs new file mode 100644 index 0000000000..dc983ea7b5 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/invalid_filter/test.rs @@ -0,0 +1,21 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct(name = Bar, fields = (field = Required(NonTerminal(Baz)))), + Fragment(name = Baz, scanner = Atom("baz")) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/invalid_filter/test.stderr b/crates/codegen/language/tests/src/fail/references/invalid_filter/test.stderr new file mode 100644 index 0000000000..a5f04e7f12 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/invalid_filter/test.stderr @@ -0,0 +1,5 @@ +error: Reference 'Baz' of kind 'Fragment' is not valid. Expected: [Struct, Enum, Repeated, Separated, Precedence]. + --> src/fail/references/invalid_filter/test.rs:14:75 + | +14 | Struct(name = Bar, fields = (field = Required(NonTerminal(Baz)))), + | ^^^ diff --git a/crates/codegen/language/tests/src/fail/references/invalid_version/test.rs b/crates/codegen/language/tests/src/fail/references/invalid_version/test.rs new file mode 100644 index 0000000000..23fb4339d4 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/invalid_version/test.rs @@ -0,0 +1,52 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = Bar, + fields = ( + field_1 = Optional( + // should have been disabled in "3.0.0" + kind = Terminal([Baz]), + enabled_in = "2.0.0" + ), + field_2 = Optional( + // should have been enabled in "2.0.0" + kind = Terminal([Baz]), + disabled_in = "3.0.0" + ), + field_3 = Optional( + // should have been enabled in "2.0.0" and disabled in "3.0.0" + kind = Terminal([Baz]) + ), + field_4 = Optional( + // correct + kind = Terminal([Baz]), + enabled_in = "2.0.0", + disabled_in = "3.0.0" + ) + ) + ), + Token( + name = Baz, + definitions = [TokenDefinition( + enabled_in = "2.0.0", + disabled_in = "3.0.0", + scanner = Atom("baz") + )] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/invalid_version/test.stderr b/crates/codegen/language/tests/src/fail/references/invalid_version/test.stderr new file mode 100644 index 0000000000..de5244fead --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/invalid_version/test.stderr @@ -0,0 +1,17 @@ +error: Reference 'Baz' is not defined in versions '3.0.0..MAX'. + --> src/fail/references/invalid_version/test.rs:19:46 + | +19 | ... kind = Terminal([Baz]), + | ^^^ + +error: Reference 'Baz' is not defined in versions '1.0.0..2.0.0'. + --> src/fail/references/invalid_version/test.rs:24:46 + | +24 | ... kind = Terminal([Baz]), + | ^^^ + +error: Reference 'Baz' is not defined in versions '1.0.0..2.0.0 | 3.0.0..MAX'. + --> src/fail/references/invalid_version/test.rs:29:46 + | +29 | ... kind = Terminal([Baz]) + | ^^^ diff --git a/crates/codegen/language/tests/src/fail/references/unknown_reference/test.rs b/crates/codegen/language/tests/src/fail/references/unknown_reference/test.rs new file mode 100644 index 0000000000..dde40bd4d3 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/unknown_reference/test.rs @@ -0,0 +1,30 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = Bar, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = Bar, + fields = ( + field_1 = Required(Terminal([Baz])), + field_2 = Required(Terminal([Unknown])) + ) + ), + Token( + name = Baz, + definitions = [TokenDefinition(scanner = Atom("baz"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/unknown_reference/test.stderr b/crates/codegen/language/tests/src/fail/references/unknown_reference/test.stderr new file mode 100644 index 0000000000..6c783e5921 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/unknown_reference/test.stderr @@ -0,0 +1,5 @@ +error: Reference to unknown item 'Unknown'. + --> src/fail/references/unknown_reference/test.rs:18:54 + | +18 | field_2 = Required(Terminal([Unknown])) + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.rs b/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.rs new file mode 100644 index 0000000000..88bffa648b --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.rs @@ -0,0 +1,34 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = One, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = One, + fields = ( + field_1 = Optional( + kind = Terminal([Two]), + enabled_in = "3.0.0", + disabled_in = "2.0.0" + ), + field_2 = Optional(kind = Terminal([Two])) + ) + ), + Token( + name = Two, + definitions = [TokenDefinition(scanner = Atom("two"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.stderr b/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.stderr new file mode 100644 index 0000000000..1efd522ede --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/unordered_version_pair/test.stderr @@ -0,0 +1,5 @@ +error: Version '3.0.0' must be less than corresponding version '2.0.0'. + --> src/fail/references/unordered_version_pair/test.rs:19:42 + | +19 | ... enabled_in = "3.0.0", + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/fail/references/version_not_found/test.rs b/crates/codegen/language/tests/src/fail/references/version_not_found/test.rs new file mode 100644 index 0000000000..34f883a0cb --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/version_not_found/test.rs @@ -0,0 +1,30 @@ +#![allow(unused_crate_dependencies)] + +codegen_language_macros::compile!(Language( + name = Foo, + root_item = One, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = One, + fields = ( + field_1 = Optional(kind = Terminal([Two]), enabled_in = "3.0.0"), + field_2 = Optional(kind = Terminal([Two])) + ) + ), + Token( + name = Two, + definitions = [TokenDefinition(scanner = Atom("two"))] + ) + ] + )] + )] +)); + +fn main() {} diff --git a/crates/codegen/language/tests/src/fail/references/version_not_found/test.stderr b/crates/codegen/language/tests/src/fail/references/version_not_found/test.stderr new file mode 100644 index 0000000000..77f9531fc5 --- /dev/null +++ b/crates/codegen/language/tests/src/fail/references/version_not_found/test.stderr @@ -0,0 +1,5 @@ +error: Version '3.0.0' does not exist in the language definition. + --> src/fail/references/version_not_found/test.rs:17:81 + | +17 | field_1 = Optional(kind = Terminal([Two]), enabled_in = "3.0.0"), + | ^^^^^^^ diff --git a/crates/codegen/language/tests/src/lib.rs b/crates/codegen/language/tests/src/lib.rs new file mode 100644 index 0000000000..2a618262ac --- /dev/null +++ b/crates/codegen/language/tests/src/lib.rs @@ -0,0 +1,4 @@ +#![cfg(test)] + +mod fail; +mod pass; diff --git a/crates/codegen/language/tests/src/pass/mod.rs b/crates/codegen/language/tests/src/pass/mod.rs new file mode 100644 index 0000000000..17b8a5aad3 --- /dev/null +++ b/crates/codegen/language/tests/src/pass/mod.rs @@ -0,0 +1 @@ +mod tiny_language; diff --git a/crates/codegen/language/tests/src/pass/tiny_language.rs b/crates/codegen/language/tests/src/pass/tiny_language.rs new file mode 100644 index 0000000000..a3b7115a45 --- /dev/null +++ b/crates/codegen/language/tests/src/pass/tiny_language.rs @@ -0,0 +1,130 @@ +use codegen_language_definition::{ + Field, FieldKind, Item, Language, Scanner, Section, StructItem, TokenDefinition, TokenItem, + Topic, TriviaParser, +}; +use semver::Version; + +codegen_language_macros::compile!(Language( + name = Tiny, + root_item = Foo, + leading_trivia = Sequence([]), + trailing_trivia = Sequence([]), + versions = ["1.0.0", "2.0.0", "3.0.0"], + sections = [Section( + title = "Section One", + topics = [Topic( + title = "Topic One", + items = [ + Struct( + name = Foo, + fields = ( + bar = Required(Terminal([Bar])), + baz = Required(Terminal([Baz])), + baz_again = Required(Terminal([Baz])) + ) + ), + Token( + name = Bar, + definitions = [TokenDefinition(scanner = Atom("bar"))] + ), + Token( + name = Baz, + definitions = [TokenDefinition(scanner = Atom("baz"))] + ) + ] + )] + )] +)); + +#[test] +fn definition() { + assert_eq!( + tiny::TinyDefinition::create(), + Language { + name: "Tiny".into(), + root_item: "Foo".into(), + leading_trivia: TriviaParser::Sequence { parsers: [].into() }, + trailing_trivia: TriviaParser::Sequence { parsers: [].into() }, + versions: [ + Version::parse("1.0.0").unwrap(), + Version::parse("2.0.0").unwrap(), + Version::parse("3.0.0").unwrap(), + ] + .into(), + sections: vec![Section { + title: "Section One".into(), + topics: vec![Topic { + title: "Topic One".into(), + notes_file: None, + lexical_context: None, + items: [ + Item::Struct { + item: StructItem { + name: "Foo".into(), + enabled_in: None, + disabled_in: None, + error_recovery: None, + fields: [ + ( + "bar".into(), + Field::Required { + kind: FieldKind::Terminal { + items: ["Bar".into()].into() + } + } + ) + .into(), + ( + "baz".into(), + Field::Required { + kind: FieldKind::Terminal { + items: ["Baz".into()].into() + } + } + ) + .into(), + ( + "baz_again".into(), + Field::Required { + kind: FieldKind::Terminal { + items: ["Baz".into()].into() + } + } + ) + .into() + ] + .into() + } + } + .into(), + Item::Token { + item: TokenItem { + name: "Bar".into(), + definitions: [TokenDefinition { + enabled_in: None, + disabled_in: None, + scanner: Scanner::Atom { atom: "bar".into() } + }] + .into() + } + } + .into(), + Item::Token { + item: TokenItem { + name: "Baz".into(), + definitions: [TokenDefinition { + enabled_in: None, + disabled_in: None, + scanner: Scanner::Atom { atom: "baz".into() } + }] + .into() + } + } + .into() + ] + .into() + }], + }], + }, + ); +} diff --git a/crates/codegen/schema/Cargo.toml b/crates/codegen/schema/Cargo.toml index f5dfbd6f62..ea9f766467 100644 --- a/crates/codegen/schema/Cargo.toml +++ b/crates/codegen/schema/Cargo.toml @@ -9,6 +9,7 @@ publish = false anyhow = { workspace = true } infra_utils = { workspace = true } indexmap = { workspace = true } +itertools = { workspace = true } schemars = { workspace = true } semver = { workspace = true } serde = { workspace = true } diff --git a/crates/codegen/schema/src/validation/visitors/receivers.rs b/crates/codegen/schema/src/validation/visitors/receivers.rs index e79dc74d45..d7cdef8c60 100644 --- a/crates/codegen/schema/src/validation/visitors/receivers.rs +++ b/crates/codegen/schema/src/validation/visitors/receivers.rs @@ -80,7 +80,7 @@ where let location = location.field("unversioned"); let first_version = language.versions[0].to_owned(); - let version_set = VersionSet::range(first_version..max_version); + let version_set = VersionSet::from_range(first_version..max_version); if visitor.visit_version(&version_set, &location, reporter) { instance.receive(visitor, language, location, reporter); @@ -101,7 +101,7 @@ where if let Some(value) = instance { let location = location.field(range.start.to_string()); - let version_set = VersionSet::range(range); + let version_set = VersionSet::from_range(range); if visitor.visit_version(&version_set, &location, reporter) { value.receive(visitor, language, location, reporter); diff --git a/crates/codegen/schema/src/validation/visitors/version_set.rs b/crates/codegen/schema/src/validation/visitors/version_set.rs index 4aa5925fd1..975c5d65f3 100644 --- a/crates/codegen/schema/src/validation/visitors/version_set.rs +++ b/crates/codegen/schema/src/validation/visitors/version_set.rs @@ -1,9 +1,11 @@ use std::{ cmp::{max, min}, fmt::Display, + iter::once, ops::Range, }; +use itertools::Itertools; use semver::Version; pub type VersionRange = Range; @@ -14,120 +16,127 @@ pub struct VersionSet { } impl VersionSet { - pub fn from(ranges: Vec) -> Self { - for range in &ranges { - // End is always exclusive: - assert!(range.start < range.end, "Invalid range: {range:?}"); - } - - return Self { ranges }; + pub fn empty() -> Self { + return Self { ranges: vec![] }; } - pub fn empty() -> Self { - return Self::from(vec![]); + pub fn from_range(range: VersionRange) -> Self { + let mut instance = Self::empty(); + + instance.add(&range); + + return instance; } - pub fn range(range: VersionRange) -> Self { - return Self::from(vec![range]); + #[cfg(test)] + fn from_ranges(ranges: Vec) -> Self { + let mut instance = Self::empty(); + + for range in &ranges { + instance.add(range); + } + + return instance; } pub fn is_empty(&self) -> bool { return self.ranges.is_empty(); } - pub fn union(&self, other: &Self) -> Self { - let mut first_iter = self.ranges.iter().peekable(); - let mut second_iter = other.ranges.iter().peekable(); - let mut results = Vec::new(); + pub fn add(&mut self, range: &VersionRange) { + if range.is_empty() { + return; + } - loop { - let first = if let Some(first) = first_iter.peek() { - first - } else { - break; - }; + let mut result = vec![]; - let second = if let Some(second) = second_iter.peek() { - second - } else { - break; - }; + // Iterate on both (existing and new) in-order, combining them if they overlap: + let mut input_iter = self + .ranges + .iter() + .chain(once(range)) + .sorted_by_key(|range| &range.start); - if first.end < second.start { - // first fully exists before second, take it: - results.push(first_iter.next().unwrap().to_owned()); - break; - } + let mut current = input_iter.next().unwrap().to_owned(); - if second.end < first.start { - // second fully exists before first, take it: - results.push(second_iter.next().unwrap().to_owned()); - break; + for next in input_iter { + if current.end < next.start { + // current fully exists before next: + result.push(current); + current = next.to_owned(); + } else { + // current and next overlap, combine them: + current = Range { + start: min(¤t.start, &next.start).to_owned(), + end: max(¤t.end, &next.end).to_owned(), + }; } + } - // overlap, take the union: - let lower = min(first.start.to_owned(), second.start.to_owned()); - let upper = max(first.end.to_owned(), second.end.to_owned()); + result.push(current); + self.ranges = result; + } - first_iter.next().unwrap(); - second_iter.next().unwrap(); + pub fn union(&self, other: &Self) -> Self { + let mut result = self.to_owned(); - results.push(lower..upper); + for range in &other.ranges { + result.add(range); } - results.extend(first_iter.cloned()); - results.extend(second_iter.cloned()); - - return Self::from(results); + return result; } pub fn difference(&self, other: &Self) -> Self { - let mut first_iter = self.ranges.iter().peekable(); + let mut result = vec![]; + + let mut first_iter = self.ranges.iter().cloned().peekable(); let mut second_iter = other.ranges.iter().peekable(); - let mut results = Vec::new(); loop { - let first = if let Some(first) = first_iter.peek() { - first - } else { + let (Some(first), Some(second)) = (first_iter.peek_mut(), second_iter.peek()) else { break; }; - let second = if let Some(second) = second_iter.peek() { - second - } else { - break; - }; - - if first.end < second.start { - // first fully exists before second, take it: - results.push(first_iter.next().unwrap().to_owned()); - break; + if first.end <= second.start { + // first fully exists before second: take it, and advance first: + result.push(first_iter.next().unwrap()); + continue; } - if second.end < first.start { - // second fully exists before first, drop it: - second_iter.next().unwrap(); - break; + if second.end <= first.start { + // second fully exists before first: advance second: + second_iter.next(); + continue; } + // first and second overlap: + if first.start < second.start { // take part of first that exists before second: - results.push(first.start.to_owned()..second.start.to_owned()); - } else if first.end > second.end { - // take part of first that exists after second: - results.push(second.end.to_owned()..first.end.to_owned()); - } else { - // first fully exists within second, do nothing: + result.push(Range { + start: first.start.to_owned(), + end: second.start.to_owned(), + }); } - first_iter.next().unwrap(); - second_iter.next().unwrap(); + if first.end <= second.end { + // first ends before second: advance first, as it's been fully processed: + first_iter.next(); + continue; + } + + // keep part of first that exists after second: + first.start = second.end.to_owned(); + + // advance second, as it's been fully processed: + second_iter.next(); } - results.extend(first_iter.cloned()); + // Take anything remaining in first: + result.extend(first_iter); - return Self::from(results); + return Self { ranges: result }; } pub fn max_version() -> Version { @@ -164,3 +173,144 @@ impl Display for VersionSet { return Ok(()); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_ranges() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(3, 0, 0)..Version::new(3, 0, 0), + Version::new(4, 0, 0)..Version::new(5, 0, 0), + ]); + + assert_eq!(set.to_string(), "1.0.0..2.0.0 | 4.0.0..5.0.0"); + } + + #[test] + fn connected_ranges_in_order() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(2, 0, 0)..Version::new(3, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + ]); + + assert_eq!(set.to_string(), "1.0.0..4.0.0"); + } + + #[test] + fn connected_ranges_out_of_order() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + Version::new(2, 0, 0)..Version::new(3, 0, 0), + ]); + + assert_eq!(set.to_string(), "1.0.0..4.0.0"); + } + + #[test] + fn disconnected_ranges_in_order() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + Version::new(5, 0, 0)..Version::new(6, 0, 0), + ]); + + assert_eq!( + set.to_string(), + "1.0.0..2.0.0 | 3.0.0..4.0.0 | 5.0.0..6.0.0" + ); + } + + #[test] + fn disconnected_ranges_out_of_order() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(5, 0, 0)..Version::new(6, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + ]); + + assert_eq!( + set.to_string(), + "1.0.0..2.0.0 | 3.0.0..4.0.0 | 5.0.0..6.0.0" + ); + } + + #[test] + fn overlap_with_multiple() { + let set = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + Version::new(5, 0, 0)..Version::new(6, 0, 0), + Version::new(1, 0, 0)..Version::new(5, 0, 0), + ]); + + assert_eq!(set.to_string(), "1.0.0..6.0.0"); + } + + #[test] + fn difference_between_same_sets_is_empty() { + let first = VersionSet::from_ranges(vec![Version::new(1, 0, 0)..Version::new(2, 0, 0)]); + + let second = VersionSet::from_ranges(vec![Version::new(1, 0, 0)..Version::new(2, 0, 0)]); + + assert!(first.difference(&second).is_empty()); + } + + #[test] + fn difference_between_connected_sets() { + let first = VersionSet::from_ranges(vec![Version::new(1, 0, 0)..Version::new(5, 0, 0)]); + + let second = VersionSet::from_ranges(vec![Version::new(3, 0, 0)..Version::new(8, 0, 0)]); + + assert_eq!(first.difference(&second).to_string(), "1.0.0..3.0.0"); + + assert_eq!(second.difference(&first).to_string(), "5.0.0..8.0.0"); + } + + #[test] + fn difference_between_disconnected_sets() { + let first = VersionSet::from_ranges(vec![Version::new(1, 0, 0)..Version::new(4, 0, 0)]); + + let second = VersionSet::from_ranges(vec![Version::new(6, 0, 0)..Version::new(10, 0, 0)]); + + assert_eq!(first.difference(&second).to_string(), "1.0.0..4.0.0"); + + assert_eq!(second.difference(&first).to_string(), "6.0.0..10.0.0"); + } + + #[test] + fn difference_between_contained_sets() { + let first = VersionSet::from_ranges(vec![Version::new(1, 0, 0)..Version::new(8, 0, 0)]); + + let second = VersionSet::from_ranges(vec![Version::new(3, 0, 0)..Version::new(5, 0, 0)]); + + assert_eq!( + first.difference(&second).to_string(), + "1.0.0..3.0.0 | 5.0.0..8.0.0" + ); + + assert!(second.difference(&first).is_empty()); + } + + #[test] + fn difference_between_multiple_contained_sets() { + let first = VersionSet::from_ranges(vec![ + Version::new(1, 0, 0)..Version::new(2, 0, 0), + Version::new(3, 0, 0)..Version::new(4, 0, 0), + Version::new(5, 0, 0)..Version::new(6, 0, 0), + ]); + + let second = VersionSet::from_ranges(vec![Version::new(0, 0, 0)..Version::new(7, 0, 0)]); + + assert!(first.difference(&second).is_empty()); + + assert_eq!( + second.difference(&first).to_string(), + "0.0.0..1.0.0 | 2.0.0..3.0.0 | 4.0.0..5.0.0 | 6.0.0..7.0.0" + ); + } +} diff --git a/crates/infra/cli/src/commands/run/mod.rs b/crates/infra/cli/src/commands/run/mod.rs index b758a49460..f8c07dacce 100644 --- a/crates/infra/cli/src/commands/run/mod.rs +++ b/crates/infra/cli/src/commands/run/mod.rs @@ -54,6 +54,7 @@ impl RunController { command .property("--bin", &crate_name) + .flag("--offline") .arg("--") .args(&self.args) // Execute in the crate dir, to make use of a local './target' dir if it exists: diff --git a/crates/infra/cli/src/commands/test/mod.rs b/crates/infra/cli/src/commands/test/mod.rs index 43354a743c..39c83ea5f1 100644 --- a/crates/infra/cli/src/commands/test/mod.rs +++ b/crates/infra/cli/src/commands/test/mod.rs @@ -1,8 +1,7 @@ +use crate::utils::{ClapExtensions, OrderedCommand, Terminal}; use anyhow::Result; use clap::{Parser, ValueEnum}; -use infra_utils::commands::Command; - -use crate::utils::{ClapExtensions, OrderedCommand, Terminal}; +use infra_utils::{commands::Command, github::GitHub}; #[derive(Clone, Debug, Default, Parser)] pub struct TestController { @@ -36,14 +35,19 @@ impl OrderedCommand for TestCommand { } fn test_cargo() -> Result<()> { - return Command::new("cargo") + let mut command = Command::new("cargo") .arg("test") - .flag("--no-fail-fast") + .flag("--quiet") .flag("--offline") .flag("--all") .flag("--all-targets") - .flag("--all-features") - .run(); + .flag("--all-features"); + + if GitHub::is_running_in_ci() { + command = command.flag("--no-fail-fast"); + } + + return command.run(); } fn test_npm() -> Result<()> { diff --git a/crates/solidity/inputs/language/Cargo.toml b/crates/solidity/inputs/language/Cargo.toml index 053ed6d631..efb29b99b0 100644 --- a/crates/solidity/inputs/language/Cargo.toml +++ b/crates/solidity/inputs/language/Cargo.toml @@ -16,7 +16,9 @@ infra_utils = { workspace = true } [dependencies] anyhow = { workspace = true } bson = { workspace = true } -codegen_schema = { workspace = true } codegen_grammar = { workspace = true } +codegen_language_definition = { workspace = true } +codegen_language_macros = { workspace = true } +codegen_schema = { workspace = true } once_cell = { workspace = true } semver = { workspace = true } diff --git a/crates/solidity/inputs/language/src/definition.rs b/crates/solidity/inputs/language/src/definition.rs new file mode 100644 index 0000000000..4b10a5302c --- /dev/null +++ b/crates/solidity/inputs/language/src/definition.rs @@ -0,0 +1,4019 @@ +pub use solidity::SolidityDefinition; + +codegen_language_macros::compile!(Language( + name = Solidity, + root_item = SourceUnit, + leading_trivia = ZeroOrMore(Choice([ + Trivia(Whitespace), + Trivia(EndOfLine), + Trivia(SingleLineComment), + Trivia(MultilineComment) + ])), + trailing_trivia = Sequence([ + Optional(Trivia(Whitespace)), + Optional(Trivia(SingleLineComment)), + Optional(Trivia(EndOfLine)) + ]), + versions = [ + "0.4.11", "0.4.12", "0.4.13", "0.4.14", "0.4.15", "0.4.16", "0.4.17", "0.4.18", "0.4.19", + "0.4.20", "0.4.21", "0.4.22", "0.4.23", "0.4.24", "0.4.25", "0.4.26", "0.5.0", "0.5.1", + "0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6", "0.5.7", "0.5.8", "0.5.9", "0.5.10", "0.5.11", + "0.5.12", "0.5.13", "0.5.14", "0.5.15", "0.5.16", "0.5.17", "0.6.0", "0.6.1", "0.6.2", + "0.6.3", "0.6.4", "0.6.5", "0.6.6", "0.6.7", "0.6.8", "0.6.9", "0.6.10", "0.6.11", + "0.6.12", "0.7.0", "0.7.1", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.8.0", "0.8.1", + "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", + "0.8.12", "0.8.13", "0.8.14", "0.8.15", "0.8.16", "0.8.17", "0.8.18", "0.8.19" + ], + sections = [ + Section( + title = "File Structure", + topics = [ + Topic( + title = "Source Unit", + items = [ + Struct( + name = SourceUnit, + fields = (members = Required(NonTerminal(SourceUnitMembers))) + ), + Repeated( + name = SourceUnitMembers, + repeated = SourceUnitMember, + allow_empty = true + ), + Enum( + name = SourceUnitMember, + default_variant = Contract, + variants = [ + EnumVariant( + name = Pragma, + fields = (directive = Required(NonTerminal(PragmaDirective))) + ), + EnumVariant( + name = Import, + fields = (directive = Required(NonTerminal(ImportDirective))) + ), + EnumVariant( + name = Contract, + fields = + (definition = Required(NonTerminal(ContractDefinition))) + ), + EnumVariant( + name = Interface, + fields = + (definition = Required(NonTerminal(InterfaceDefinition))) + ), + EnumVariant( + name = Library, + fields = + (definition = Required(NonTerminal(LibraryDefinition))) + ), + EnumVariant( + name = Struct, + enabled_in = "0.6.0", + fields = (definition = Required(NonTerminal(StructDefinition))) + ), + EnumVariant( + name = Enum, + enabled_in = "0.6.0", + fields = (definition = Required(NonTerminal(EnumDefinition))) + ), + EnumVariant( + name = Function, + enabled_in = "0.7.1", + fields = + (definition = Required(NonTerminal(FunctionDefinition))) + ), + EnumVariant( + name = Constant, + enabled_in = "0.7.4", + fields = + (definition = Required(NonTerminal(ConstantDefinition))) + ), + EnumVariant( + name = Error, + enabled_in = "0.8.4", + fields = (definition = Required(NonTerminal(ErrorDefinition))) + ), + EnumVariant( + name = UserDefinedValueType, + enabled_in = "0.8.8", + fields = (definition = + Required(NonTerminal(UserDefinedValueTypeDefinition))) + ), + EnumVariant( + name = Using, + enabled_in = "0.8.13", + fields = (directive = Required(NonTerminal(UsingDirective))) + ) + ] + ) + ] + ), + Topic( + title = "Pragma Directives", + lexical_context = Pragma, + items = [ + Struct( + name = PragmaDirective, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + pragma_keyword = Required(Terminal([PragmaKeyword])), + pragma = Required(NonTerminal(Pragma)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Enum( + name = Pragma, + default_variant = Version, + variants = [ + EnumVariant( + name = ABICoder, + fields = ( + abicoder_keyword = Required(Terminal([AbicoderKeyword])), + version = Required(Terminal([Identifier])) + ) + ), + EnumVariant( + name = Experimental, + fields = ( + experimental_keyword = + Required(Terminal([ExperimentalKeyword])), + feature = + Required(Terminal([AsciiStringLiteral, Identifier])) + ) + ), + EnumVariant( + name = Version, + fields = ( + solidity_keyword = Required(Terminal([SolidityKeyword])), + expressions = + Required(NonTerminal(VersionPragmaExpressions)) + ) + ) + ] + ), + Repeated( + name = VersionPragmaExpressions, + repeated = VersionPragmaExpression + ), + Precedence( + name = VersionPragmaExpression, + precedence_expressions = [ + PrecedenceExpression( + name = VersionPragmaOrExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([BarBar]))) + )] + ), + PrecedenceExpression( + name = VersionPragmaRangeExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([Minus]))) + )] + ), + PrecedenceExpression( + name = VersionPragmaPrefixExpression, + operators = [PrecedenceOperator( + model = Prefix, + fields = (operator = Required(Terminal([ + Caret, + Tilde, + Equal, + LessThan, + GreaterThan, + LessThanEqual, + GreaterThanEqual + ]))) + )] + ) + ], + default_primary_expression = VersionPragmaSpecifier, + primary_expressions = + [PrimaryExpression(expression = VersionPragmaSpecifier)] + ), + Separated( + name = VersionPragmaSpecifier, + separated = VersionPragmaValue, + separator = Period + ), + Token( + name = VersionPragmaValue, + definitions = [TokenDefinition( + scanner = OneOrMore(Choice([ + Range(inclusive_start = '0', inclusive_end = '9'), + Atom("x"), + Atom("X"), + Atom("*") + ])) + )] + ) + ] + ), + Topic( + title = "Import Directives", + items = [ + Struct( + name = ImportDirective, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + import_keyword = Required(Terminal([ImportKeyword])), + symbol = Required(NonTerminal(ImportSymbol)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Enum( + name = ImportSymbol, + default_variant = Path, + variants = [ + EnumVariant( + name = Path, + fields = ( + path = Required(Terminal([AsciiStringLiteral])), + alias = Optional(kind = NonTerminal(ImportAlias)) + ) + ), + EnumVariant( + name = Named, + fields = ( + asterisk = Required(Terminal([Asterisk])), + alias = Required(NonTerminal(ImportAlias)), + from_keyword = Required(Terminal([FromKeyword])), + path = Required(Terminal([AsciiStringLiteral])) + ) + ), + EnumVariant( + name = Deconstruction, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + open_brace = Required(Terminal([OpenBrace])), + fields = Required(NonTerminal(ImportDeconstructionFields)), + close_brace = Required(Terminal([CloseBrace])), + from_keyword = Required(Terminal([FromKeyword])), + path = Required(Terminal([AsciiStringLiteral])) + ) + ) + ] + ), + Separated( + name = ImportDeconstructionFields, + separated = ImportDeconstructionField, + separator = Comma, + allow_empty = true + ), + Struct( + name = ImportDeconstructionField, + fields = ( + name = Required(Terminal([Identifier])), + alias = Optional(kind = NonTerminal(ImportAlias)) + ) + ), + Struct( + name = ImportAlias, + fields = ( + as_keyword = Required(Terminal([AsKeyword])), + identifier = Required(Terminal([Identifier])) + ) + ) + ] + ), + Topic( + title = "Using Directives", + items = [ + Struct( + name = UsingDirective, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + using_keyword = Required(Terminal([UsingKeyword])), + symbol = Required(NonTerminal(UsingSymbol)), + for_keyword = Required(Terminal([ForKeyword])), + target = Required(NonTerminal(UsingTarget)), + global_keyword = Optional( + kind = Terminal([GlobalKeyword]), + enabled_in = "0.8.13" + ), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Enum( + name = UsingSymbol, + default_variant = Path, + variants = [ + EnumVariant( + name = Path, + fields = (path = Required(NonTerminal(IdentifierPath))) + ), + EnumVariant( + name = Deconstruction, + enabled_in = "0.8.13", + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + open_brace = Required(Terminal([OpenBrace])), + fields = Required(NonTerminal(UsingDeconstructionFields)), + close_brace = Required(Terminal([CloseBrace])) + ) + ) + ] + ), + Separated( + name = UsingDeconstructionFields, + separated = UsingDeconstructionField, + separator = Comma, + enabled_in = "0.8.13", + allow_empty = true + ), + Struct( + name = UsingDeconstructionField, + enabled_in = "0.8.13", + fields = ( + name = Required(NonTerminal(IdentifierPath)), + alias = + Optional(kind = NonTerminal(UsingAlias), enabled_in = "0.8.19") + ) + ), + Struct( + name = UsingAlias, + enabled_in = "0.8.19", + fields = ( + as_keyword = Required(Terminal([AsKeyword])), + operator = Required(Terminal([ + Ampersand, + Asterisk, + BangEqual, + Bar, + Caret, + EqualEqual, + GreaterThan, + GreaterThanEqual, + LessThan, + LessThanEqual, + Minus, + Percent, + Plus, + Slash, + Tilde + ])) + ) + ), + Enum( + name = UsingTarget, + default_variant = Asterisk, + variants = [ + EnumVariant( + name = TypeName, + fields = (type_name = Required(NonTerminal(TypeName))) + ), + EnumVariant( + name = Asterisk, + fields = (asterisk = Required(Terminal([Asterisk]))) + ) + ] + ) + ] + ), + Topic( + title = "Trivia", + items = [ + Trivia( + name = Whitespace, + scanner = OneOrMore(Choice([Atom(" "), Atom("\t")])) + ), + Trivia( + name = EndOfLine, + scanner = Sequence([Optional(Atom("\r")), Atom("\n")]) + ), + Trivia( + name = MultilineComment, + scanner = Sequence([ + Atom("/"), + Atom("*"), + ZeroOrMore(Choice([ + Not(['*']), + TrailingContext( + scanner = Atom("*"), + not_followed_by = Atom("/") + ) + ])), + Atom("*"), + Atom("/") + ]) + ), + Trivia( + name = SingleLineComment, + scanner = Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))]) + ) + ] + ), + Topic( + title = "Keywords", + items = [ + Keyword( + name = AbicoderKeyword, + identifier = Identifier, + value = Atom("abicoder") + ), + Keyword( + name = AbstractKeyword, + identifier = Identifier, + enabled_in = "0.6.0", + value = Atom("abstract") + ), + Keyword( + name = AddressKeyword, + identifier = Identifier, + value = Atom("address") + ), + Keyword( + name = AfterKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("after") + ), + Keyword( + name = AliasKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("alias") + ), + Keyword( + name = AnonymousKeyword, + identifier = Identifier, + value = Atom("anonymous") + ), + Keyword( + name = ApplyKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("apply") + ), + Keyword( + name = AsKeyword, + identifier = Identifier, + value = Atom("as") + ), + Keyword( + name = AssemblyKeyword, + identifier = Identifier, + value = Atom("assembly") + ), + Keyword( + name = AutoKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("auto") + ), + Keyword( + name = BoolKeyword, + identifier = Identifier, + value = Atom("bool") + ), + Keyword( + name = BreakKeyword, + identifier = Identifier, + value = Atom("break") + ), + Keyword( + name = ByteKeyword, + identifier = Identifier, + disabled_in = "0.8.0", + value = Atom("byte") + ), + Keyword( + name = BytesKeyword, + identifier = Identifier, + value = Sequence([ + Atom("bytes"), + Range(inclusive_start = 1, inclusive_end = 32, increment = 1) + ]) + ), + Keyword( + name = CallDataKeyword, + identifier = Identifier, + enabled_in = "0.5.0", + value = Atom("calldata") + ), + Keyword( + name = CaseKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("case") + ), + Keyword( + name = CatchKeyword, + identifier = Identifier, + enabled_in = "0.6.0", + value = Atom("catch") + ), + Keyword( + name = ConstantKeyword, + identifier = Identifier, + value = Atom("constant") + ), + Keyword( + name = ConstructorKeyword, + identifier = Identifier, + enabled_in = "0.4.22", + value = Atom("constructor") + ), + Keyword( + name = ContinueKeyword, + identifier = Identifier, + value = Atom("continue") + ), + Keyword( + name = ContractKeyword, + identifier = Identifier, + value = Atom("contract") + ), + Keyword( + name = CopyOfKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("copyof") + ), + Keyword( + name = DaysKeyword, + identifier = Identifier, + value = Atom("days") + ), + Keyword( + name = DefaultKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("default") + ), + Keyword( + name = DefineKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("define") + ), + Keyword( + name = DeleteKeyword, + identifier = Identifier, + value = Atom("delete") + ), + Keyword( + name = DoKeyword, + identifier = Identifier, + value = Atom("do") + ), + Keyword( + name = ElseKeyword, + identifier = Identifier, + value = Atom("else") + ), + Keyword( + name = EmitKeyword, + identifier = Identifier, + enabled_in = "0.4.21", + value = Atom("emit") + ), + Keyword( + name = EnumKeyword, + identifier = Identifier, + value = Atom("enum") + ), + Keyword( + name = ErrorKeyword, + identifier = Identifier, + enabled_in = "0.8.4", + value = Atom("error") + ), + Keyword( + name = EtherKeyword, + identifier = Identifier, + value = Atom("ether") + ), + Keyword( + name = EventKeyword, + identifier = Identifier, + value = Atom("event") + ), + Keyword( + name = ExperimentalKeyword, + identifier = Identifier, + value = Atom("experimental") + ), + Keyword( + name = ExternalKeyword, + identifier = Identifier, + value = Atom("external") + ), + Keyword( + name = FallbackKeyword, + identifier = Identifier, + value = Atom("fallback") + ), + Keyword( + name = FalseKeyword, + identifier = Identifier, + value = Atom("false") + ), + Keyword( + name = FinalKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("final") + ), + Keyword( + name = FinneyKeyword, + identifier = Identifier, + disabled_in = "0.7.0", + value = Atom("finney") + ), + Keyword( + name = FixedKeyword, + identifier = Identifier, + value = Sequence([ + Atom("fixed"), + Optional(Sequence([ + Range(inclusive_start = 8, inclusive_end = 256, increment = 8), + Atom("x"), + Range(inclusive_start = 0, inclusive_end = 80, increment = 1) + ])) + ]) + ), + Keyword( + name = ForKeyword, + identifier = Identifier, + value = Atom("for") + ), + Keyword( + name = FromKeyword, + identifier = Identifier, + value = Atom("from") + ), + Keyword( + name = FunctionKeyword, + identifier = Identifier, + value = Atom("function") + ), + Keyword( + name = GlobalKeyword, + identifier = Identifier, + enabled_in = "0.8.13", + value = Atom("global") + ), + Keyword( + name = GweiKeyword, + identifier = Identifier, + enabled_in = "0.6.11", + value = Atom("gwei") + ), + Keyword( + name = HexKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("hex") + ), + Keyword( + name = HoursKeyword, + identifier = Identifier, + value = Atom("hours") + ), + Keyword( + name = IfKeyword, + identifier = Identifier, + value = Atom("if") + ), + Keyword( + name = ImmutableKeyword, + identifier = Identifier, + enabled_in = "0.6.5", + value = Atom("immutable") + ), + Keyword( + name = ImplementsKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("implements") + ), + Keyword( + name = ImportKeyword, + identifier = Identifier, + value = Atom("import") + ), + Keyword( + name = IndexedKeyword, + identifier = Identifier, + value = Atom("indexed") + ), + Keyword( + name = InKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("in") + ), + Keyword( + name = InlineKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("inline") + ), + Keyword( + name = InterfaceKeyword, + identifier = Identifier, + value = Atom("interface") + ), + Keyword( + name = InternalKeyword, + identifier = Identifier, + value = Atom("internal") + ), + Keyword( + name = IntKeyword, + identifier = Identifier, + value = Sequence([ + Atom("int"), + Optional(Range( + inclusive_start = 8, + inclusive_end = 256, + increment = 8 + )) + ]) + ), + Keyword( + name = IsKeyword, + identifier = Identifier, + value = Atom("is") + ), + Keyword( + name = LetKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("let") + ), + Keyword( + name = LibraryKeyword, + identifier = Identifier, + value = Atom("library") + ), + Keyword( + name = MacroKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("macro") + ), + Keyword( + name = MappingKeyword, + identifier = Identifier, + value = Atom("mapping") + ), + Keyword( + name = MatchKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("match") + ), + Keyword( + name = MemoryKeyword, + identifier = Identifier, + value = Atom("memory") + ), + Keyword( + name = MinutesKeyword, + identifier = Identifier, + value = Atom("minutes") + ), + Keyword( + name = ModifierKeyword, + identifier = Identifier, + value = Atom("modifier") + ), + Keyword( + name = MutableKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("mutable") + ), + Keyword( + name = NewKeyword, + identifier = Identifier, + value = Atom("new") + ), + Keyword( + name = NullKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("null") + ), + Keyword( + name = OfKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("of") + ), + Keyword( + name = OverrideKeyword, + identifier = Identifier, + value = Atom("override") + ), + Keyword( + name = PartialKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("partial") + ), + Keyword( + name = PayableKeyword, + identifier = Identifier, + value = Atom("payable") + ), + Keyword( + name = PragmaKeyword, + identifier = Identifier, + value = Atom("pragma") + ), + Keyword( + name = PrivateKeyword, + identifier = Identifier, + value = Atom("private") + ), + Keyword( + name = PromiseKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("promise") + ), + Keyword( + name = PublicKeyword, + identifier = Identifier, + value = Atom("public") + ), + Keyword( + name = PureKeyword, + identifier = Identifier, + value = Atom("pure") + ), + Keyword( + name = ReceiveKeyword, + identifier = Identifier, + value = Atom("receive") + ), + Keyword( + name = ReferenceKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("reference") + ), + Keyword( + name = RelocatableKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("relocatable") + ), + Keyword( + name = ReturnKeyword, + identifier = Identifier, + value = Atom("return") + ), + Keyword( + name = ReturnsKeyword, + identifier = Identifier, + value = Atom("returns") + ), + Keyword( + name = RevertKeyword, + identifier = Identifier, + enabled_in = "0.8.4", + value = Atom("revert") + ), + Keyword( + name = SealedKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("sealed") + ), + Keyword( + name = SecondsKeyword, + identifier = Identifier, + value = Atom("seconds") + ), + Keyword( + name = SizeOfKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("sizeof") + ), + Keyword( + name = SolidityKeyword, + identifier = Identifier, + value = Atom("solidity") + ), + Keyword( + name = StaticKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("static") + ), + Keyword( + name = StorageKeyword, + identifier = Identifier, + value = Atom("storage") + ), + Keyword( + name = StringKeyword, + identifier = Identifier, + value = Atom("string") + ), + Keyword( + name = StructKeyword, + identifier = Identifier, + value = Atom("struct") + ), + Keyword( + name = SupportsKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("supports") + ), + Keyword( + name = SwitchKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("switch") + ), + Keyword( + name = SzaboKeyword, + identifier = Identifier, + disabled_in = "0.7.0", + value = Atom("szabo") + ), + Keyword( + name = ThrowKeyword, + identifier = Identifier, + disabled_in = "0.5.0", + value = Atom("throw") + ), + Keyword( + name = TrueKeyword, + identifier = Identifier, + value = Atom("true") + ), + Keyword( + name = TryKeyword, + identifier = Identifier, + enabled_in = "0.6.0", + value = Atom("try") + ), + Keyword( + name = TypeDefKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + reserved_in = "0.5.0", + value = Atom("typedef") + ), + Keyword( + name = TypeKeyword, + identifier = Identifier, + enabled_in = "0.5.3", + value = Atom("type") + ), + Keyword( + name = TypeOfKeyword, + identifier = Identifier, + disabled_in = "0.4.11", + value = Atom("typeof") + ), + Keyword( + name = UfixedKeyword, + identifier = Identifier, + value = Sequence([ + Atom("ufixed"), + Optional(Sequence([ + Range(inclusive_start = 8, inclusive_end = 256, increment = 8), + Atom("x"), + Range(inclusive_start = 0, inclusive_end = 80, increment = 1) + ])) + ]) + ), + Keyword( + name = UintKeyword, + identifier = Identifier, + value = Sequence([ + Atom("uint"), + Optional(Range( + inclusive_start = 8, + inclusive_end = 256, + increment = 8 + )) + ]) + ), + Keyword( + name = UncheckedKeyword, + identifier = Identifier, + enabled_in = "0.8.0", + value = Atom("unchecked") + ), + Keyword( + name = UsingKeyword, + identifier = Identifier, + value = Atom("using") + ), + Keyword( + name = VarKeyword, + identifier = Identifier, + disabled_in = "0.5.0", + value = Atom("var") + ), + Keyword( + name = ViewKeyword, + identifier = Identifier, + value = Atom("view") + ), + Keyword( + name = VirtualKeyword, + identifier = Identifier, + enabled_in = "0.6.0", + value = Atom("virtual") + ), + Keyword( + name = WeeksKeyword, + identifier = Identifier, + value = Atom("weeks") + ), + Keyword( + name = WeiKeyword, + identifier = Identifier, + value = Atom("wei") + ), + Keyword( + name = WhileKeyword, + identifier = Identifier, + value = Atom("while") + ), + Keyword( + name = YearsKeyword, + identifier = Identifier, + disabled_in = "0.5.0", + value = Atom("years") + ) + ] + ), + Topic( + title = "Punctuation", + items = [ + Token( + name = OpenParen, + definitions = [TokenDefinition(scanner = Atom("("))] + ), + Token( + name = CloseParen, + definitions = [TokenDefinition(scanner = Atom(")"))] + ), + Token( + name = OpenBracket, + definitions = [TokenDefinition(scanner = Atom("["))] + ), + Token( + name = CloseBracket, + definitions = [TokenDefinition(scanner = Atom("]"))] + ), + Token( + name = OpenBrace, + definitions = [TokenDefinition(scanner = Atom("{"))] + ), + Token( + name = CloseBrace, + definitions = [TokenDefinition(scanner = Atom("}"))] + ), + Token( + name = Comma, + definitions = [TokenDefinition(scanner = Atom(","))] + ), + Token( + name = Period, + definitions = [TokenDefinition(scanner = Atom("."))] + ), + Token( + name = QuestionMark, + definitions = [TokenDefinition(scanner = Atom("?"))] + ), + Token( + name = Semicolon, + definitions = [TokenDefinition(scanner = Atom(";"))] + ), + Token( + name = Colon, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom(":"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = ColonEqual, + definitions = [TokenDefinition(scanner = Atom(":="))] + ), + Token( + name = Equal, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("="), + not_followed_by = Choice([Atom("="), Atom(">")]) + ) + )] + ), + Token( + name = EqualEqual, + definitions = [TokenDefinition(scanner = Atom("=="))] + ), + Token( + name = EqualGreaterThan, + definitions = [TokenDefinition(scanner = Atom("=>"))] + ), + Token( + name = Asterisk, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("*"), + not_followed_by = Choice([Atom("="), Atom("*")]) + ) + )] + ), + Token( + name = AsteriskEqual, + definitions = [TokenDefinition(scanner = Atom("*="))] + ), + Token( + name = AsteriskAsterisk, + definitions = [TokenDefinition(scanner = Atom("**"))] + ), + Token( + name = Bar, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("|"), + not_followed_by = Choice([Atom("="), Atom("|")]) + ) + )] + ), + Token( + name = BarEqual, + definitions = [TokenDefinition(scanner = Atom("|="))] + ), + Token( + name = BarBar, + definitions = [TokenDefinition(scanner = Atom("||"))] + ), + Token( + name = Ampersand, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("&"), + not_followed_by = Choice([Atom("="), Atom("&")]) + ) + )] + ), + Token( + name = AmpersandEqual, + definitions = [TokenDefinition(scanner = Atom("&="))] + ), + Token( + name = AmpersandAmpersand, + definitions = [TokenDefinition(scanner = Atom("&&"))] + ), + Token( + name = LessThan, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("<"), + not_followed_by = Choice([Atom("="), Atom("<")]) + ) + )] + ), + Token( + name = LessThanEqual, + definitions = [TokenDefinition(scanner = Atom("<="))] + ), + Token( + name = LessThanLessThan, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("<<"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = LessThanLessThanEqual, + definitions = [TokenDefinition(scanner = Atom("<<="))] + ), + Token( + name = GreaterThan, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom(">"), + not_followed_by = Choice([Atom("="), Atom(">")]) + ) + )] + ), + Token( + name = GreaterThanEqual, + definitions = [TokenDefinition(scanner = Atom(">="))] + ), + Token( + name = GreaterThanGreaterThan, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom(">>"), + not_followed_by = Choice([Atom("="), Atom(">")]) + ) + )] + ), + Token( + name = GreaterThanGreaterThanEqual, + definitions = [TokenDefinition(scanner = Atom(">>="))] + ), + Token( + name = GreaterThanGreaterThanGreaterThan, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom(">>>"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = GreaterThanGreaterThanGreaterThanEqual, + definitions = [TokenDefinition(scanner = Atom(">>>="))] + ), + Token( + name = Plus, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("+"), + not_followed_by = Choice([Atom("="), Atom("+")]) + ) + )] + ), + Token( + name = PlusEqual, + definitions = [TokenDefinition(scanner = Atom("+="))] + ), + Token( + name = PlusPlus, + definitions = [TokenDefinition(scanner = Atom("++"))] + ), + Token( + name = Minus, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("-"), + not_followed_by = Choice([Atom("="), Atom("-"), Atom(">")]) + ) + )] + ), + Token( + name = MinusEqual, + definitions = [TokenDefinition(scanner = Atom("-="))] + ), + Token( + name = MinusMinus, + definitions = [TokenDefinition(scanner = Atom("--"))] + ), + Token( + name = MinusGreaterThan, + definitions = [TokenDefinition(scanner = Atom("->"))] + ), + Token( + name = Slash, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("/"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = SlashEqual, + definitions = [TokenDefinition(scanner = Atom("/="))] + ), + Token( + name = Percent, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("%"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = PercentEqual, + definitions = [TokenDefinition(scanner = Atom("%="))] + ), + Token( + name = Bang, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("!"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = BangEqual, + definitions = [TokenDefinition(scanner = Atom("!="))] + ), + Token( + name = Caret, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Atom("^"), + not_followed_by = Atom("=") + ) + )] + ), + Token( + name = CaretEqual, + definitions = [TokenDefinition(scanner = Atom("^="))] + ), + Token( + name = Tilde, + definitions = [TokenDefinition(scanner = Atom("~"))] + ) + ] + ) + ] + ), + Section( + title = "Definitions", + topics = [ + Topic( + title = "Contracts", + items = [ + Struct( + name = ContractDefinition, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + abstract_keyword = Optional( + kind = Terminal([AbstractKeyword]), + enabled_in = "0.6.0" + ), + contract_keyword = Required(Terminal([ContractKeyword])), + name = Required(Terminal([Identifier])), + inheritence = Optional(kind = NonTerminal(InheritanceSpecifier)), + open_brace = Required(Terminal([OpenBrace])), + members = Required(NonTerminal(ContractMembers)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Struct( + name = InheritanceSpecifier, + fields = ( + is_keyword = Required(Terminal([IsKeyword])), + types = Required(NonTerminal(InheritanceTypes)) + ) + ), + Separated( + name = InheritanceTypes, + separated = InheritanceType, + separator = Comma + ), + Struct( + name = InheritanceType, + fields = ( + type_name = Required(NonTerminal(IdentifierPath)), + arguments = Optional(kind = NonTerminal(ArgumentsDeclaration)) + ) + ), + Repeated( + name = ContractMembers, + repeated = ContractMember, + allow_empty = true + ), + Enum( + name = ContractMember, + default_variant = StateVariable, + variants = [ + EnumVariant( + name = Using, + fields = (directive = Required(NonTerminal(UsingDirective))) + ), + EnumVariant( + name = Function, + fields = + (definition = Required(NonTerminal(FunctionDefinition))) + ), + EnumVariant( + name = Constructor, + enabled_in = "0.4.22", + fields = + (definition = Required(NonTerminal(ConstructorDefinition))) + ), + EnumVariant( + name = ReceiveFunction, + enabled_in = "0.6.0", + fields = (definition = + Required(NonTerminal(ReceiveFunctionDefinition))) + ), + EnumVariant( + name = FallbackFunction, + enabled_in = "0.6.0", + fields = (definition = + Required(NonTerminal(FallbackFunctionDefinition))) + ), + EnumVariant( + name = UnnamedFunction, + disabled_in = "0.6.0", + fields = (definition = + Required(NonTerminal(UnnamedFunctionDefinition))) + ), + EnumVariant( + name = Modifier, + fields = + (definition = Required(NonTerminal(ModifierDefinition))) + ), + EnumVariant( + name = Struct, + fields = (definition = Required(NonTerminal(StructDefinition))) + ), + EnumVariant( + name = Enum, + fields = (definition = Required(NonTerminal(EnumDefinition))) + ), + EnumVariant( + name = Event, + fields = (definition = Required(NonTerminal(EventDefinition))) + ), + EnumVariant( + name = StateVariable, + fields = (definition = + Required(NonTerminal(StateVariableDefinition))) + ), + EnumVariant( + name = Error, + enabled_in = "0.8.4", + fields = (definition = Required(NonTerminal(ErrorDefinition))) + ), + EnumVariant( + name = UserDefinedValueType, + enabled_in = "0.8.8", + fields = (definition = + Required(NonTerminal(UserDefinedValueTypeDefinition))) + ) + ] + ) + ] + ), + Topic( + title = "Interfaces", + items = [ + Struct( + name = InterfaceDefinition, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + interface_keyword = Required(Terminal([InterfaceKeyword])), + name = Required(Terminal([Identifier])), + inheritence = Optional(kind = NonTerminal(InheritanceSpecifier)), + open_brace = Required(Terminal([OpenBrace])), + members = Required(NonTerminal(InterfaceMembers)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Repeated( + name = InterfaceMembers, + repeated = ContractMember, + allow_empty = true + ) + ] + ), + Topic( + title = "Libraries", + items = [ + Struct( + name = LibraryDefinition, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + library_keyword = Required(Terminal([LibraryKeyword])), + name = Required(Terminal([Identifier])), + open_brace = Required(Terminal([OpenBrace])), + members = Required(NonTerminal(LibraryMembers)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Repeated( + name = LibraryMembers, + repeated = ContractMember, + allow_empty = true + ) + ] + ), + Topic( + title = "Structs", + items = [ + Struct( + name = StructDefinition, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + struct_keyword = Required(Terminal([StructKeyword])), + name = Required(Terminal([Identifier])), + open_brace = Required(Terminal([OpenBrace])), + members = Required(NonTerminal(StructMembers)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Repeated( + name = StructMembers, + repeated = StructMember, + allow_empty = true + ), + Struct( + name = StructMember, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + type_name = Required(NonTerminal(TypeName)), + name = Required(Terminal([Identifier])), + semicolon = Required(Terminal([Semicolon])) + ) + ) + ] + ), + Topic( + title = "Enums", + items = [ + Struct( + name = EnumDefinition, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + enum_keyword = Required(Terminal([EnumKeyword])), + name = Required(Terminal([Identifier])), + open_brace = Required(Terminal([OpenBrace])), + members = Required(NonTerminal(EnumMembers)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Separated( + name = EnumMembers, + separated = Identifier, + separator = Comma, + allow_empty = true + ) + ] + ), + Topic( + title = "Constants", + items = [Struct( + name = ConstantDefinition, + enabled_in = "0.7.4", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + type_name = Required(NonTerminal(TypeName)), + constant_keyword = Required(Terminal([ConstantKeyword])), + name = Required(Terminal([Identifier])), + equal = Required(Terminal([Equal])), + value = Required(NonTerminal(Expression)), + semicolon = Required(Terminal([Semicolon])) + ) + )] + ), + Topic( + title = "State Variables", + items = [ + Struct( + name = StateVariableDefinition, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + type_name = Required(NonTerminal(TypeName)), + attributes = Required(NonTerminal(StateVariableAttributes)), + name = Required(Terminal([Identifier])), + value = Optional(kind = NonTerminal(StateVariableDefinitionValue)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = StateVariableDefinitionValue, + fields = ( + equal = Required(Terminal([Equal])), + value = Required(NonTerminal(Expression)) + ) + ), + Repeated( + name = StateVariableAttributes, + repeated = StateVariableAttribute, + allow_empty = true + ), + Enum( + name = StateVariableAttribute, + default_variant = Public, + variants = [ + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = Constant, + fields = (keyword = Required(Terminal([ConstantKeyword]))) + ), + EnumVariant( + name = Internal, + fields = (keyword = Required(Terminal([InternalKeyword]))) + ), + EnumVariant( + name = Private, + fields = (keyword = Required(Terminal([PrivateKeyword]))) + ), + EnumVariant( + name = Public, + fields = (keyword = Required(Terminal([PublicKeyword]))) + ), + EnumVariant( + name = Immutable, + enabled_in = "0.6.5", + fields = (keyword = Required(Terminal([ImmutableKeyword]))) + ) + ] + ) + ] + ), + Topic( + title = "Functions", + items = [ + Struct( + name = FunctionDefinition, + fields = ( + function_keyword = Required(Terminal([FunctionKeyword])), + name = Required(Terminal([ + Identifier, + FallbackKeyword, + ReceiveKeyword + ])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(FunctionAttributes)), + returns = Optional(kind = NonTerminal(ReturnsDeclaration)), + body = Required(NonTerminal(FunctionBody)) + ) + ), + Struct( + name = ParametersDeclaration, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + parameters = Required(NonTerminal(Parameters)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = Parameters, + separated = Parameter, + separator = Comma, + allow_empty = true + ), + Struct( + name = Parameter, + fields = ( + type_name = Required(NonTerminal(TypeName)), + storage_location = Optional(kind = NonTerminal(StorageLocation)), + name = Optional(kind = Terminal([Identifier])) + ) + ), + Repeated( + name = FunctionAttributes, + repeated = FunctionAttribute, + allow_empty = true + ), + Enum( + name = FunctionAttribute, + default_variant = Public, + variants = [ + EnumVariant( + name = Modifier, + fields = (modifier = Required(NonTerminal(ModifierInvocation))) + ), + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = Constant, + disabled_in = "0.5.0", + fields = (keyword = Required(Terminal([ConstantKeyword]))) + ), + EnumVariant( + name = External, + fields = (keyword = Required(Terminal([ExternalKeyword]))) + ), + EnumVariant( + name = Internal, + fields = (keyword = Required(Terminal([InternalKeyword]))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ), + EnumVariant( + name = Private, + fields = (keyword = Required(Terminal([PrivateKeyword]))) + ), + EnumVariant( + name = Public, + fields = (keyword = Required(Terminal([PublicKeyword]))) + ), + EnumVariant( + name = Pure, + fields = (keyword = Required(Terminal([PureKeyword]))) + ), + EnumVariant( + name = View, + fields = (keyword = Required(Terminal([ViewKeyword]))) + ), + EnumVariant( + name = Virtual, + enabled_in = "0.6.0", + fields = (keyword = Required(Terminal([VirtualKeyword]))) + ) + ] + ), + Struct( + name = OverrideSpecifier, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + override_keyword = Required(Terminal([OverrideKeyword])), + open_paren = Required(Terminal([OpenParen])), + overridden = Required(NonTerminal(OverridePaths)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = OverridePaths, + separated = IdentifierPath, + separator = Comma + ), + Struct( + name = ReturnsDeclaration, + fields = ( + returns_keyword = Required(Terminal([ReturnsKeyword])), + variables = Required(NonTerminal(ParametersDeclaration)) + ) + ), + Enum( + name = FunctionBody, + default_variant = Semicolon, + variants = [ + EnumVariant( + name = Block, + fields = (block = Required(NonTerminal(Block))) + ), + EnumVariant( + name = Semicolon, + fields = (semicolon = Required(Terminal([Semicolon]))) + ) + ] + ), + Struct( + name = ConstructorDefinition, + enabled_in = "0.4.22", + fields = ( + constructor_keyword = Required(Terminal([ConstructorKeyword])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(ConstructorAttributes)), + body = Required(NonTerminal(Block)) + ) + ), + Repeated( + name = ConstructorAttributes, + repeated = ConstructorAttribute, + enabled_in = "0.4.22", + allow_empty = true + ), + Enum( + name = ConstructorAttribute, + enabled_in = "0.4.22", + default_variant = Public, + variants = [ + EnumVariant( + name = Modifier, + fields = (modifier = Required(NonTerminal(ModifierInvocation))) + ), + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ), + EnumVariant( + name = Public, + fields = (keyword = Required(Terminal([PublicKeyword]))) + ) + ] + ), + Struct( + name = UnnamedFunctionDefinition, + disabled_in = "0.6.0", + fields = ( + function_keyword = Required(Terminal([FunctionKeyword])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(UnnamedFunctionAttributes)), + body = Required(NonTerminal(FunctionBody)) + ) + ), + Repeated( + name = UnnamedFunctionAttributes, + repeated = UnnamedFunctionAttribute, + disabled_in = "0.6.0", + allow_empty = true + ), + Enum( + name = UnnamedFunctionAttribute, + disabled_in = "0.6.0", + default_variant = View, + variants = [ + EnumVariant( + name = Modifier, + fields = (modifier = Required(NonTerminal(ModifierInvocation))) + ), + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = External, + fields = (keyword = Required(Terminal([ExternalKeyword]))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ), + EnumVariant( + name = Pure, + fields = (keyword = Required(Terminal([PureKeyword]))) + ), + EnumVariant( + name = View, + fields = (keyword = Required(Terminal([ViewKeyword]))) + ) + ] + ), + Struct( + name = FallbackFunctionDefinition, + enabled_in = "0.6.0", + fields = ( + fallback_keyword = Required(Terminal([FallbackKeyword])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(FallbackFunctionAttributes)), + returns = Optional(kind = NonTerminal(ReturnsDeclaration)), + body = Required(NonTerminal(FunctionBody)) + ) + ), + Repeated( + name = FallbackFunctionAttributes, + repeated = FallbackFunctionAttribute, + enabled_in = "0.6.0", + allow_empty = true + ), + Enum( + name = FallbackFunctionAttribute, + enabled_in = "0.6.0", + default_variant = View, + variants = [ + EnumVariant( + name = Modifier, + fields = (modifier = Required(NonTerminal(ModifierInvocation))) + ), + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = External, + fields = (keyword = Required(Terminal([ExternalKeyword]))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ), + EnumVariant( + name = Pure, + fields = (keyword = Required(Terminal([PureKeyword]))) + ), + EnumVariant( + name = View, + fields = (keyword = Required(Terminal([ViewKeyword]))) + ), + EnumVariant( + name = Virtual, + fields = (keyword = Required(Terminal([VirtualKeyword]))) + ) + ] + ), + Struct( + name = ReceiveFunctionDefinition, + enabled_in = "0.6.0", + fields = ( + receive_keyword = Required(Terminal([ReceiveKeyword])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(ReceiveFunctionAttributes)), + body = Required(NonTerminal(FunctionBody)) + ) + ), + Repeated( + name = ReceiveFunctionAttributes, + repeated = ReceiveFunctionAttribute, + enabled_in = "0.6.0", + allow_empty = true + ), + Enum( + name = ReceiveFunctionAttribute, + enabled_in = "0.6.0", + default_variant = Virtual, + variants = [ + EnumVariant( + name = Modifier, + fields = (modifier = Required(NonTerminal(ModifierInvocation))) + ), + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = External, + fields = (keyword = Required(Terminal([ExternalKeyword]))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ), + EnumVariant( + name = Virtual, + fields = (keyword = Required(Terminal([VirtualKeyword]))) + ) + ] + ) + ] + ), + Topic( + title = "Modifiers", + items = [ + Struct( + name = ModifierDefinition, + fields = ( + modifier_keyword = Required(Terminal([ModifierKeyword])), + name = Required(Terminal([Identifier])), + parameters = Optional(kind = NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(ModifierAttributes)), + body = Required(NonTerminal(FunctionBody)) + ) + ), + Repeated( + name = ModifierAttributes, + repeated = ModifierAttribute, + allow_empty = true + ), + Enum( + name = ModifierAttribute, + default_variant = Virtual, + variants = [ + EnumVariant( + name = Override, + fields = (specifier = Required(NonTerminal(OverrideSpecifier))) + ), + EnumVariant( + name = Virtual, + enabled_in = "0.6.0", + fields = (keyword = Required(Terminal([VirtualKeyword]))) + ) + ] + ), + Struct( + name = ModifierInvocation, + fields = ( + name = Required(NonTerminal(IdentifierPath)), + arguments = Optional(kind = NonTerminal(ArgumentsDeclaration)) + ) + ) + ] + ), + Topic( + title = "Events", + items = [ + Struct( + name = EventDefinition, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + event_keyword = Required(Terminal([EventKeyword])), + name = Required(Terminal([Identifier])), + parameters = + Optional(kind = NonTerminal(EventParametersDeclaration)), + anonymous_keyword = Optional(kind = Terminal([AnonymousKeyword])), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = EventParametersDeclaration, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + parameters = Required(NonTerminal(EventParameters)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = EventParameters, + separated = EventParameter, + separator = Comma, + allow_empty = true + ), + Struct( + name = EventParameter, + fields = ( + type_name = Required(NonTerminal(TypeName)), + indexed_keyword = Optional(kind = Terminal([IndexedKeyword])), + name = Optional(kind = Terminal([Identifier])) + ) + ) + ] + ), + Topic( + title = "User Defined Value Types", + items = [Struct( + name = UserDefinedValueTypeDefinition, + enabled_in = "0.8.8", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + type_keyword = Required(Terminal([TypeKeyword])), + name = Required(Terminal([Identifier])), + is_keyword = Required(Terminal([IsKeyword])), + value_type = Required(NonTerminal(ElementaryType)), + semicolon = Required(Terminal([Semicolon])) + ) + )] + ), + Topic( + title = "Errors", + items = [ + Struct( + name = ErrorDefinition, + enabled_in = "0.8.4", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + error_keyword = Required(Terminal([ErrorKeyword])), + name = Required(Terminal([Identifier])), + members = Required(NonTerminal(ErrorParametersDeclaration)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = ErrorParametersDeclaration, + enabled_in = "0.8.4", + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + parameters = Required(NonTerminal(ErrorParameters)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = ErrorParameters, + separated = ErrorParameter, + separator = Comma, + enabled_in = "0.8.4", + allow_empty = true + ), + Struct( + name = ErrorParameter, + enabled_in = "0.8.4", + fields = ( + type_name = Required(NonTerminal(TypeName)), + name = Optional(kind = Terminal([Identifier])) + ) + ) + ] + ) + ] + ), + Section( + title = "Types", + topics = [ + Topic( + title = "Advanced Types", + items = [ + Precedence( + name = TypeName, + precedence_expressions = [PrecedenceExpression( + name = ArrayTypeName, + operators = [PrecedenceOperator( + model = Postfix, + error_recovery = FieldsErrorRecovery( + delimiters = FieldDelimiters( + open = open_bracket, + close = close_bracket + ) + ), + fields = ( + open_bracket = Required(Terminal([OpenBracket])), + index = Optional(kind = NonTerminal(Expression)), + close_bracket = Required(Terminal([CloseBracket])) + ) + )] + )], + default_primary_expression = ElementaryType, + primary_expressions = [ + PrimaryExpression(expression = FunctionType), + PrimaryExpression(expression = MappingType), + PrimaryExpression(expression = ElementaryType), + PrimaryExpression(expression = IdentifierPath) + ] + ), + Struct( + name = FunctionType, + fields = ( + function_keyword = Required(Terminal([FunctionKeyword])), + parameters = Required(NonTerminal(ParametersDeclaration)), + attributes = Required(NonTerminal(FunctionTypeAttributes)), + returns = Optional(kind = NonTerminal(ReturnsDeclaration)) + ) + ), + Repeated( + name = FunctionTypeAttributes, + repeated = FunctionTypeAttribute, + allow_empty = true + ), + Enum( + name = FunctionTypeAttribute, + default_variant = Public, + variants = [ + EnumVariant( + name = Internal, + fields = (keyword = Required(Terminal([InternalKeyword]))) + ), + EnumVariant( + name = External, + fields = (keyword = Required(Terminal([ExternalKeyword]))) + ), + EnumVariant( + name = Private, + fields = (keyword = Required(Terminal([PrivateKeyword]))) + ), + EnumVariant( + name = Public, + fields = (keyword = Required(Terminal([PublicKeyword]))) + ), + EnumVariant( + name = Pure, + fields = (keyword = Required(Terminal([PureKeyword]))) + ), + EnumVariant( + name = View, + fields = (keyword = Required(Terminal([ViewKeyword]))) + ), + EnumVariant( + name = Payable, + fields = (keyword = Required(Terminal([PayableKeyword]))) + ) + ] + ), + Struct( + name = MappingType, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + mapping_keyword = Required(Terminal([MappingKeyword])), + open_paren = Required(Terminal([OpenParen])), + key_type = Required(NonTerminal(MappingKey)), + equal_greater_than = Required(Terminal([EqualGreaterThan])), + value_type = Required(NonTerminal(MappingValue)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Struct( + name = MappingKey, + fields = ( + key_type = Required(NonTerminal(MappingKeyType)), + name = + Optional(kind = Terminal([Identifier]), enabled_in = "0.8.18") + ) + ), + Enum( + name = MappingKeyType, + default_variant = ElementaryType, + variants = [ + EnumVariant( + name = ElementaryType, + fields = (type_name = Required(NonTerminal(ElementaryType))) + ), + EnumVariant( + name = IdentifierPath, + fields = (type_name = Required(NonTerminal(IdentifierPath))) + ) + ] + ), + Struct( + name = MappingValue, + fields = ( + type_name = Required(NonTerminal(TypeName)), + name = + Optional(kind = Terminal([Identifier]), enabled_in = "0.8.18") + ) + ) + ] + ), + Topic( + title = "Elementary Types", + items = [ + Enum( + name = ElementaryType, + default_variant = Bool, + variants = [ + EnumVariant( + name = Bool, + fields = (type_name = Required(Terminal([BoolKeyword]))) + ), + EnumVariant( + name = Byte, + disabled_in = "0.8.0", + fields = (type_name = Required(Terminal([ByteKeyword]))) + ), + EnumVariant( + name = String, + fields = (type_name = Required(Terminal([StringKeyword]))) + ), + EnumVariant( + name = Address, + fields = (type_name = Required(NonTerminal(AddressType))) + ), + EnumVariant( + name = ByteArray, + fields = (type_name = Required(Terminal([BytesKeyword]))) + ), + EnumVariant( + name = SignedInteger, + fields = (type_name = Required(Terminal([IntKeyword]))) + ), + EnumVariant( + name = UnsignedInteger, + fields = (type_name = Required(Terminal([UintKeyword]))) + ), + EnumVariant( + name = SignedFixedPointNumber, + fields = (type_name = Required(Terminal([FixedKeyword]))) + ), + EnumVariant( + name = UnsignedFixedPointNumber, + fields = (type_name = Required(Terminal([UfixedKeyword]))) + ) + ] + ), + Enum( + name = AddressType, + default_variant = Address, + variants = [ + EnumVariant( + name = Address, + fields = ( + address_keyword = Required(Terminal([AddressKeyword])), + payable_keyword = + Optional(kind = Terminal([PayableKeyword])) + ) + ), + EnumVariant( + name = Payable, + fields = + (payable_keyword = Required(Terminal([PayableKeyword]))) + ) + ] + ) + ] + ) + ] + ), + Section( + title = "Statements", + topics = [ + Topic( + title = "Blocks", + items = [ + Struct( + name = Block, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + open_brace = Required(Terminal([OpenBrace])), + statements = Required(NonTerminal(Statements)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Repeated(name = Statements, repeated = Statement, allow_empty = true), + Enum( + name = Statement, + default_variant = Block, + variants = [ + EnumVariant( + name = TupleDeconstruction, + fields = (statement = + Required(NonTerminal(TupleDeconstructionStatement))) + ), + EnumVariant( + name = VariableDeclaration, + fields = (statement = + Required(NonTerminal(VariableDeclarationStatement))) + ), + EnumVariant( + name = If, + fields = (statement = Required(NonTerminal(IfStatement))) + ), + EnumVariant( + name = For, + fields = (statement = Required(NonTerminal(ForStatement))) + ), + EnumVariant( + name = While, + fields = (statement = Required(NonTerminal(WhileStatement))) + ), + EnumVariant( + name = DoWhile, + fields = (statement = Required(NonTerminal(DoWhileStatement))) + ), + EnumVariant( + name = Continue, + fields = (statement = Required(NonTerminal(ContinueStatement))) + ), + EnumVariant( + name = Break, + fields = (statement = Required(NonTerminal(BreakStatement))) + ), + EnumVariant( + name = Delete, + fields = (statement = Required(NonTerminal(DeleteStatement))) + ), + EnumVariant( + name = Return, + fields = (statement = Required(NonTerminal(ReturnStatement))) + ), + EnumVariant( + name = Throw, + disabled_in = "0.5.0", + fields = (statement = Required(NonTerminal(ThrowStatement))) + ), + EnumVariant( + name = Emit, + enabled_in = "0.4.21", + fields = (statement = Required(NonTerminal(EmitStatement))) + ), + EnumVariant( + name = Try, + enabled_in = "0.6.0", + fields = (statement = Required(NonTerminal(TryStatement))) + ), + EnumVariant( + name = Revert, + enabled_in = "0.8.4", + fields = (statement = Required(NonTerminal(RevertStatement))) + ), + EnumVariant( + name = Assembly, + fields = (statement = Required(NonTerminal(AssemblyStatement))) + ), + EnumVariant( + name = Block, + fields = (block = Required(NonTerminal(Block))) + ), + EnumVariant( + name = UncheckedBlock, + enabled_in = "0.8.0", + fields = (block = Required(NonTerminal(UncheckedBlock))) + ), + EnumVariant( + name = Expression, + fields = + (statement = Required(NonTerminal(ExpressionStatement))) + ) + ] + ), + Struct( + name = UncheckedBlock, + enabled_in = "0.8.0", + fields = ( + unchecked_keyword = Required(Terminal([UncheckedKeyword])), + block = Required(NonTerminal(Block)) + ) + ), + Struct( + name = ExpressionStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + expression = Required(NonTerminal(Expression)), + semicolon = Required(Terminal([Semicolon])) + ) + ) + ] + ), + Topic( + title = "Declaration Statements", + items = [ + Struct( + name = TupleDeconstructionStatement, + error_recovery = FieldsErrorRecovery( + terminator = semicolon, + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + members = Required(NonTerminal(TupleMembersDeconstruction)), + close_paren = Required(Terminal([CloseParen])), + equal = Required(Terminal([Equal])), + expression = Required(NonTerminal(Expression)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Separated( + name = TupleMembersDeconstruction, + separated = TupleMemberDeconstruction, + separator = Comma, + allow_empty = true + ), + Struct( + name = TupleMemberDeconstruction, + fields = (member = Optional(kind = NonTerminal(TupleMember))) + ), + Enum( + name = TupleMember, + default_variant = Typed, + variants = [ + EnumVariant( + name = Typed, + fields = ( + type_name = Required(NonTerminal(TypeName)), + storage_location = + Optional(kind = NonTerminal(StorageLocation)), + name = Required(Terminal([Identifier])) + ) + ), + EnumVariant( + name = Untyped, + fields = ( + storage_location = + Optional(kind = NonTerminal(StorageLocation)), + name = Required(Terminal([Identifier])) + ) + ) + ] + ), + Struct( + name = VariableDeclarationStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + variable_type = Required(NonTerminal(VariableDeclarationType)), + storage_location = Optional(kind = NonTerminal(StorageLocation)), + name = Required(Terminal([Identifier])), + value = Optional(kind = NonTerminal(VariableDeclarationValue)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Enum( + name = VariableDeclarationType, + default_variant = Typed, + variants = [ + EnumVariant( + name = Typed, + fields = (type_name = Required(NonTerminal(TypeName))) + ), + EnumVariant( + name = Untyped, + disabled_in = "0.5.0", + fields = (type_name = Required(Terminal([VarKeyword]))) + ) + ] + ), + Struct( + name = VariableDeclarationValue, + fields = ( + equal = Required(Terminal([Equal])), + expression = Required(NonTerminal(Expression)) + ) + ), + Enum( + name = StorageLocation, + default_variant = Memory, + variants = [ + EnumVariant( + name = Memory, + fields = (keyword = Required(Terminal([MemoryKeyword]))) + ), + EnumVariant( + name = Storage, + fields = (keyword = Required(Terminal([StorageKeyword]))) + ), + EnumVariant( + name = CallData, + enabled_in = "0.5.0", + fields = (keyword = Required(Terminal([CallDataKeyword]))) + ) + ] + ) + ] + ), + Topic( + title = "Control Statements", + items = [ + Struct( + name = IfStatement, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + if_keyword = Required(Terminal([IfKeyword])), + open_paren = Required(Terminal([OpenParen])), + condition = Required(NonTerminal(Expression)), + close_paren = Required(Terminal([CloseParen])), + body = Required(NonTerminal(Statement)), + else_branch = Optional(kind = NonTerminal(ElseBranch)) + ) + ), + Struct( + name = ElseBranch, + fields = ( + else_keyword = Optional(kind = Terminal([ElseKeyword])), + body = Optional(kind = NonTerminal(Statement)) + ) + ), + Struct( + name = ForStatement, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + for_keyword = Required(Terminal([ForKeyword])), + open_paren = Required(Terminal([OpenParen])), + initialization = Required(NonTerminal(ForStatementInitialization)), + condition = Required(NonTerminal(ForStatementCondition)), + iterator = Optional(kind = NonTerminal(Expression)), + close_paren = Required(Terminal([CloseParen])), + body = Required(NonTerminal(Statement)) + ) + ), + Enum( + name = ForStatementInitialization, + default_variant = Semicolon, + variants = [ + EnumVariant( + name = Expression, + fields = + (statement = Required(NonTerminal(ExpressionStatement))) + ), + EnumVariant( + name = VariableDeclaration, + fields = (statement = + Required(NonTerminal(VariableDeclarationStatement))) + ), + EnumVariant( + name = TupleDeconstruction, + fields = (statement = + Required(NonTerminal(TupleDeconstructionStatement))) + ), + EnumVariant( + name = Semicolon, + fields = (semicolon = Required(Terminal([Semicolon]))) + ) + ] + ), + Enum( + name = ForStatementCondition, + default_variant = Semicolon, + variants = [ + EnumVariant( + name = Expression, + fields = + (statement = Required(NonTerminal(ExpressionStatement))) + ), + EnumVariant( + name = Semicolon, + fields = (semicolon = Required(Terminal([Semicolon]))) + ) + ] + ), + Struct( + name = WhileStatement, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + while_keyword = Required(Terminal([WhileKeyword])), + open_paren = Required(Terminal([OpenParen])), + condition = Required(NonTerminal(Expression)), + close_paren = Required(Terminal([CloseParen])), + body = Required(NonTerminal(Statement)) + ) + ), + Struct( + name = DoWhileStatement, + error_recovery = FieldsErrorRecovery( + terminator = semicolon, + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + do_keyword = Required(Terminal([DoKeyword])), + body = Required(NonTerminal(Statement)), + while_keyword = Required(Terminal([WhileKeyword])), + open_paren = Required(Terminal([OpenParen])), + condition = Required(NonTerminal(Expression)), + close_paren = Required(Terminal([CloseParen])), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = ContinueStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + continue_keyword = Required(Terminal([ContinueKeyword])), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = BreakStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + break_keyword = Required(Terminal([BreakKeyword])), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = ReturnStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + return_keyword = Required(Terminal([ReturnKeyword])), + expression = Optional(kind = NonTerminal(Expression)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = EmitStatement, + enabled_in = "0.4.21", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + emit_keyword = Required(Terminal([EmitKeyword])), + event = Required(NonTerminal(IdentifierPath)), + arguments = Required(NonTerminal(ArgumentsDeclaration)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = DeleteStatement, + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + delete_keyword = Required(Terminal([DeleteKeyword])), + expression = Required(NonTerminal(Expression)), + semicolon = Required(Terminal([Semicolon])) + ) + ) + ] + ), + Topic( + title = "Error Handling", + items = [ + Struct( + name = TryStatement, + enabled_in = "0.6.0", + fields = ( + try_keyword = Required(Terminal([TryKeyword])), + expression = Required(NonTerminal(Expression)), + returns = Optional(kind = NonTerminal(ReturnsDeclaration)), + body = Required(NonTerminal(Block)), + catch_clauses = Required(NonTerminal(CatchClauses)) + ) + ), + Repeated( + name = CatchClauses, + repeated = CatchClause, + enabled_in = "0.6.0" + ), + Struct( + name = CatchClause, + enabled_in = "0.6.0", + fields = ( + catch_keyword = Required(Terminal([CatchKeyword])), + error = Optional(kind = NonTerminal(CatchClauseError)), + body = Required(NonTerminal(Block)) + ) + ), + Struct( + name = CatchClauseError, + enabled_in = "0.6.0", + fields = ( + name = Optional(kind = Terminal([Identifier])), + parameters = Required(NonTerminal(ParametersDeclaration)) + ) + ), + Struct( + name = RevertStatement, + enabled_in = "0.8.4", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + revert_keyword = Required(Terminal([RevertKeyword])), + error = Optional(kind = NonTerminal(IdentifierPath)), + arguments = Required(NonTerminal(ArgumentsDeclaration)), + semicolon = Required(Terminal([Semicolon])) + ) + ), + Struct( + name = ThrowStatement, + disabled_in = "0.5.0", + error_recovery = FieldsErrorRecovery(terminator = semicolon), + fields = ( + throw_keyword = Required(Terminal([ThrowKeyword])), + semicolon = Required(Terminal([Semicolon])) + ) + ) + ] + ) + ] + ), + Section( + title = "Expressions", + topics = [ + Topic( + title = "Base Expressions", + items = [ + Precedence( + name = Expression, + precedence_expressions = [ + PrecedenceExpression( + name = AssignmentExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([ + Equal, + BarEqual, + PlusEqual, + MinusEqual, + CaretEqual, + SlashEqual, + PercentEqual, + AsteriskEqual, + AmpersandEqual, + LessThanLessThanEqual, + GreaterThanGreaterThanEqual, + GreaterThanGreaterThanGreaterThanEqual + ]))) + )] + ), + PrecedenceExpression( + name = ConditionalExpression, + operators = [PrecedenceOperator( + model = Postfix, + fields = ( + question_mark = Required(Terminal([QuestionMark])), + true_expression = Required(NonTerminal(Expression)), + colon = Required(Terminal([Colon])), + false_expression = Required(NonTerminal(Expression)) + ) + )] + ), + PrecedenceExpression( + name = OrExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([BarBar]))) + )] + ), + PrecedenceExpression( + name = AndExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = + (operator = Required(Terminal([AmpersandAmpersand]))) + )] + ), + PrecedenceExpression( + name = EqualityExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = + Required(Terminal([EqualEqual, BangEqual]))) + )] + ), + PrecedenceExpression( + name = ComparisonExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([ + LessThan, + GreaterThan, + LessThanEqual, + GreaterThanEqual + ]))) + )] + ), + PrecedenceExpression( + name = BitwiseOrExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([Bar]))) + )] + ), + PrecedenceExpression( + name = BitwiseXorExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([Caret]))) + )] + ), + PrecedenceExpression( + name = BitwiseAndExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([Ampersand]))) + )] + ), + PrecedenceExpression( + name = ShiftExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([ + LessThanLessThan, + GreaterThanGreaterThan, + GreaterThanGreaterThanGreaterThan + ]))) + )] + ), + PrecedenceExpression( + name = AdditiveExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = Required(Terminal([Plus, Minus]))) + )] + ), + PrecedenceExpression( + name = MultiplicativeExpression, + operators = [PrecedenceOperator( + model = BinaryLeftAssociative, + fields = (operator = + Required(Terminal([Asterisk, Slash, Percent]))) + )] + ), + PrecedenceExpression( + name = ExponentiationExpression, + operators = [ + // Before '0.6.0', it was left-associative: + PrecedenceOperator( + model = BinaryLeftAssociative, + disabled_in = "0.6.0", + fields = + (operator = Required(Terminal([AsteriskAsterisk]))) + ), + // In '0.6.0', it became right-associative: + PrecedenceOperator( + model = BinaryRightAssociative, + enabled_in = "0.6.0", + fields = + (operator = Required(Terminal([AsteriskAsterisk]))) + ) + ] + ), + PrecedenceExpression( + name = PostfixExpression, + operators = [PrecedenceOperator( + model = Postfix, + fields = + (operator = Required(Terminal([PlusPlus, MinusMinus]))) + )] + ), + PrecedenceExpression( + name = PrefixExpression, + operators = [ + // Before '0.5.0', 'Plus' was supported: + PrecedenceOperator( + model = Prefix, + disabled_in = "0.5.0", + fields = (operator = Required(Terminal([ + PlusPlus, MinusMinus, Tilde, Bang, Minus, Plus + ]))) + ), + // In '0.5.0', 'Plus' was removed: + PrecedenceOperator( + model = Prefix, + enabled_in = "0.5.0", + fields = (operator = Required(Terminal([ + PlusPlus, MinusMinus, Tilde, Bang, Minus + ]))) + ) + ] + ), + PrecedenceExpression( + name = FunctionCallExpression, + operators = [PrecedenceOperator( + model = Postfix, + fields = ( + options = Required(NonTerminal(FunctionCallOptions)), + arguments = Required(NonTerminal(ArgumentsDeclaration)) + ) + )] + ), + PrecedenceExpression( + name = MemberAccessExpression, + operators = [PrecedenceOperator( + model = Postfix, + fields = ( + period = Required(Terminal([Period])), + member = + Required(Terminal([Identifier, AddressKeyword])) + ) + )] + ), + PrecedenceExpression( + name = IndexAccessExpression, + operators = [PrecedenceOperator( + model = Postfix, + error_recovery = FieldsErrorRecovery( + delimiters = FieldDelimiters( + open = open_bracket, + close = close_bracket + ) + ), + fields = ( + open_bracket = Required(Terminal([OpenBracket])), + start = Optional(kind = NonTerminal(Expression)), + end = Optional(kind = NonTerminal(IndexAccessEnd)), + close_bracket = Required(Terminal([CloseBracket])) + ) + )] + ) + ], + default_primary_expression = Identifier, + primary_expressions = [ + PrimaryExpression(expression = NewExpression), + PrimaryExpression(expression = TupleExpression), + PrimaryExpression( + expression = TypeExpression, + enabled_in = "0.5.3" + ), + PrimaryExpression(expression = ArrayExpression), + PrimaryExpression(expression = NumberExpression), + PrimaryExpression(expression = StringExpression), + PrimaryExpression(expression = ElementaryType), + PrimaryExpression(expression = TrueKeyword), + PrimaryExpression(expression = FalseKeyword), + PrimaryExpression(expression = Identifier) + ] + ), + Struct( + name = IndexAccessEnd, + fields = ( + colon = Required(Terminal([Colon])), + end = Optional(kind = NonTerminal(Expression)) + ) + ) + ] + ), + Topic( + title = "Function Calls", + items = [ + Enum( + name = FunctionCallOptions, + default_variant = None, + variants = [ + EnumVariant( + name = Multiple, + enabled_in = "0.6.2", + disabled_in = "0.8.0", + fields = (options = + Required(NonTerminal(NamedArgumentsDeclarations))) + ), + EnumVariant( + name = Single, + enabled_in = "0.8.0", + fields = (options = + Optional(kind = NonTerminal(NamedArgumentsDeclaration))) + ), + EnumVariant(name = None, disabled_in = "0.6.2", fields = ()) + ] + ), + Enum( + name = ArgumentsDeclaration, + default_variant = Positional, + variants = [ + EnumVariant( + name = Positional, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + arguments = Required(NonTerminal(PositionalArguments)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + EnumVariant( + name = Named, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + arguments = + Optional(kind = NonTerminal(NamedArgumentsDeclaration)), + close_paren = Required(Terminal([CloseParen])) + ) + ) + ] + ), + Separated( + name = PositionalArguments, + separated = Expression, + separator = Comma, + allow_empty = true + ), + Repeated( + name = NamedArgumentsDeclarations, + repeated = NamedArgumentsDeclaration, + enabled_in = "0.6.2", + disabled_in = "0.8.0", + allow_empty = true + ), + Struct( + name = NamedArgumentsDeclaration, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_brace, close = close_brace) + ), + fields = ( + open_brace = Required(Terminal([OpenBrace])), + arguments = Required(NonTerminal(NamedArguments)), + close_brace = Required(Terminal([CloseBrace])) + ) + ), + Separated( + name = NamedArguments, + separated = NamedArgument, + separator = Comma, + allow_empty = true + ), + Struct( + name = NamedArgument, + fields = ( + name = Required(Terminal([Identifier])), + colon = Required(Terminal([Colon])), + value = Required(NonTerminal(Expression)) + ) + ) + ] + ), + Topic( + title = "Primary Expressions", + items = [ + Struct( + name = TypeExpression, + enabled_in = "0.5.3", + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + type_keyword = Required(Terminal([TypeKeyword])), + open_paren = Required(Terminal([OpenParen])), + type_name = Required(NonTerminal(TypeName)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Struct( + name = NewExpression, + fields = ( + new_keyword = Required(Terminal([NewKeyword])), + type_name = Required(NonTerminal(TypeName)) + ) + ), + Struct( + name = TupleExpression, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + items = Required(NonTerminal(TupleValues)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = TupleValues, + separated = TupleValue, + separator = Comma + ), + Struct( + name = TupleValue, + fields = (expression = Optional(kind = NonTerminal(Expression))) + ), + Struct( + name = ArrayExpression, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_bracket, close = close_bracket) + ), + fields = ( + open_bracket = Required(Terminal([OpenBracket])), + items = Required(NonTerminal(ArrayValues)), + close_bracket = Required(Terminal([CloseBracket])) + ) + ), + Separated( + name = ArrayValues, + separated = Expression, + separator = Comma + ) + ] + ), + Topic( + title = "Numbers", + items = [ + Enum( + name = NumberExpression, + default_variant = Decimal, + variants = [ + EnumVariant( + name = Hex, + fields = ( + literal = Required(Terminal([HexLiteral])), + unit = Optional( + kind = NonTerminal(NumberUnit), + disabled_in = "0.5.0" + ) + ) + ), + EnumVariant( + name = Decimal, + fields = ( + literal = Required(Terminal([DecimalLiteral])), + unit = Optional(kind = NonTerminal(NumberUnit)) + ) + ) + ] + ), + Token( + name = HexLiteral, + definitions = [ + TokenDefinition( + // Lowercase "0x" enabled in all versions: + scanner = TrailingContext( + scanner = Sequence([ + Atom("0x"), + OneOrMore(Fragment(HexCharacter)), + ZeroOrMore(Sequence([ + Atom("_"), + OneOrMore(Fragment(HexCharacter)) + ])) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ), + TokenDefinition( + // Uppercase "0X" only enabled before "0.5.0": + disabled_in = "0.5.0", + scanner = TrailingContext( + scanner = Sequence([ + Atom("0X"), + OneOrMore(Fragment(HexCharacter)), + ZeroOrMore(Sequence([ + Atom("_"), + OneOrMore(Fragment(HexCharacter)) + ])) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ) + ] + ), + Token( + name = DecimalLiteral, + definitions = [ + TokenDefinition( + // An integer (without a dot or a fraction) is enabled in all versions: + scanner = TrailingContext( + scanner = Sequence([ + Fragment(DecimalDigits), + Optional(Fragment(DecimalExponent)) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ), + TokenDefinition( + // An integer and a dot (without a fraction) is disabled in "0.5.0" + disabled_in = "0.5.0", + scanner = TrailingContext( + scanner = Sequence([ + Fragment(DecimalDigits), + Atom("."), + Optional(Fragment(DecimalExponent)) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ), + TokenDefinition( + // A dot and a fraction (without an integer) is enabled in all versions: + scanner = TrailingContext( + scanner = Sequence([ + Atom("."), + Fragment(DecimalDigits), + Optional(Fragment(DecimalExponent)) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ), + TokenDefinition( + // An integer, a dot, and a fraction is enabled in all versions: + scanner = TrailingContext( + scanner = Sequence([ + Fragment(DecimalDigits), + Atom("."), + Fragment(DecimalDigits), + Optional(Fragment(DecimalExponent)) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + ) + ] + ), + Fragment( + name = DecimalDigits, + scanner = Sequence([ + OneOrMore(Range(inclusive_start = '0', inclusive_end = '9')), + ZeroOrMore(Sequence([ + Atom("_"), + OneOrMore(Range(inclusive_start = '0', inclusive_end = '9')) + ])) + ]) + ), + Fragment( + name = DecimalExponent, + scanner = Sequence([ + Choice([Atom("e"), Atom("E")]), + Optional(Atom("-")), + Fragment(DecimalDigits) + ]) + ), + Enum( + name = NumberUnit, + default_variant = Seconds, + variants = [ + EnumVariant( + name = Wei, + // 1e-18 ETH + fields = (keyword = Required(Terminal([WeiKeyword]))) + ), + EnumVariant( + name = Gwei, + // 1e-9 ETH + enabled_in = "0.6.11", + fields = (keyword = Required(Terminal([GweiKeyword]))) + ), + EnumVariant( + name = Szabo, + // 1e-6 ETH + disabled_in = "0.7.0", + fields = (keyword = Required(Terminal([SzaboKeyword]))) + ), + EnumVariant( + name = Finney, + // 1e-3 ETH + disabled_in = "0.7.0", + fields = (keyword = Required(Terminal([FinneyKeyword]))) + ), + EnumVariant( + name = Ether, + // 1 ETH + fields = (keyword = Required(Terminal([EtherKeyword]))) + ), + EnumVariant( + name = Seconds, + fields = (keyword = Required(Terminal([SecondsKeyword]))) + ), + EnumVariant( + name = Minutes, + fields = (keyword = Required(Terminal([MinutesKeyword]))) + ), + EnumVariant( + name = Hours, + fields = (keyword = Required(Terminal([HoursKeyword]))) + ), + EnumVariant( + name = Days, + fields = (keyword = Required(Terminal([DaysKeyword]))) + ), + EnumVariant( + name = Weeks, + fields = (keyword = Required(Terminal([WeeksKeyword]))) + ), + EnumVariant( + name = Years, + disabled_in = "0.5.0", + fields = (keyword = Required(Terminal([YearsKeyword]))) + ) + ] + ) + ] + ), + Topic( + title = "Strings", + items = [ + Enum( + name = StringExpression, + default_variant = Ascii, + variants = [ + EnumVariant( + name = Hex, + fields = (literals = Required(NonTerminal(HexStringLiterals))) + ), + EnumVariant( + name = Ascii, + fields = + (literals = Required(NonTerminal(AsciiStringLiterals))) + ), + EnumVariant( + name = Unicode, + enabled_in = "0.7.0", + fields = + (literals = Required(NonTerminal(UnicodeStringLiterals))) + ) + ] + ), + Repeated(name = HexStringLiterals, repeated = HexStringLiteral), + Token( + name = HexStringLiteral, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Choice([ + Fragment(SingleQuotedHexString), + Fragment(DoubleQuotedHexString) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + )] + ), + Fragment( + name = SingleQuotedHexString, + scanner = Sequence([ + Atom("hex'"), + Optional(Fragment(HexStringContents)), + Atom("'") + ]) + ), + Fragment( + name = DoubleQuotedHexString, + scanner = Sequence([ + Atom("hex\""), + Optional(Fragment(HexStringContents)), + Atom("\"") + ]) + ), + Fragment( + name = HexStringContents, + scanner = Sequence([ + Fragment(HexCharacter), + Fragment(HexCharacter), + ZeroOrMore(Sequence([ + Optional(Atom("_")), + Fragment(HexCharacter), + Fragment(HexCharacter) + ])) + ]) + ), + Fragment( + name = HexCharacter, + scanner = Choice([ + Range(inclusive_start = '0', inclusive_end = '9'), + Range(inclusive_start = 'a', inclusive_end = 'f'), + Range(inclusive_start = 'A', inclusive_end = 'F') + ]) + ), + Repeated(name = AsciiStringLiterals, repeated = AsciiStringLiteral), + Token( + name = AsciiStringLiteral, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Choice([ + Fragment(SingleQuotedAsciiString), + Fragment(DoubleQuotedAsciiString) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + )] + ), + Fragment( + name = SingleQuotedAsciiString, + scanner = Sequence([ + Atom("'"), + ZeroOrMore(Choice([ + Fragment(EscapeSequence), + Range(inclusive_start = ' ', inclusive_end = '&'), + Range(inclusive_start = '(', inclusive_end = '['), + Range(inclusive_start = ']', inclusive_end = '~') + ])), + Atom("'") + ]) + ), + Fragment( + name = DoubleQuotedAsciiString, + scanner = Sequence([ + Atom("\""), + ZeroOrMore(Choice([ + Fragment(EscapeSequence), + Range(inclusive_start = ' ', inclusive_end = '!'), + Range(inclusive_start = '#', inclusive_end = '['), + Range(inclusive_start = ']', inclusive_end = '~') + ])), + Atom("\"") + ]) + ), + Repeated( + name = UnicodeStringLiterals, + repeated = UnicodeStringLiteral, + enabled_in = "0.7.0" + ), + Token( + name = UnicodeStringLiteral, + definitions = [TokenDefinition( + enabled_in = "0.7.0", + scanner = TrailingContext( + scanner = Choice([ + Fragment(SingleQuotedUnicodeString), + Fragment(DoubleQuotedUnicodeString) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + )] + ), + Fragment( + name = SingleQuotedUnicodeString, + enabled_in = "0.7.0", + scanner = Sequence([ + Atom("unicode'"), + ZeroOrMore(Choice([ + Fragment(EscapeSequence), + Not(['\'', '\\', '\r', '\n']) + ])), + Atom("'") + ]) + ), + Fragment( + name = DoubleQuotedUnicodeString, + enabled_in = "0.7.0", + scanner = Sequence([ + Atom("unicode\""), + ZeroOrMore(Choice([ + Fragment(EscapeSequence), + Not(['"', '\\', '\r', '\n']) + ])), + Atom("\"") + ]) + ), + Fragment( + name = EscapeSequence, + scanner = Sequence([ + Atom("\\"), + Choice([ + Fragment(AsciiEscape), + Fragment(HexByteEscape), + Fragment(UnicodeEscape) + ]) + ]) + ), + Fragment( + name = AsciiEscape, + scanner = Choice([ + Atom("n"), + Atom("r"), + Atom("t"), + Atom("'"), + Atom("\""), + Atom("\\"), + Atom("\n"), + Atom("\r") + ]) + ), + Fragment( + name = HexByteEscape, + scanner = Sequence([ + Atom("x"), + Fragment(HexCharacter), + Fragment(HexCharacter) + ]) + ), + Fragment( + name = UnicodeEscape, + scanner = Sequence([ + Atom("u"), + Fragment(HexCharacter), + Fragment(HexCharacter), + Fragment(HexCharacter), + Fragment(HexCharacter) + ]) + ) + ] + ), + Topic( + title = "Identifiers", + items = [ + Separated( + name = IdentifierPath, + separated = Identifier, + separator = Period + ), + Token( + name = Identifier, + definitions = [TokenDefinition(scanner = Fragment(RawIdentifier))] + ), + Fragment( + name = RawIdentifier, + scanner = Sequence([ + Fragment(IdentifierStart), + ZeroOrMore(Fragment(IdentifierPart)) + ]) + ), + Fragment( + name = IdentifierStart, + scanner = Choice([ + Atom("_"), + Atom("$"), + Range(inclusive_start = 'a', inclusive_end = 'z'), + Range(inclusive_start = 'A', inclusive_end = 'Z') + ]) + ), + Fragment( + name = IdentifierPart, + scanner = Choice([ + Fragment(IdentifierStart), + Range(inclusive_start = '0', inclusive_end = '9') + ]) + ) + ] + ) + ] + ), + Section( + title = "Yul", + topics = [ + Topic( + title = "Yul Statements", + lexical_context = Yul, + items = [ + Struct( + name = AssemblyStatement, + fields = ( + assembly_keyword = Required(Terminal([AssemblyKeyword])), + label = Optional(kind = Terminal([AsciiStringLiteral])), + flags = Optional(kind = NonTerminal(AssemblyFlagsDeclaration)), + body = Required(NonTerminal(YulBlock)) + ) + ), + Struct( + name = AssemblyFlagsDeclaration, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + flags = Required(NonTerminal(AssemblyFlags)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = AssemblyFlags, + separated = AsciiStringLiteral, + separator = Comma, + allow_empty = true + ), + Struct( + name = YulBlock, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + statements = Required(NonTerminal(YulStatements)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Repeated( + name = YulStatements, + repeated = YulStatement, + allow_empty = true + ), + Enum( + name = YulStatement, + default_variant = Block, + variants = [ + EnumVariant( + name = Block, + fields = (block = Required(NonTerminal(YulBlock))) + ), + EnumVariant( + name = Function, + fields = + (definition = Required(NonTerminal(YulFunctionDefinition))) + ), + EnumVariant( + name = VariableDeclaration, + fields = (statement = + Required(NonTerminal(YulVariableDeclarationStatement))) + ), + EnumVariant( + name = Assignment, + fields = + (statement = Required(NonTerminal(YulAssignmentStatement))) + ), + EnumVariant( + name = If, + fields = (statement = Required(NonTerminal(YulIfStatement))) + ), + EnumVariant( + name = For, + fields = (statement = Required(NonTerminal(YulForStatement))) + ), + EnumVariant( + name = Switch, + fields = + (statement = Required(NonTerminal(YulSwitchStatement))) + ), + EnumVariant( + name = Leave, + enabled_in = "0.6.0", + fields = (statement = Required(NonTerminal(YulLeaveStatement))) + ), + EnumVariant( + name = Break, + fields = (statement = Required(NonTerminal(YulBreakStatement))) + ), + EnumVariant( + name = Continue, + fields = + (statement = Required(NonTerminal(YulContinueStatement))) + ), + EnumVariant( + name = Expression, + fields = (expression = Required(NonTerminal(YulExpression))) + ) + ] + ), + Struct( + name = YulFunctionDefinition, + fields = ( + function_keyword = Required(Terminal([YulFunctionKeyword])), + name = Required(Terminal([YulIdentifier])), + parameters = Required(NonTerminal(YulParametersDeclaration)), + returns = Optional(kind = NonTerminal(YulReturnsDeclaration)), + body = Required(NonTerminal(YulBlock)) + ) + ), + Struct( + name = YulParametersDeclaration, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + parameters = Required(NonTerminal(YulParameters)), + close_paren = Required(Terminal([CloseParen])) + ) + ), + Separated( + name = YulParameters, + separated = YulIdentifier, + separator = Comma, + allow_empty = true + ), + Struct( + name = YulReturnsDeclaration, + fields = ( + minus_greater_than = Required(Terminal([MinusGreaterThan])), + variables = Required(NonTerminal(YulReturnVariables)) + ) + ), + Separated( + name = YulReturnVariables, + separated = YulIdentifier, + separator = Comma + ), + Struct( + name = YulVariableDeclarationStatement, + fields = ( + let_keyword = Required(Terminal([YulLetKeyword])), + names = Required(NonTerminal(YulIdentifierPaths)), + value = Optional(kind = NonTerminal(YulVariableDeclarationValue)) + ) + ), + Struct( + name = YulVariableDeclarationValue, + fields = ( + colon_equal = Required(Terminal([ColonEqual])), + expression = Required(NonTerminal(YulExpression)) + ) + ), + Struct( + name = YulAssignmentStatement, + fields = ( + names = Required(NonTerminal(YulIdentifierPaths)), + colon_equal = Required(Terminal([ColonEqual])), + expression = Required(NonTerminal(YulExpression)) + ) + ), + Struct( + name = YulIfStatement, + fields = ( + if_keyword = Required(Terminal([YulIfKeyword])), + condition = Required(NonTerminal(YulExpression)), + body = Required(NonTerminal(YulBlock)) + ) + ), + Struct( + name = YulLeaveStatement, + enabled_in = "0.6.0", + fields = (leave_keyword = Required(Terminal([YulLeaveKeyword]))) + ), + Struct( + name = YulBreakStatement, + fields = (break_keyword = Required(Terminal([YulBreakKeyword]))) + ), + Struct( + name = YulContinueStatement, + fields = (continue_keyword = Required(Terminal([YulContinueKeyword]))) + ), + Struct( + name = YulForStatement, + fields = ( + for_keyword = Required(Terminal([YulForKeyword])), + initialization = Required(NonTerminal(YulBlock)), + condition = Required(NonTerminal(YulExpression)), + iterator = Required(NonTerminal(YulBlock)), + body = Required(NonTerminal(YulBlock)) + ) + ), + Struct( + name = YulSwitchStatement, + fields = ( + switch_keyword = Required(Terminal([YulSwitchKeyword])), + expression = Required(NonTerminal(YulExpression)), + cases = Required(NonTerminal(YulSwitchCases)) + ) + ), + Repeated(name = YulSwitchCases, repeated = YulSwitchCase), + Enum( + name = YulSwitchCase, + default_variant = Default, + variants = [ + EnumVariant( + name = Default, + fields = ( + default_keyword = Required(Terminal([YulDefaultKeyword])), + body = Required(NonTerminal(YulBlock)) + ) + ), + EnumVariant( + name = Case, + fields = ( + case_keyword = Required(Terminal([YulCaseKeyword])), + value = Required(NonTerminal(YulLiteral)), + body = Required(NonTerminal(YulBlock)) + ) + ) + ] + ) + ] + ), + Topic( + title = "Yul Expressions", + lexical_context = Yul, + items = [ + Precedence( + name = YulExpression, + precedence_expressions = [PrecedenceExpression( + name = YulFunctionCallExpression, + operators = [PrecedenceOperator( + model = Postfix, + error_recovery = FieldsErrorRecovery( + delimiters = + FieldDelimiters(open = open_paren, close = close_paren) + ), + fields = ( + open_paren = Required(Terminal([OpenParen])), + arguments = Required(NonTerminal(YulArguments)), + close_paren = Required(Terminal([CloseParen])) + ) + )] + )], + default_primary_expression = YulLiteral, + primary_expressions = [ + PrimaryExpression(expression = YulLiteral), + PrimaryExpression(expression = YulIdentifierPath) + ] + ), + Separated( + name = YulArguments, + separated = YulExpression, + separator = Comma, + allow_empty = true + ), + Separated( + name = YulIdentifierPaths, + separated = YulIdentifierPath, + separator = Comma + ), + Separated( + name = YulIdentifierPath, + separated = YulIdentifier, + separator = Period + ), + Token( + name = YulIdentifier, + definitions = [TokenDefinition(scanner = Fragment(RawIdentifier))] + ), + Enum( + name = YulLiteral, + default_variant = Decimal, + variants = [ + EnumVariant( + name = True, + fields = (literal = Required(Terminal([YulTrueKeyword]))) + ), + EnumVariant( + name = False, + fields = (literal = Required(Terminal([YulFalseKeyword]))) + ), + EnumVariant( + name = Decimal, + fields = (literal = Required(Terminal([YulDecimalLiteral]))) + ), + EnumVariant( + name = Hex, + fields = (literal = Required(Terminal([YulHexLiteral]))) + ), + EnumVariant( + name = HexString, + fields = (literal = Required(Terminal([HexStringLiteral]))) + ), + EnumVariant( + name = AsciiString, + fields = (literal = Required(Terminal([AsciiStringLiteral]))) + ) + ] + ), + Token( + name = YulDecimalLiteral, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = Choice([ + Atom("0"), + Sequence([ + Range(inclusive_start = '1', inclusive_end = '9'), + ZeroOrMore(Range( + inclusive_start = '0', + inclusive_end = '9' + )) + ]) + ]), + not_followed_by = Fragment(IdentifierPart) + ) + )] + ), + Token( + name = YulHexLiteral, + definitions = [TokenDefinition( + scanner = TrailingContext( + scanner = + Sequence([Atom("0x"), OneOrMore(Fragment(HexCharacter))]), + not_followed_by = Fragment(IdentifierPart) + ) + )] + ) + ] + ), + Topic( + title = "Keywords", + lexical_context = Yul, + items = [ + Keyword( + name = YulBreakKeyword, + identifier = YulIdentifier, + value = Atom("break") + ), + Keyword( + name = YulCaseKeyword, + identifier = YulIdentifier, + value = Atom("case") + ), + Keyword( + name = YulContinueKeyword, + identifier = YulIdentifier, + value = Atom("continue") + ), + Keyword( + name = YulDefaultKeyword, + identifier = YulIdentifier, + value = Atom("default") + ), + Keyword( + name = YulFalseKeyword, + identifier = YulIdentifier, + value = Atom("false") + ), + Keyword( + name = YulForKeyword, + identifier = YulIdentifier, + value = Atom("for") + ), + Keyword( + name = YulFunctionKeyword, + identifier = YulIdentifier, + value = Atom("function") + ), + Keyword( + name = YulHexKeyword, + identifier = YulIdentifier, + disabled_in = "0.4.11", + value = Atom("hex") + ), + Keyword( + name = YulIfKeyword, + identifier = YulIdentifier, + value = Atom("if") + ), + Keyword( + name = YulLeaveKeyword, + identifier = YulIdentifier, + enabled_in = "0.6.0", + value = Atom("leave") + ), + Keyword( + name = YulLetKeyword, + identifier = YulIdentifier, + value = Atom("let") + ), + Keyword( + name = YulSwitchKeyword, + identifier = YulIdentifier, + value = Atom("switch") + ), + Keyword( + name = YulTrueKeyword, + identifier = YulIdentifier, + value = Atom("true") + ) + ] + ) + ] + ) + ] +)); diff --git a/crates/solidity/inputs/language/src/lib.rs b/crates/solidity/inputs/language/src/lib.rs index 286947d606..27caf8de49 100644 --- a/crates/solidity/inputs/language/src/lib.rs +++ b/crates/solidity/inputs/language/src/lib.rs @@ -6,7 +6,10 @@ //! //! Call the [`SolidityLanguageExtensions::load_solidity`] method to load the precompiled language definition. +mod definition; mod dsl; + +pub use definition::*; pub use dsl::*; use anyhow::Result;