Skip to content

Commit

Permalink
Freeze AST option
Browse files Browse the repository at this point in the history
To make it so that you can pass `freeze: true` to Prism parse
methods and get back a deeply-frozen AST that is Ractor-
shareable.
  • Loading branch information
kddnewton committed Jan 12, 2025
1 parent 8d9d429 commit 8e6a93b
Show file tree
Hide file tree
Showing 14 changed files with 586 additions and 229 deletions.
2 changes: 1 addition & 1 deletion docs/ruby_api.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ The full API is documented below.
* `Prism.parse_stream(io)` - parse the syntax tree corresponding to the source that is read out of the given IO object using the `#gets` method and return it within a parse result
* `Prism.parse_lex(source)` - parse the syntax tree corresponding to the given source string and return it within a parse result, along with the tokens
* `Prism.parse_lex_file(filepath)` - parse the syntax tree corresponding to the given source file and return it within a parse result, along with the tokens
* `Prism.load(source, serialized)` - load the serialized syntax tree using the source as a reference into a syntax tree
* `Prism.load(source, serialized, freeze = false)` - load the serialized syntax tree using the source as a reference into a syntax tree
* `Prism.parse_comments(source)` - parse the comments corresponding to the given source string and return them
* `Prism.parse_file_comments(source)` - parse the comments corresponding to the given source file and return them
* `Prism.parse_success?(source)` - parse the syntax tree corresponding to the given source string and return true if it was parsed without errors
Expand Down
195 changes: 137 additions & 58 deletions ext/prism/extension.c

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions ext/prism/extension.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
#include <ruby/encoding.h>
#include "prism.h"

VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding);
VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source);
VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source);
VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze);
VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze);
VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze);
VALUE pm_integer_new(const pm_integer_t *integer);

void Init_prism_api_node(void);
Expand Down
16 changes: 16 additions & 0 deletions include/prism/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,13 @@ typedef struct pm_options {
* inside another script.
*/
bool partial_script;

/**
* Whether or not the parser should freeze the nodes that it creates. This
* makes it possible to have a deeply frozen AST that is safe to share
* between concurrency primitives.
*/
bool freeze;
} pm_options_t;

/**
Expand Down Expand Up @@ -285,6 +292,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, b
*/
PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);

/**
* Set the freeze option on the given options struct.
*
* @param options The options struct to set the freeze value on.
* @param freeze The freeze value to set.
*/
PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze);

/**
* Allocate and zero out the scopes array on the given options struct.
*
Expand Down Expand Up @@ -355,6 +370,7 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
* | `1` | encoding locked |
* | `1` | main script |
* | `1` | partial script |
* | `1` | freeze |
* | `4` | the number of scopes |
* | ... | the scopes |
*
Expand Down
3 changes: 3 additions & 0 deletions java/org/prism/ParsingOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boole
// partialScript
output.write(partialScript ? 1 : 0);

// freeze
output.write(0);

// scopes

// number of scopes
Expand Down
3 changes: 3 additions & 0 deletions javascript/src/parsePrism.js
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@ function dumpOptions(options) {
template.push("C");
values.push(dumpBooleanOption(options.partial_script));

template.push("C");
values.push(0);

template.push("L");
if (options.scopes) {
const scopes = options.scopes;
Expand Down
6 changes: 3 additions & 3 deletions lib/prism.rb
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ def self.lex_ripper(source)
end

# :call-seq:
# Prism::load(source, serialized) -> ParseResult
# Prism::load(source, serialized, freeze) -> ParseResult
#
# Load the serialized AST using the source as a reference into a tree.
def self.load(source, serialized)
Serialize.load(source, serialized)
def self.load(source, serialized, freeze = false)
Serialize.load(source, serialized, freeze)
end
end

Expand Down
72 changes: 60 additions & 12 deletions lib/prism/ffi.rb
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ def parse_stream(stream, **options)
# access to the IO object already through the closure of the lambda, we
# can pass a null pointer here and not worry.
LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
Prism.load(source, buffer.read)
Prism.load(source, buffer.read, options.fetch(:freeze, false))
end
end

Expand Down Expand Up @@ -354,22 +354,37 @@ def profile_file(filepath, **options)
def dump_common(string, options) # :nodoc:
LibRubyParser::PrismBuffer.with do |buffer|
LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
buffer.read

dumped = buffer.read
dumped.freeze if options.fetch(:freeze, false)

dumped
end
end

def lex_common(string, code, options) # :nodoc:
serialized = LibRubyParser::PrismBuffer.with do |buffer|
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
buffer.read
serialized =
LibRubyParser::PrismBuffer.with do |buffer|
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
buffer.read
end

freeze = options.fetch(:freeze, false)
source = Source.for(code)
result = Serialize.load_tokens(source, serialized, freeze)

if freeze
source.source.freeze
source.offsets.freeze
source.freeze
end

Serialize.load_tokens(Source.for(code), serialized)
result
end

def parse_common(string, code, options) # :nodoc:
serialized = dump_common(string, options)
Prism.load(code, serialized)
Prism.load(code, serialized, options.fetch(:freeze, false))
end

def parse_comments_common(string, code, options) # :nodoc:
Expand All @@ -382,7 +397,14 @@ def parse_comments_common(string, code, options) # :nodoc:
loader.load_header
loader.load_encoding
loader.load_start_line
loader.load_comments

if (freeze = options.fetch(:freeze, false))
source.source.freeze
source.offsets.freeze
source.freeze
end

loader.load_comments(freeze)
end
end

Expand All @@ -392,12 +414,35 @@ def parse_lex_common(string, code, options) # :nodoc:

source = Source.for(code)
loader = Serialize::Loader.new(source, buffer.read)
freeze = options.fetch(:freeze, false)

tokens = loader.load_tokens
node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
tokens.each { |token,| token.value.force_encoding(loader.encoding) }
tokens = loader.load_tokens(false)
node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes(freeze)

ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
tokens.each do |token,|
token.value.force_encoding(loader.encoding)

if freeze
token.value.freeze
token.location.freeze
token.freeze
end
end

value = [node, tokens]
result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, source)

if freeze
source.source.freeze
source.offsets.freeze
source.freeze
tokens.each(&:freeze)
tokens.freeze
value.freeze
result.freeze
end

result
end
end

Expand Down Expand Up @@ -482,6 +527,9 @@ def dump_options(options)
template << "C"
values << (options.fetch(:partial_script, false) ? 1 : 0)

template << "C"
values << (options.fetch(:freeze, false) ? 1 : 0)

template << "L"
if (scopes = options[:scopes])
values << scopes.length
Expand Down
Loading

0 comments on commit 8e6a93b

Please sign in to comment.