Skip to content

Commit 5d64b43

Browse files
kddnewtonkddnewton
committed
Freeze AST option
To make it so that you can pass `freeze: true` to Prism parse methods and get back a deeply-frozen AST that is Ractor- shareable.
1 parent 8d9d429 commit 5d64b43

File tree

11 files changed

+544
-191
lines changed

11 files changed

+544
-191
lines changed

‎ext/prism/extension.c‎

Lines changed: 137 additions & 58 deletions
Large diffs are not rendered by default.

‎ext/prism/extension.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
#include <ruby/encoding.h>
88
#include "prism.h"
99

10-
VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding);
11-
VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source);
12-
VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source);
10+
VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze);
11+
VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze);
12+
VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze);
1313
VALUE pm_integer_new(const pm_integer_t *integer);
1414

1515
void Init_prism_api_node(void);

‎include/prism/options.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,13 @@ typedef struct pm_options {
160160
* inside another script.
161161
*/
162162
bool partial_script;
163+
164+
/**
165+
* Whether or not the parser should freeze the nodes that it creates. This
166+
* makes it possible to have a deeply frozen AST that is safe to share
167+
* between concurrency primitives.
168+
*/
169+
bool freeze;
163170
} pm_options_t;
164171

165172
/**
@@ -285,6 +292,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, b
285292
*/
286293
PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
287294

295+
/**
296+
* Set the freeze option on the given options struct.
297+
*
298+
* @param options The options struct to set the freeze value on.
299+
* @param freeze The freeze value to set.
300+
*/
301+
PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze);
302+
288303
/**
289304
* Allocate and zero out the scopes array on the given options struct.
290305
*
@@ -355,6 +370,7 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
355370
* | `1` | encoding locked |
356371
* | `1` | main script |
357372
* | `1` | partial script |
373+
* | `1` | freeze |
358374
* | `4` | the number of scopes |
359375
* | ... | the scopes |
360376
*

‎java/org/prism/ParsingOptions.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boole
8282
// partialScript
8383
output.write(partialScript ? 1 : 0);
8484

85+
// freeze
86+
output.write(0);
87+
8588
// scopes
8689

8790
// number of scopes

‎javascript/src/parsePrism.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ function dumpOptions(options) {
122122
template.push("C");
123123
values.push(dumpBooleanOption(options.partial_script));
124124

125+
template.push("C");
126+
values.push(0);
127+
125128
template.push("L");
126129
if (options.scopes) {
127130
const scopes = options.scopes;

‎lib/prism.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,11 @@ def self.lex_ripper(source)
5959
end
6060

6161
# :call-seq:
62-
# Prism::load(source, serialized) -> ParseResult
62+
# Prism::load(source, serialized, freeze) -> ParseResult
6363
#
6464
# Load the serialized AST using the source as a reference into a tree.
65-
def self.load(source, serialized)
66-
Serialize.load(source, serialized)
65+
def self.load(source, serialized, freeze = false)
66+
Serialize.load(source, serialized, freeze)
6767
end
6868
end
6969

‎lib/prism/ffi.rb‎

Lines changed: 60 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ def parse_stream(stream, **options)
279279
# access to the IO object already through the closure of the lambda, we
280280
# can pass a null pointer here and not worry.
281281
LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
282-
Prism.load(source, buffer.read)
282+
Prism.load(source, buffer.read, options.fetch(:freeze, false))
283283
end
284284
end
285285

@@ -354,22 +354,37 @@ def profile_file(filepath, **options)
354354
def dump_common(string, options) # :nodoc:
355355
LibRubyParser::PrismBuffer.with do |buffer|
356356
LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
357-
buffer.read
357+
358+
dumped = buffer.read
359+
dumped.freeze if options.fetch(:freeze, false)
360+
361+
dumped
358362
end
359363
end
360364

361365
def lex_common(string, code, options) # :nodoc:
362-
serialized = LibRubyParser::PrismBuffer.with do |buffer|
363-
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
364-
buffer.read
366+
serialized =
367+
LibRubyParser::PrismBuffer.with do |buffer|
368+
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
369+
buffer.read
370+
end
371+
372+
freeze = options.fetch(:freeze, false)
373+
source = Source.for(code)
374+
result = Serialize.load_tokens(source, serialized, freeze)
375+
376+
if freeze
377+
source.source.freeze
378+
source.offsets.freeze
379+
source.freeze
365380
end
366381

367-
Serialize.load_tokens(Source.for(code), serialized)
382+
result
368383
end
369384

370385
def parse_common(string, code, options) # :nodoc:
371386
serialized = dump_common(string, options)
372-
Prism.load(code, serialized)
387+
Prism.load(code, serialized, options.fetch(:freeze, false))
373388
end
374389

375390
def parse_comments_common(string, code, options) # :nodoc:
@@ -382,7 +397,14 @@ def parse_comments_common(string, code, options) # :nodoc:
382397
loader.load_header
383398
loader.load_encoding
384399
loader.load_start_line
385-
loader.load_comments
400+
401+
if (freeze = options.fetch(:freeze, false))
402+
source.source.freeze
403+
source.offsets.freeze
404+
source.freeze
405+
end
406+
407+
loader.load_comments(freeze)
386408
end
387409
end
388410

@@ -392,12 +414,35 @@ def parse_lex_common(string, code, options) # :nodoc:
392414

393415
source = Source.for(code)
394416
loader = Serialize::Loader.new(source, buffer.read)
417+
freeze = options.fetch(:freeze, false)
395418

396-
tokens = loader.load_tokens
397-
node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
398-
tokens.each { |token,| token.value.force_encoding(loader.encoding) }
419+
tokens = loader.load_tokens(false)
420+
node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes(freeze)
399421

400-
ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
422+
tokens.each do |token,|
423+
token.value.force_encoding(loader.encoding)
424+
425+
if freeze
426+
token.value.freeze
427+
token.location.freeze
428+
token.freeze
429+
end
430+
end
431+
432+
value = [node, tokens]
433+
result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, source)
434+
435+
if freeze
436+
source.source.freeze
437+
source.offsets.freeze
438+
source.freeze
439+
tokens.each(&:freeze)
440+
tokens.freeze
441+
value.freeze
442+
result.freeze
443+
end
444+
445+
result
401446
end
402447
end
403448

@@ -482,6 +527,9 @@ def dump_options(options)
482527
template << "C"
483528
values << (options.fetch(:partial_script, false) ? 1 : 0)
484529

530+
template << "C"
531+
values << (options.fetch(:freeze, false) ? 1 : 0)
532+
485533
template << "L"
486534
if (scopes = options[:scopes])
487535
values << scopes.length

‎src/options.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,14 @@ pm_options_partial_script_set(pm_options_t *options, bool partial_script) {
139139
options->partial_script = partial_script;
140140
}
141141

142+
/**
143+
* Set the freeze option on the given options struct.
144+
*/
145+
PRISM_EXPORTED_FUNCTION void
146+
pm_options_freeze_set(pm_options_t *options, bool freeze) {
147+
options->freeze = freeze;
148+
}
149+
142150
// For some reason, GCC analyzer thinks we're leaking allocated scopes and
143151
// locals here, even though we definitely aren't. This is a false positive.
144152
// Ideally we wouldn't need to suppress this.
@@ -274,6 +282,7 @@ pm_options_read(pm_options_t *options, const char *data) {
274282
options->encoding_locked = ((uint8_t) *data++) > 0;
275283
options->main_script = ((uint8_t) *data++) > 0;
276284
options->partial_script = ((uint8_t) *data++) > 0;
285+
options->freeze = ((uint8_t) *data++) > 0;
277286

278287
uint32_t scopes_count = pm_options_read_u32(data);
279288
data += 4;

‎templates/ext/prism/api_node.c.erb‎

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,34 @@ static VALUE rb_cPrism<%= node.name %>;
1212
<%- end -%>
1313

1414
static VALUE
15-
pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
16-
uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start)));
17-
return ULL2NUM(value);
15+
pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) {
16+
if (freeze) {
17+
VALUE location_argv[] = {
18+
source,
19+
LONG2FIX(start - parser->start),
20+
LONG2FIX(end - start)
21+
};
22+
23+
return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation));
24+
} else {
25+
uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start)));
26+
return ULL2NUM(value);
27+
}
1828
}
1929

2030
VALUE
21-
pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source) {
31+
pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) {
2232
ID type = rb_intern(pm_token_type_name(token->type));
23-
VALUE location = pm_location_new(parser, token->start, token->end);
33+
VALUE location = pm_location_new(parser, token->start, token->end, source, freeze);
2434

25-
VALUE argv[] = {
26-
source,
27-
ID2SYM(type),
28-
rb_enc_str_new((const char *) token->start, token->end - token->start, encoding),
29-
location
30-
};
35+
VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding);
36+
if (freeze) rb_obj_freeze(slice);
3137

32-
return rb_class_new_instance(4, argv, rb_cPrismToken);
38+
VALUE argv[] = { source, ID2SYM(type), slice, location };
39+
VALUE value = rb_class_new_instance(4, argv, rb_cPrismToken);
40+
if (freeze) rb_obj_freeze(value);
41+
42+
return value;
3343
}
3444

3545
static VALUE
@@ -68,15 +78,23 @@ pm_integer_new(const pm_integer_t *integer) {
6878

6979
// Create a Prism::Source object from the given parser, after pm_parse() was called.
7080
VALUE
71-
pm_source_new(const pm_parser_t *parser, rb_encoding *encoding) {
81+
pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze) {
7282
VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding);
7383

7484
VALUE offsets = rb_ary_new_capa(parser->newline_list.size);
7585
for (size_t index = 0; index < parser->newline_list.size; index++) {
7686
rb_ary_push(offsets, ULONG2NUM(parser->newline_list.offsets[index]));
7787
}
7888

79-
return rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets);
89+
if (freeze) {
90+
rb_obj_freeze(source_string);
91+
rb_obj_freeze(offsets);
92+
}
93+
94+
VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets);
95+
if (freeze) rb_obj_freeze(source);
96+
97+
return source;
8098
}
8199

82100
typedef struct pm_node_stack_node {
@@ -106,7 +124,7 @@ pm_node_stack_pop(pm_node_stack_node_t **stack) {
106124
}
107125

108126
VALUE
109-
pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source) {
127+
pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) {
110128
VALUE constants = rb_ary_new_capa(parser->constant_pool.size);
111129

112130
for (uint32_t index = 0; index < parser->constant_pool.size; index++) {
@@ -182,7 +200,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
182200
argv[1] = ULONG2NUM(node->node_id);
183201

184202
// location
185-
argv[2] = pm_location_new(parser, node->location.start, node->location.end);
203+
argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze);
186204

187205
// flags
188206
argv[3] = ULONG2NUM(node->flags);
@@ -199,6 +217,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
199217
for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
200218
rb_ary_push(argv[<%= index %>], rb_ary_pop(value_stack));
201219
}
220+
if (freeze) rb_obj_freeze(argv[<%= index %>]);
202221
<%- when Prism::Template::StringField -%>
203222
#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
204223
argv[<%= index %>] = pm_string_new(&cast-><%= field.name %>, encoding);
@@ -215,12 +234,13 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
215234
assert(cast-><%= field.name %>.ids[index] != 0);
216235
rb_ary_push(argv[<%= index %>], RARRAY_AREF(constants, cast-><%= field.name %>.ids[index] - 1));
217236
}
237+
if (freeze) rb_obj_freeze(argv[<%= index %>]);
218238
<%- when Prism::Template::LocationField -%>
219239
#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
220-
argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end);
240+
argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
221241
<%- when Prism::Template::OptionalLocationField -%>
222242
#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
223-
argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end);
243+
argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
224244
<%- when Prism::Template::UInt8Field -%>
225245
#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
226246
argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>);
@@ -238,7 +258,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
238258
<%- end -%>
239259
<%- end -%>
240260

241-
rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 4 %>, argv, rb_cPrism<%= node.name %>));
261+
VALUE value = rb_class_new_instance(<%= node.fields.length + 4 %>, argv, rb_cPrism<%= node.name %>);
262+
if (freeze) rb_obj_freeze(value);
263+
264+
rb_ary_push(value_stack, value);
242265
break;
243266
}
244267
<%- end -%>

0 commit comments

Comments
 (0)