Parser: Treat javascript_tag content as foreign content

Author: marcoroth

ext/herb/extension.c

@@ -156,6 +156,10 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
     if (NIL_P(prism_program)) { prism_program = rb_hash_lookup(options, ID2SYM(rb_intern("prism_program"))); }
     if (!NIL_P(prism_program) && RTEST(prism_program)) { parser_options.prism_program = true; }
 
+    VALUE html = rb_hash_lookup(options, rb_utf8_str_new_cstr("html"));
+    if (NIL_P(html)) { html = rb_hash_lookup(options, ID2SYM(rb_intern("html"))); }
+    if (!NIL_P(html) && !RTEST(html)) { parser_options.html = false; }
+
     VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
     if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
     if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }

java/herb_jni.c

@@ -108,6 +108,14 @@ Java_org_herb_Herb_parse(JNIEnv* env, jclass clazz, jstring source, jobject opti
       jboolean prismProgram = (*env)->CallBooleanMethod(env, options, getPrismProgram);
       parser_options.prism_program = (prismProgram == JNI_TRUE);
     }
+
+    jmethodID getHtml =
+        (*env)->GetMethodID(env, optionsClass, "isHtml", "()Z");
+
+    if (getHtml != NULL) {
+      jboolean html = (*env)->CallBooleanMethod(env, options, getHtml);
+      parser_options.html = (html == JNI_TRUE);
+    }
   }
 
   hb_allocator_T allocator;

java/org/herb/ParserOptions.java

@@ -10,6 +10,7 @@ public class ParserOptions {
   private boolean prismNodes = false;
   private boolean prismNodesDeep = false;
   private boolean prismProgram = false;
+  private boolean html = true;
 
   public ParserOptions() {}
 
@@ -94,6 +95,15 @@ public boolean isPrismProgram() {
     return prismProgram;
   }
 
+  public ParserOptions html(boolean value) {
+    this.html = value;
+    return this;
+  }
+
+  public boolean isHtml() {
+    return html;
+  }
+
   public static ParserOptions create() {
     return new ParserOptions();
   }

javascript/packages/core/src/parser-options.ts

@@ -8,6 +8,7 @@ export interface ParseOptions {
   prism_nodes?: boolean
   prism_nodes_deep?: boolean
   prism_program?: boolean
+  html?: boolean
 }
 
 export type SerializedParserOptions = Required<ParseOptions>
@@ -22,6 +23,7 @@ export const DEFAULT_PARSER_OPTIONS: SerializedParserOptions = {
   prism_nodes: false,
   prism_nodes_deep: false,
   prism_program: false,
+  html: true,
 }
 
 /**
@@ -55,6 +57,9 @@ export class ParserOptions {
   /** Whether the full Prism ProgramNode was serialized on the DocumentNode. */
   readonly prism_program: boolean
 
+  /** Whether HTML tag parsing is enabled during parsing. When false, HTML-like content is treated as literal text. */
+  readonly html: boolean
+
   static from(options: SerializedParserOptions): ParserOptions {
     return new ParserOptions(options)
   }
@@ -69,5 +74,6 @@ export class ParserOptions {
     this.prism_nodes = options.prism_nodes ?? DEFAULT_PARSER_OPTIONS.prism_nodes
     this.prism_nodes_deep = options.prism_nodes_deep ?? DEFAULT_PARSER_OPTIONS.prism_nodes_deep
     this.prism_program = options.prism_program ?? DEFAULT_PARSER_OPTIONS.prism_program
+    this.html = options.html ?? DEFAULT_PARSER_OPTIONS.html
   }
 }

javascript/packages/linter/test/parse-cache.test.ts

@@ -75,6 +75,7 @@ describe("ParseCache", () => {
         strict: true,
         strict_locals: false,
         action_view_helpers: false,
+        html: true,
       })
     })
 
@@ -92,6 +93,7 @@ describe("ParseCache", () => {
         strict: false,
         strict_locals: false,
         action_view_helpers: false,
+        html: true,
       })
     })
 
@@ -109,6 +111,7 @@ describe("ParseCache", () => {
         strict: true,
         strict_locals: false,
         action_view_helpers: false,
+        html: true,
       })
     })
 
@@ -126,6 +129,7 @@ describe("ParseCache", () => {
         strict: false,
         strict_locals: false,
         action_view_helpers: false,
+        html: true,
       })
     })
   })

playground/index.html

@@ -490,6 +490,17 @@
                     />
                     <span class="select-none">Prism nodes (deep)</span>
                   </label>
+
+                  <label class="flex items-center gap-1.5 text-gray-300 text-sm" title="Enable HTML tag parsing — uncheck to treat all HTML-like content as literal text (ERB is still parsed)">
+                    <input
+                      type="checkbox"
+                      data-option="html"
+                      data-action="change->playground#onOptionChange"
+                      class="rounded border-gray-600 text-green-600 focus:ring-green-500 bg-gray-700"
+                      checked
+                    />
+                    <span class="select-none">HTML</span>
+                  </label>
                 </div>
 
                 <pre

playground/src/controllers/playground_controller.js

@@ -1271,6 +1271,7 @@ export default class extends Controller {
       prism_program: false,
       prism_nodes: false,
       prism_nodes_deep: false,
+      html: true,
     }
 
     const nonDefaultOptions = {}

rust/src/herb.rs

@@ -14,6 +14,7 @@ pub struct ParserOptions {
   pub prism_nodes: bool,
   pub prism_nodes_deep: bool,
   pub prism_program: bool,
+  pub html: bool,
 }
 
 impl Default for ParserOptions {
@@ -28,6 +29,7 @@ impl Default for ParserOptions {
       prism_nodes: false,
       prism_nodes_deep: false,
       prism_program: false,
+      html: true,
     }
   }
 }
@@ -109,6 +111,7 @@ pub fn parse_with_options(source: &str, options: &ParserOptions) -> Result<Parse
       prism_program: options.prism_program,
       prism_nodes: options.prism_nodes,
       prism_nodes_deep: options.prism_nodes_deep,
+      html: options.html,
     };
 
     let ast = crate::ffi::herb_parse(c_source.as_ptr(), &c_parser_options, &mut allocator);

src/analyze/action_view/tag_helpers.c

@@ -4,6 +4,7 @@
 #include "../../include/analyze/action_view/tag_helper_node_builders.h"
 #include "../../include/analyze/analyze.h"
 #include "../../include/ast_nodes.h"
+#include "../../include/herb.h"
 #include "../../include/html_util.h"
 #include "../../include/parser_helpers.h"
 #include "../../include/position.h"
@@ -687,23 +688,26 @@ static AST_NODE_T* transform_erb_block_to_tag_helper(
   AST_NODE_T* close_tag = (AST_NODE_T*) block_node->end_node;
 
   if (tag_name && parser_is_foreign_content_tag(hb_string_from_c_string(tag_name)) && context->source
-      && block_node->body && hb_array_size(block_node->body) > 0) {
+      && block_node->body && hb_array_size(block_node->body) > 0 && block_node->end_node
+      && block_node->end_node->tag_opening) {
     size_t start_offset = block_node->tag_closing->range.to;
     size_t end_offset = block_node->end_node->tag_opening->range.from;
 
     if (end_offset > start_offset) {
       position_T body_start = block_node->tag_closing->location.end;
-      position_T body_end = block_node->end_node->tag_opening->location.start;
 
       size_t content_length = end_offset - start_offset;
       char* raw_copy = hb_allocator_strndup(allocator, context->source + start_offset, content_length);
-      hb_string_T raw_content = { .data = raw_copy, .length = (uint32_t) content_length };
 
-      AST_LITERAL_NODE_T* literal_node =
-        ast_literal_node_init(raw_content, body_start, body_end, hb_array_init(0, allocator), allocator);
+      parser_options_T body_options = HERB_DEFAULT_PARSER_OPTIONS;
+      body_options.html = false;
+      body_options.analyze = false;
+      body_options.strict = false;
+      body_options.start_line = body_start.line;
+      body_options.start_column = body_start.column;
 
-      body = hb_array_init(1, allocator);
-      hb_array_append(body, literal_node);
+      AST_DOCUMENT_NODE_T* body_document = herb_parse(raw_copy, &body_options, allocator);
+      body = body_document->children;
     }
   }
 

src/herb.c

@@ -44,6 +44,16 @@ HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(
   parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
   if (options != NULL) { parser_options = *options; }
 
+  if (parser_options.start_line > 0) {
+    lexer.current_line = parser_options.start_line;
+    lexer.previous_line = parser_options.start_line;
+  }
+
+  if (parser_options.start_column > 0) {
+    lexer.current_column = parser_options.start_column;
+    lexer.previous_column = parser_options.start_column;
+  }
+
   herb_parser_init(&parser, &lexer, parser_options);
 
   AST_DOCUMENT_NODE_T* document = herb_parser_parse(&parser);