Parser: Detect javascript_tag and javascript_include_tag helpers (#1374)

This pull request allows the parser to be able to detect and transform
the `javascript_tag` and `javascript_include_tag` helpers using the
`action_view_helpers` parser option.

For example:
```html+erb
<%= javascript_tag do %>
  alert("Hello")
<% end %>
```

Gets parsed as:

```js
@ DocumentNode (location: (1:0)-(4:0))
└── children: (2 items)
    ├── @ HTMLElementNode (location: (1:0)-(3:9))
    │   ├── open_tag:
    │   │   └── @ ERBOpenTagNode (location: (1:0)-(1:24))
    │   │       ├── tag_opening: "<%=" (location: (1:0)-(1:3))
    │   │       ├── content: " javascript_tag do " (location: (1:3)-(1:22))
    │   │       ├── tag_closing: "%>" (location: (1:22)-(1:24))
    │   │       ├── tag_name: "script" (location: (1:4)-(1:18))
    │   │       └── children: []
    │   │
    │   ├── tag_name: "script" (location: (1:4)-(1:18))
    │   ├── body: (1 item)
    │   │   └── @ LiteralNode (location: (1:24)-(3:0))
    │   │       └── content: "\n  alert(\"Hello\")\n"
    │   │
    │   ├── close_tag:
    │   │   └── @ ERBEndNode (location: (3:0)-(3:9))
    │   │       ├── tag_opening: "<%" (location: (3:0)-(3:2))
    │   │       ├── content: " end " (location: (3:2)-(3:7))
    │   │       └── tag_closing: "%>" (location: (3:7)-(3:9))
    │   │
    │   ├── is_void: false
    │   └── element_source: "ActionView::Helpers::JavaScriptHelper#javascript_tag"
    │
    └── @ HTMLTextNode (location: (3:9)-(4:0))
        └── content: "\n"
```

This pull request also updates the rewriters to be able to rewrite
from/to the `javascript_tag` syntax. The above example can be
transformed to this and back:

```html+erb
<script>
  alert('Hello')
</script>
```

And for `javascript_include_tag`, the following: 

```erb
<%= javascript_include_tag "common.javascript", "/elsewhere/cools" %>
```

gets transformed to:
```erb
<script src="<%= javascript_path("common.javascript") %>"></script>
<script src="<%= javascript_path("/elsewhere/cools") %>"></script>
```

Follow up on #1122 
Follow up on #1354 

Fixes #991

Author: marcoroth

javascript/packages/core/src/html-constants.ts

@@ -0,0 +1,13 @@
+// https://html.spec.whatwg.org/multipage/common-microsyntaxes.html#boolean-attributes
+export const HTML_BOOLEAN_ATTRIBUTES = new Set([
+  "allowfullscreen", "async", "autofocus", "autoplay", "checked", "compact",
+  "controls", "declare", "default", "defer", "disabled", "formnovalidate",
+  "hidden", "inert", "ismap", "itemscope", "loop", "multiple", "muted",
+  "nomodule", "nohref", "noresize", "noshade", "novalidate", "nowrap",
+  "open", "playsinline", "readonly", "required", "reversed", "scoped",
+  "seamless", "selected", "sortable", "truespeed", "typemustmatch",
+])
+
+export function isBooleanAttribute(attributeName: string): boolean {
+  return HTML_BOOLEAN_ATTRIBUTES.has(attributeName.toLowerCase())
+}

javascript/packages/core/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./ast-utils.js"
+export * from "./html-constants.js"
 export * from "./backend.js"
 export * from "./diagnostic.js"
 export * from "./didyoumean.js"

javascript/packages/language-server/src/action_view_helpers.ts

@@ -20,4 +20,12 @@ export const ACTION_VIEW_HELPERS: Record<string, ActionViewHelperInfo> = {
     signature: "turbo_frame_tag(*ids, src: nil, target: nil, **attributes, &block)",
     documentationURL: "https://www.rubydoc.info/github/hotwired/turbo-rails/Turbo/FramesHelper:turbo_frame_tag",
   },
+  "ActionView::Helpers::JavaScriptHelper#javascript_tag": {
+    signature: "javascript_tag(content_or_options_with_block = nil, html_options = {}, &block)",
+    documentationURL: "https://api.rubyonrails.org/classes/ActionView/Helpers/JavaScriptHelper.html#method-i-javascript_tag",
+  },
+  "ActionView::Helpers::AssetTagHelper#javascript_include_tag": {
+    signature: "javascript_include_tag(*sources)",
+    documentationURL: "https://api.rubyonrails.org/classes/ActionView/Helpers/AssetTagHelper.html#method-i-javascript_include_tag",
+  },
 }

javascript/packages/linter/src/rules/rule-utils.ts

@@ -188,13 +188,7 @@ export const HTML_VOID_ELEMENTS = new Set([
   "param", "source", "track", "wbr",
 ])
 
-export const HTML_BOOLEAN_ATTRIBUTES = new Set([
-  "autofocus", "autoplay", "checked", "controls", "defer", "disabled", "hidden",
-  "loop", "multiple", "muted", "readonly", "required", "reversed", "selected",
-  "open", "default", "formnovalidate", "novalidate", "itemscope", "scoped",
-  "seamless", "allowfullscreen", "async", "compact", "declare", "nohref",
-  "noresize", "noshade", "nowrap", "sortable", "truespeed", "typemustmatch"
-])
+export { HTML_BOOLEAN_ATTRIBUTES, isBooleanAttribute } from "@herb-tools/core"
 
 export const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6"])
 
@@ -401,13 +395,6 @@ export function isVoidElement(tagName: string): boolean {
   return HTML_VOID_ELEMENTS.has(tagName.toLowerCase())
 }
 
-/**
- * Checks if an attribute is a boolean attribute
- */
-export function isBooleanAttribute(attributeName: string): boolean {
-  return HTML_BOOLEAN_ATTRIBUTES.has(attributeName.toLowerCase())
-}
-
 /**
  * Attribute visitor that provides granular processing based on both
  * attribute name type (static/dynamic) and value type (static/dynamic)

javascript/packages/node/binding.gyp

@@ -25,6 +25,8 @@
         "./extension/libherb/analyze/transform.c",
         "./extension/libherb/analyze/action_view/attribute_extraction_helpers.c",
         "./extension/libherb/analyze/action_view/content_tag.c",
+        "./extension/libherb/analyze/action_view/javascript_include_tag.c",
+        "./extension/libherb/analyze/action_view/javascript_tag.c",
         "./extension/libherb/analyze/action_view/link_to.c",
         "./extension/libherb/analyze/action_view/registry.c",
         "./extension/libherb/analyze/action_view/tag_helper_node_builders.c",

javascript/packages/rewriter/src/built-ins/action-view-tag-helper-to-html.ts

@@ -166,7 +166,7 @@ export class ActionViewTagHelperToHTMLRewriter extends ASTRewriter {
   }
 
   get description(): string {
-    return "Converts ActionView tag helpers (tag.*, content_tag, link_to, turbo_frame_tag) to raw HTML elements"
+    return "Converts ActionView tag helpers to raw HTML elements"
   }
 
   rewrite<T extends Node>(node: T, _context: RewriteContext): T {

javascript/packages/rewriter/src/built-ins/html-to-action-view-tag-helper.ts

@@ -35,14 +35,16 @@ interface SerializedAttributes {
   attributes: string
   href: string | null
   id: string | null
+  src: string | null
 }
 
-function serializeAttributes(children: Node[], options: { extractHref?: boolean, extractId?: boolean } = {}): SerializedAttributes {
+function serializeAttributes(children: Node[], options: { extractHref?: boolean, extractId?: boolean, extractSrc?: boolean } = {}): SerializedAttributes {
   const regular: string[] = []
   const prefixed: Map<string, string[]> = new Map()
 
   let href: string | null = null
   let id: string | null = null
+  let src: string | null = null
 
   for (const child of children) {
     if (!isHTMLAttributeNode(child)) continue
@@ -62,6 +64,11 @@ function serializeAttributes(children: Node[], options: { extractHref?: boolean,
       continue
     }
 
+    if (options.extractSrc && name === "src") {
+      src = value
+      continue
+    }
+
     const dataMatch = name.match(/^(data|aria)-(.+)$/)
 
     if (dataMatch) {
@@ -83,7 +90,7 @@ function serializeAttributes(children: Node[], options: { extractHref?: boolean,
     parts.push(`${prefix}: { ${entries.join(", ")} }`)
   }
 
-  return { attributes: parts.join(", "), href, id }
+  return { attributes: parts.join(", "), href, id, src }
 }
 
 function isTextOnlyBody(body: Node[]): boolean {
@@ -116,8 +123,10 @@ class HTMLToActionViewTagHelperVisitor extends Visitor {
 
     const isAnchor = tagName.value === "a"
     const isTurboFrame = tagName.value === "turbo-frame"
+    const isScript = tagName.value === "script"
     const attributes = openTag.children.filter(child => !isWhitespaceNode(child))
-    const { attributes: attributesString, href, id } = serializeAttributes(attributes, { extractHref: isAnchor, extractId: isTurboFrame })
+    const hasSrcAttribute = isScript && attributes.some(child => isHTMLAttributeNode(child) && getStaticAttributeName(child.name!) === "src")
+    const { attributes: attributesString, href, id, src } = serializeAttributes(attributes, { extractHref: isAnchor, extractId: isTurboFrame, extractSrc: isScript })
     const hasBody = node.body && node.body.length > 0 && !node.is_void
     const isInlineContent = hasBody && isTextOnlyBody(node.body)
 
@@ -130,6 +139,12 @@ class HTMLToActionViewTagHelperVisitor extends Visitor {
     } else if (isTurboFrame) {
       content = this.buildTurboFrameTagContent(node, attributesString, id, isInlineContent)
       elementSource = "Turbo::FramesHelper#turbo_frame_tag"
+    } else if (isScript && hasSrcAttribute) {
+      content = this.buildJavascriptIncludeTagContent(attributesString, src)
+      elementSource = "ActionView::Helpers::AssetTagHelper#javascript_include_tag"
+    } else if (isScript) {
+      content = this.buildJavascriptTagContent(node, attributesString, isInlineContent)
+      elementSource = "ActionView::Helpers::JavaScriptHelper#javascript_tag"
     } else {
       content = this.buildTagContent(tagName.value, node, attributesString, isInlineContent)
       elementSource = "ActionView::Helpers::TagHelper#tag"
@@ -149,7 +164,8 @@ class HTMLToActionViewTagHelperVisitor extends Visitor {
     asMutable(node).open_tag = erbOpenTag
     asMutable(node).element_source = elementSource
 
-    const isInlineForm = isInlineContent || (isTurboFrame && !hasBody)
+    const isInlineLiteralContent = isScript && hasBody && node.body.length === 1 && isLiteralNode(node.body[0]) && !node.body[0].content.includes("\n")
+    const isInlineForm = isInlineContent || isInlineLiteralContent || (isTurboFrame && !hasBody) || (isScript && hasSrcAttribute)
 
     if (node.is_void) {
       asMutable(node).close_tag = null
@@ -224,6 +240,36 @@ class HTMLToActionViewTagHelperVisitor extends Visitor {
     return argString ? ` turbo_frame_tag ${argString} do ` : ` turbo_frame_tag do `
   }
 
+  private buildJavascriptTagContent(node: HTMLElementNode, attributes: string, isInlineContent: boolean): string {
+    const bodyNode = node.body?.[0]
+    const isInlineLiteral = bodyNode && isLiteralNode(bodyNode) && !bodyNode.content.includes("\n")
+    const isInlineText = isInlineContent && isHTMLTextNode(bodyNode)
+
+    if (isInlineText || isInlineLiteral) {
+      const textContent = isHTMLTextNode(bodyNode) ? bodyNode.content : bodyNode.content
+      const args = [`"${textContent}"`]
+
+      if (attributes) args.push(attributes)
+
+      return ` javascript_tag ${args.join(", ")} `
+    }
+
+    return attributes
+      ? ` javascript_tag ${attributes} do `
+      : ` javascript_tag do `
+  }
+
+  private buildJavascriptIncludeTagContent(attributes: string, source: string | null): string {
+    const args: string[] = []
+
+    if (source) args.push(source)
+    if (attributes) args.push(attributes)
+
+    const argString = args.join(", ")
+
+    return argString ? ` javascript_include_tag ${argString} ` : ` javascript_include_tag `
+  }
+
   private buildLinkToContent(node: HTMLElementNode, attribute: string, href: string | null, isInlineContent: boolean): string {
     const args: string[] = []
 
@@ -255,7 +301,7 @@ export class HTMLToActionViewTagHelperRewriter extends ASTRewriter {
   }
 
   get description(): string {
-    return "Converts raw HTML elements to ActionView tag helpers (tag.*, turbo_frame_tag)"
+    return "Converts raw HTML elements to ActionView tag helpers (tag.*, turbo_frame_tag, javascript_tag, javascript_include_tag)"
   }
 
   rewrite<T extends Node>(node: T, _context: RewriteContext): T {

javascript/packages/rewriter/test/action-view-tag-helper-to-html.test.ts

@@ -396,6 +396,158 @@ describe("ActionViewTagHelperToHTMLRewriter", () => {
     })
   })
 
+  describe("javascript_tag helpers", () => {
+    test("javascript_tag with content as argument", () => {
+      expect(transform(`<%= javascript_tag "alert('Hello')" %>`)).toBe(
+        `<script>alert('Hello')</script>`
+      )
+    })
+
+    test("javascript_tag with block", () => {
+      const input = dedent`
+        <%= javascript_tag do %>
+          alert('Hello')
+        <% end %>
+      `
+
+      const expected = dedent`
+        <script>
+          alert('Hello')
+        </script>
+      `
+
+      expect(transform(input)).toBe(expected)
+    })
+
+    test("javascript_tag with type attribute", () => {
+      expect(transform(`<%= javascript_tag "alert('Hello')", type: "application/javascript" %>`)).toBe(
+        `<script type="application/javascript">alert('Hello')</script>`
+      )
+    })
+  })
+
+  describe("javascript_include_tag helpers", () => {
+    test("javascript_include_tag with single source", () => {
+      expect(transform(`<%= javascript_include_tag "application" %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>"></script>`
+      )
+    })
+
+    test("javascript_include_tag with defer", () => {
+      expect(transform(`<%= javascript_include_tag "application", defer: true %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>" defer></script>`
+      )
+    })
+
+    test("javascript_include_tag with multiple sources", () => {
+      expect(transform(`<%= javascript_include_tag "application", "vendor" %>`)).toBe(
+        dedent`
+          <script src="<%= javascript_path("application") %>"></script>
+          <script src="<%= javascript_path("vendor") %>"></script>
+        `
+      )
+    })
+
+    test("javascript_include_tag with nonce", () => {
+      expect(transform(`<%= javascript_include_tag "application", nonce: true %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>" nonce="true"></script>`
+      )
+    })
+
+    test("javascript_include_tag with nonce false", () => {
+      expect(transform(`<%= javascript_include_tag "application", nonce: false %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>" nonce="false"></script>`
+      )
+    })
+
+    test("javascript_include_tag with interpolated nonce", () => {
+      expect(transform('<%= javascript_include_tag "application", nonce: "static-#{dynamic}" %>')).toBe(
+        '<script src="<%= javascript_path("application") %>" nonce="static-<%= dynamic %>"></script>'
+      )
+    })
+
+    test("javascript_include_tag with data attributes", () => {
+      expect(transform(`<%= javascript_include_tag "application", data: { turbo_track: "reload" } %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>" data-turbo-track="reload"></script>`
+      )
+    })
+
+    test("javascript_include_tag with .js extension", () => {
+      expect(transform(`<%= javascript_include_tag "xmlhr.js" %>`)).toBe(
+        `<script src="<%= javascript_path("xmlhr.js") %>"></script>`
+      )
+    })
+
+    test("javascript_include_tag with URL", () => {
+      expect(transform(`<%= javascript_include_tag "http://www.example.com/xmlhr" %>`)).toBe(
+        `<script src="http://www.example.com/xmlhr"></script>`
+      )
+    })
+
+    test("javascript_include_tag with URL ending in .js", () => {
+      expect(transform(`<%= javascript_include_tag "http://www.example.com/xmlhr.js" %>`)).toBe(
+        `<script src="http://www.example.com/xmlhr.js"></script>`
+      )
+    })
+
+    test("javascript_include_tag with protocol-relative URL", () => {
+      expect(transform(`<%= javascript_include_tag "//cdn.example.com/app.js" %>`)).toBe(
+        `<script src="//cdn.example.com/app.js"></script>`
+      )
+    })
+
+    test("javascript_include_tag with URL and nonce", () => {
+      expect(transform(`<%= javascript_include_tag "http://www.example.com/xmlhr.js", nonce: true %>`)).toBe(
+        `<script src="http://www.example.com/xmlhr.js" nonce="true"></script>`
+      )
+    })
+
+    test("javascript_include_tag with URL and async", () => {
+      expect(transform(`<%= javascript_include_tag "http://www.example.com/xmlhr.js", async: true %>`)).toBe(
+        `<script src="http://www.example.com/xmlhr.js" async></script>`
+      )
+    })
+
+    test("javascript_include_tag with URL and defer", () => {
+      expect(transform(`<%= javascript_include_tag "http://www.example.com/xmlhr.js", defer: true %>`)).toBe(
+        `<script src="http://www.example.com/xmlhr.js" defer></script>`
+      )
+    })
+
+    test("javascript_include_tag with defer as string", () => {
+      expect(transform(`<%= javascript_include_tag "application", defer: "true" %>`)).toBe(
+        `<script src="<%= javascript_path("application") %>" defer="true"></script>`
+      )
+    })
+
+    test("javascript_include_tag with extname false", () => {
+      expect(transform(`<%= javascript_include_tag "template.jst", extname: false %>`)).toBe(
+        `<script src="<%= javascript_path("template.jst") %>" extname="false"></script>`
+      )
+    })
+
+    test("javascript_include_tag with host and protocol", () => {
+      expect(transform(`<%= javascript_include_tag "xmlhr", host: "localhost", protocol: "https" %>`)).toBe(
+        `<script src="<%= javascript_path("xmlhr") %>" host="localhost" protocol="https"></script>`
+      )
+    })
+
+    test("javascript_include_tag with multiple sources including path", () => {
+      expect(transform(`<%= javascript_include_tag "common.javascript", "/elsewhere/cools" %>`)).toBe(
+        dedent`
+          <script src="<%= javascript_path("common.javascript") %>"></script>
+          <script src="<%= javascript_path("/elsewhere/cools") %>"></script>
+        `
+      )
+    })
+
+    test("javascript_include_tag with asset_path", () => {
+      expect(transform(`<%= javascript_include_tag asset_path("application.js") %>`)).toBe(
+        `<script src="<%= asset_path("application.js") %>"></script>`
+      )
+    })
+  })
+
   describe("non-ActionView elements", () => {
     test("regular HTML elements are not modified", () => {
       expect(transform('<div class="content">Hello</div>')).toBe(