[WIP] Hydrate components immediately after downloading chunks (#1656)

* hydrate the component immediately when loaded and registered
* auto register server components and immediately hydrate stores
* move react-server-dom-webpack.d.ts to types directory
* ensure to initialize registered stores array before accessing
* refactor registration callback into separate class
* make the early hydration compatible with turbopack, backward compatible and refactor
* pass rsc path to RSC Client Root and move the config to RORP
* update min node version to 18
* export registerServerComponent as a separate entrypoint to avoid client bundle increase
* Update webpack assets status checker to use server bundle configuration
* Update webpack assets status checker to handle bundle file paths
* [WIP] handle errors happen in rsc payload (#1663)
* stream rsc payload in json objects like streamed react components
* make path to rsc bundle and react client manifest configurable

* feat: Improve client manifest path handling for dev server

- Add `dev_server_url` helper to centralize dev server URL construction
- Add `public_output_uri_path` to get relative webpack output path
- Add `asset_uri_from_packer` to handle asset URIs consistently
- Update `react_client_manifest_file_path` to return dev server URLs when appropriate
- Add comprehensive specs for new asset URI handling

This change ensures client manifest paths are properly resolved to dev server
URLs during development, improving hot-reloading functionality.

* fix: normalize RSC URL path by absorbing leading/trailing slashes
* specify Shakapacker as top-level module
* add tests for RSCClientRoot
* Make RSCClientRoot tests run with react 18
* Update webpack asset path configuration for client manifest

---------

Co-authored-by: Judah Meek <[email protected]>

Author: AbanoubGhadban

.github/workflows/main.yml

@@ -28,7 +28,7 @@ jobs:
     - name: Setup Node
       uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.versions == 'oldest' && '16' || '20' }}
+        node-version: ${{ matrix.versions == 'oldest' && '18' || '20' }}
     - name: Print system information
       run: |
         echo "Linux release: "; cat /etc/issue
@@ -104,7 +104,7 @@ jobs:
     - name: Setup Node
       uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.versions == 'oldest' && '16' || '20' }}
+        node-version: ${{ matrix.versions == 'oldest' && '18' || '20' }}
     - name: Print system information
       run: |
         echo "Linux release: "; cat /etc/issue

.github/workflows/package-js-tests.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Setup Node
       uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.versions == 'oldest' && '16' || '20' }}
+        node-version: ${{ matrix.versions == 'oldest' && '18' || '20' }}
     - name: Print system information
       run: |
         echo "Linux release: "; cat /etc/issue
@@ -43,3 +43,6 @@ jobs:
         sudo yarn global add yalc
     - name: Run JS unit tests for Renderer package
       run: yarn test
+    # TODO: Remove this once we made these tests compatible with React 19
+    - name: Run JS unit tests for Renderer package with React 18 (for tests not compatible with React 19)
+      run: yarn test:react-18

Gemfile.lock

@@ -385,11 +385,6 @@ GEM
       nokogiri (~> 1.6)
       rubyzip (>= 1.3.0)
       selenium-webdriver (~> 4.0, < 4.11)
-    webpacker (6.0.0.rc.6)
-      activesupport (>= 5.2)
-      rack-proxy (>= 0.6.1)
-      railties (>= 5.2)
-      semantic_range (>= 2.3.0)
     webrick (1.8.1)
     websocket (1.2.10)
     websocket-driver (0.7.6)
@@ -444,7 +439,6 @@ DEPENDENCIES
   turbolinks
   uglifier
   webdrivers (= 5.3.0)
-  webpacker (= 6.0.0.rc.6)
 
 BUNDLED WITH
    2.5.9

jest.config.js

@@ -2,4 +2,15 @@ module.exports = {
   preset: 'ts-jest/presets/js-with-ts',
   testEnvironment: 'jsdom',
   setupFiles: ['<rootDir>/node_package/tests/jest.setup.js'],
+  // TODO: Remove this once we made RSCClientRoot compatible with React 19
+  moduleNameMapper: process.env.USE_REACT_18
+    ? {
+        '^react$': '<rootDir>/node_modules/react-18',
+        '^react/(.*)$': '<rootDir>/node_modules/react-18/$1',
+        '^react-dom$': '<rootDir>/node_modules/react-dom-18',
+        '^react-dom/(.*)$': '<rootDir>/node_modules/react-dom-18/$1',
+      }
+    : {
+        'react-server-dom-webpack/client': '<rootDir>/node_package/tests/emptyForTesting.js',
+      },
 };

knip.ts

@@ -9,6 +9,8 @@ const config: KnipConfig = {
         'node_package/src/ReactOnRails.node.ts!',
         'node_package/src/ReactOnRailsRSC.ts!',
         'node_package/src/RSCWebpackLoader.js!',
+        'node_package/src/registerServerComponent.ts!',
+        'node_package/src/RSCClientRoot.ts!',
       ],
       project: ['node_package/src/**/*.[jt]s!', 'node_package/tests/**/*.[jt]s'],
       babel: {
@@ -32,6 +34,10 @@ const config: KnipConfig = {
         // Used in CI
         '@arethetypeswrong/cli',
         'react-server-dom-webpack',
+        'cross-fetch',
+        'jsdom',
+        'react-18',
+        'react-dom-18',
       ],
     },
     'spec/dummy': {

lib/react_on_rails/configuration.rb

@@ -9,6 +9,7 @@ def self.configure
   end
 
   DEFAULT_GENERATED_ASSETS_DIR = File.join(%w[public webpack], Rails.env).freeze
+  DEFAULT_REACT_CLIENT_MANIFEST_FILE = "react-client-manifest.json"
 
   def self.configuration
     @configuration ||= Configuration.new(
@@ -18,6 +19,7 @@ def self.configuration
       generated_assets_dir: "",
       server_bundle_js_file: "",
       rsc_bundle_js_file: "",
+      react_client_manifest_file: DEFAULT_REACT_CLIENT_MANIFEST_FILE,
       prerender: false,
       auto_load_bundle: false,
       replay_console: true,
@@ -57,8 +59,8 @@ class Configuration
                   :server_render_method, :random_dom_id, :auto_load_bundle,
                   :same_bundle_for_client_and_server, :rendering_props_extension,
                   :make_generated_server_bundle_the_entrypoint,
-                  :defer_generated_component_packs, :rsc_bundle_js_file,
-                  :force_load
+                  :defer_generated_component_packs, :force_load, :rsc_bundle_js_file,
+                  :react_client_manifest_file
 
     # rubocop:disable Metrics/AbcSize
     def initialize(node_modules_location: nil, server_bundle_js_file: nil, prerender: nil,
@@ -74,7 +76,7 @@ def initialize(node_modules_location: nil, server_bundle_js_file: nil, prerender
                    i18n_dir: nil, i18n_yml_dir: nil, i18n_output_format: nil, i18n_yml_safe_load_options: nil,
                    random_dom_id: nil, server_render_method: nil, rendering_props_extension: nil,
                    components_subdirectory: nil, auto_load_bundle: nil, force_load: nil,
-                   rsc_bundle_js_file: nil)
+                   rsc_bundle_js_file: nil, react_client_manifest_file: nil)
       self.node_modules_location = node_modules_location.present? ? node_modules_location : Rails.root
       self.generated_assets_dirs = generated_assets_dirs
       self.generated_assets_dir = generated_assets_dir
@@ -102,6 +104,7 @@ def initialize(node_modules_location: nil, server_bundle_js_file: nil, prerender
       # Server rendering:
       self.server_bundle_js_file = server_bundle_js_file
       self.rsc_bundle_js_file = rsc_bundle_js_file
+      self.react_client_manifest_file = react_client_manifest_file
       self.same_bundle_for_client_and_server = same_bundle_for_client_and_server
       self.server_renderer_pool_size = self.development_mode ? 1 : server_renderer_pool_size
       self.server_renderer_timeout = server_renderer_timeout # seconds
@@ -247,6 +250,8 @@ def ensure_webpack_generated_files_exists
       files = ["manifest.json"]
       files << server_bundle_js_file if server_bundle_js_file.present?
       files << rsc_bundle_js_file if rsc_bundle_js_file.present?
+      files << react_client_manifest_file if react_client_manifest_file.present?
+
       self.webpack_generated_files = files
     end
 

lib/react_on_rails/helper.rb

@@ -124,6 +124,7 @@ def react_component(component_name, options = {})
     # @option options [Boolean] :raise_on_prerender_error Set to true to raise exceptions during server-side rendering
     # Any other options are passed to the content tag, including the id.
     def stream_react_component(component_name, options = {})
+      options = options.merge(force_load: true) unless options.key?(:force_load)
       run_stream_inside_fiber do
         internal_stream_react_component(component_name, options)
       end
@@ -193,17 +194,18 @@ def react_component_hash(component_name, options = {})
     # props: Ruby Hash or JSON string which contains the properties to pass to the redux store.
     # Options
     #    defer: false -- pass as true if you wish to render this below your component.
-    def redux_store(store_name, props: {}, defer: false)
+    #    force_load: false -- pass as true if you wish to hydrate this store immediately instead of
+    #                        waiting for the page to load.
+    def redux_store(store_name, props: {}, defer: false, force_load: false)
       redux_store_data = { store_name: store_name,
-                           props: props }
+                           props: props,
+                           force_load: force_load }
       if defer
-        @registered_stores_defer_render ||= []
-        @registered_stores_defer_render << redux_store_data
+        registered_stores_defer_render << redux_store_data
         "YOU SHOULD NOT SEE THIS ON YOUR VIEW -- Uses as a code block, like <% redux_store %> " \
           "and not <%= redux store %>"
       else
-        @registered_stores ||= []
-        @registered_stores << redux_store_data
+        registered_stores << redux_store_data
         result = render_redux_store_data(redux_store_data)
         prepend_render_rails_context(result)
       end
@@ -215,9 +217,9 @@ def redux_store(store_name, props: {}, defer: false)
     # client side rendering of this hydration data, which is a hidden div with a matching class
     # that contains a data props.
     def redux_store_hydration_data
-      return if @registered_stores_defer_render.blank?
+      return if registered_stores_defer_render.blank?
 
-      @registered_stores_defer_render.reduce(+"") do |accum, redux_store_data|
+      registered_stores_defer_render.reduce(+"") do |accum, redux_store_data|
         accum << render_redux_store_data(redux_store_data)
       end.html_safe
     end
@@ -400,6 +402,25 @@ def run_stream_inside_fiber
       rendering_fiber.resume
     end
 
+    def registered_stores
+      @registered_stores ||= []
+    end
+
+    def registered_stores_defer_render
+      @registered_stores_defer_render ||= []
+    end
+
+    def registered_stores_including_deferred
+      registered_stores + registered_stores_defer_render
+    end
+
+    def create_render_options(react_component_name, options)
+      # If no store dependencies are passed, default to all registered stores up till now
+      options[:store_dependencies] ||= registered_stores_including_deferred.map { |store| store[:store_name] }
+      ReactOnRails::ReactComponent::RenderOptions.new(react_component_name: react_component_name,
+                                                      options: options)
+    end
+
     def internal_stream_react_component(component_name, options = {})
       options = options.merge(stream?: true)
       result = internal_react_component(component_name, options)
@@ -415,7 +436,7 @@ def internal_rsc_react_component(react_component_name, options = {})
       render_options = create_render_options(react_component_name, options)
       json_stream = server_rendered_react_component(render_options)
       json_stream.transform do |chunk|
-        chunk[:html].html_safe
+        "#{chunk.to_json}\n".html_safe
       end
     end
 
@@ -510,7 +531,8 @@ def build_react_component_result_for_server_rendered_hash(
       )
     end
 
-    def compose_react_component_html_with_spec_and_console(component_specification_tag, rendered_output, console_script)
+    def compose_react_component_html_with_spec_and_console(component_specification_tag, rendered_output,
+                                                           console_script)
       # IMPORTANT: Ensure that we mark string as html_safe to avoid escaping.
       html_content = <<~HTML
         #{rendered_output}
@@ -546,18 +568,20 @@ def internal_react_component(react_component_name, options = {})
       # (re-hydrate the data). This enables react rendered on the client to see that the
       # server has already rendered the HTML.
 
-      render_options = ReactOnRails::ReactComponent::RenderOptions.new(react_component_name: react_component_name,
-                                                                       options: options)
+      render_options = create_render_options(react_component_name, options)
 
       # Setup the page_loaded_js, which is the same regardless of prerendering or not!
       # The reason is that React is smart about not doing extra work if the server rendering did its job.
       component_specification_tag = content_tag(:script,
                                                 json_safe_and_pretty(render_options.client_props).html_safe,
                                                 type: "application/json",
                                                 class: "js-react-on-rails-component",
+                                                id: "js-react-on-rails-component-#{render_options.dom_id}",
                                                 "data-component-name" => render_options.react_component_name,
                                                 "data-trace" => (render_options.trace ? true : nil),
-                                                "data-dom-id" => render_options.dom_id)
+                                                "data-dom-id" => render_options.dom_id,
+                                                "data-store-dependencies" => render_options.store_dependencies.to_json,
+                                                "data-force-load" => (render_options.force_load ? true : nil))
 
       if render_options.force_load
         component_specification_tag.concat(
@@ -579,12 +603,22 @@ def internal_react_component(react_component_name, options = {})
     end
 
     def render_redux_store_data(redux_store_data)
-      result = content_tag(:script,
-                           json_safe_and_pretty(redux_store_data[:props]).html_safe,
-                           type: "application/json",
-                           "data-js-react-on-rails-store" => redux_store_data[:store_name].html_safe)
+      store_hydration_data = content_tag(:script,
+                                         json_safe_and_pretty(redux_store_data[:props]).html_safe,
+                                         type: "application/json",
+                                         "data-js-react-on-rails-store" => redux_store_data[:store_name].html_safe,
+                                         "data-force-load" => (redux_store_data[:force_load] ? true : nil))
+
+      if redux_store_data[:force_load]
+        store_hydration_data.concat(
+          content_tag(:script, <<~JS.strip_heredoc.html_safe
+            ReactOnRails.reactOnRailsStoreLoaded('#{redux_store_data[:store_name]}');
+          JS
+          )
+        )
+      end
 
-      prepend_render_rails_context(result)
+      prepend_render_rails_context(store_hydration_data)
     end
 
     def props_string(props)
@@ -641,7 +675,7 @@ def server_rendered_react_component(render_options) # rubocop:disable Metrics/Cy
       js_code = ReactOnRails::ServerRenderingJsCode.server_rendering_component_js_code(
         props_string: props_string(props).gsub("\u2028", '\u2028').gsub("\u2029", '\u2029'),
         rails_context: rails_context(server_side: true).to_json,
-        redux_stores: initialize_redux_stores,
+        redux_stores: initialize_redux_stores(render_options),
         react_component_name: react_component_name,
         render_options: render_options
       )
@@ -657,10 +691,7 @@ def server_rendered_react_component(render_options) # rubocop:disable Metrics/Cy
                                                js_code: js_code)
       end
 
-      # TODO: handle errors for rsc streams
-      return result if render_options.rsc?
-
-      if render_options.stream?
+      if render_options.stream? || render_options.rsc?
         result.transform do |chunk_json_result|
           if should_raise_streaming_prerender_error?(chunk_json_result, render_options)
             raise_prerender_error(chunk_json_result, react_component_name, props, js_code)
@@ -675,17 +706,20 @@ def server_rendered_react_component(render_options) # rubocop:disable Metrics/Cy
       result
     end
 
-    def initialize_redux_stores
+    def initialize_redux_stores(render_options)
       result = +<<-JS
       ReactOnRails.clearHydratedStores();
       JS
 
-      return result unless @registered_stores.present? || @registered_stores_defer_render.present?
+      store_dependencies = render_options.store_dependencies
+      return result unless store_dependencies.present?
 
       declarations = +"var reduxProps, store, storeGenerator;\n"
-      all_stores = (@registered_stores || []) + (@registered_stores_defer_render || [])
+      store_objects = registered_stores_including_deferred.select do |store|
+        store_dependencies.include?(store[:store_name])
+      end
 
-      result << all_stores.each_with_object(declarations) do |redux_store_data, memo|
+      result << store_objects.each_with_object(declarations) do |redux_store_data, memo|
         store_name = redux_store_data[:store_name]
         props = props_string(redux_store_data[:props])
         memo << <<-JS.strip_heredoc

lib/react_on_rails/packer_utils.rb

@@ -45,6 +45,10 @@ def self.dev_server_running?
       packer.dev_server.running?
     end
 
+    def self.dev_server_url
+      "#{packer.dev_server.protocol}://#{packer.dev_server.host_with_port}"
+    end
+
     def self.shakapacker_version
       return @shakapacker_version if defined?(@shakapacker_version)
       return nil unless ReactOnRails::Utils.gem_available?("shakapacker")
@@ -79,12 +83,27 @@ def self.bundle_js_uri_from_packer(bundle_name)
 
       if packer.dev_server.running? && (!is_bundle_running_on_server ||
         ReactOnRails.configuration.same_bundle_for_client_and_server)
-        "#{packer.dev_server.protocol}://#{packer.dev_server.host_with_port}#{hashed_bundle_name}"
+        "#{dev_server_url}#{hashed_bundle_name}"
       else
         File.expand_path(File.join("public", hashed_bundle_name)).to_s
       end
     end
 
+    def self.public_output_uri_path
+      "#{packer.config.public_output_path.relative_path_from(packer.config.public_path)}/"
+    end
+
+    # The function doesn't ensure that the asset exists.
+    # - It just returns url to the asset if dev server is running
+    # - Otherwise it returns file path to the asset
+    def self.asset_uri_from_packer(asset_name)
+      if dev_server_running?
+        "#{dev_server_url}/#{public_output_uri_path}#{asset_name}"
+      else
+        File.join(packer_public_output_path, asset_name).to_s
+      end
+    end
+
     def self.precompile?
       return ::Webpacker.config.webpacker_precompile? if using_webpacker_const?
       return ::Shakapacker.config.shakapacker_precompile? if using_shakapacker_const?