feat: Streaming file uploads (#14775)

* start

* pass in form_dat

* serialization

* start deserializer

* finished? deserializer

* upload progress via XHR

* simplify file offsets, sort small files first

* don't cache stream

* fix scoped ids

* tests

* re-add comment

* move location & pathname back to headers

* skip test on node 18

* changeset

* polyfill file for node 18 test

* fix refreshes

* optimize file offset table

* typo

* add lazyfile tests

* avoid double-sending form keys

* remove xhr for next PR

* fix requests stalling if files aren't read

* Update new-rivers-run.md

* encode text before determining length

---------

Co-authored-by: Rich Harris <[email protected]>

Author: ottomated

.changeset/new-rivers-run.md

@@ -0,0 +1,5 @@
+---
+'@sveltejs/kit': minor
+---
+
+feat: stream file uploads inside `form` remote functions allowing form data to be accessed before large files finish uploading

packages/kit/src/runtime/app/server/remote/form.js

@@ -4,7 +4,6 @@
 import { get_request_store } from '@sveltejs/kit/internal/server';
 import { DEV } from 'esm-env';
 import {
-	convert_formdata,
 	create_field_proxy,
 	set_nested_value,
 	throw_on_old_property_access,
@@ -105,19 +104,7 @@ export function form(validate_or_fn, maybe_fn) {
 			type: 'form',
 			name: '',
 			id: '',
-			/** @param {FormData} form_data */
-			fn: async (form_data) => {
-				const validate_only = form_data.get('sveltekit:validate_only') === 'true';
-
-				let data = maybe_fn ? convert_formdata(form_data) : undefined;
-
-				if (data && data.id === undefined) {
-					const id = form_data.get('sveltekit:id');
-					if (typeof id === 'string') {
-						data.id = JSON.parse(id);
-					}
-				}
-
+			fn: async (data, meta, form_data) => {
 				// TODO 3.0 remove this warning
 				if (DEV && !data) {
 					const error = () => {
@@ -153,12 +140,12 @@ export function form(validate_or_fn, maybe_fn) {
 				const { event, state } = get_request_store();
 				const validated = await schema?.['~standard'].validate(data);
 
-				if (validate_only) {
+				if (meta.validate_only) {
 					return validated?.issues?.map((issue) => normalize_issue(issue, true)) ?? [];
 				}
 
 				if (validated?.issues !== undefined) {
-					handle_issues(output, validated.issues, event.isRemoteRequest, form_data);
+					handle_issues(output, validated.issues, form_data);
 				} else {
 					if (validated !== undefined) {
 						data = validated.value;
@@ -179,7 +166,7 @@ export function form(validate_or_fn, maybe_fn) {
 						);
 					} catch (e) {
 						if (e instanceof ValidationError) {
-							handle_issues(output, e.issues, event.isRemoteRequest, form_data);
+							handle_issues(output, e.issues, form_data);
 						} else {
 							throw e;
 						}
@@ -298,15 +285,14 @@ export function form(validate_or_fn, maybe_fn) {
 /**
  * @param {{ issues?: InternalRemoteFormIssue[], input?: Record<string, any>, result: any }} output
  * @param {readonly StandardSchemaV1.Issue[]} issues
- * @param {boolean} is_remote_request
- * @param {FormData} form_data
+ * @param {FormData | null} form_data - null if the form is progressively enhanced
  */
-function handle_issues(output, issues, is_remote_request, form_data) {
+function handle_issues(output, issues, form_data) {
 	output.issues = issues.map((issue) => normalize_issue(issue, true));
 
 	// if it was a progressively-enhanced submission, we don't need
 	// to return the input — it's already there
-	if (!is_remote_request) {
+	if (form_data) {
 		output.input = {};
 
 		for (let key of form_data.keys()) {

packages/kit/src/runtime/client/remote-functions/form.svelte.js

@@ -18,7 +18,9 @@ import {
 	set_nested_value,
 	throw_on_old_property_access,
 	build_path_string,
-	normalize_issue
+	normalize_issue,
+	serialize_binary_form,
+	BINARY_FORM_CONTENT_TYPE
 } from '../../form-utils.js';
 
 /**
@@ -55,6 +57,7 @@ export function form(id) {
 
 	/** @param {string | number | boolean} [key] */
 	function create_instance(key) {
+		const action_id_without_key = id;
 		const action_id = id + (key != undefined ? `/${JSON.stringify(key)}` : '');
 		const action = '?/remote=' + encodeURIComponent(action_id);
 
@@ -182,17 +185,18 @@ export function form(id) {
 				try {
 					await Promise.resolve();
 
-					if (updates.length > 0) {
-						data.set('sveltekit:remote_refreshes', JSON.stringify(updates.map((u) => u._key)));
-					}
+					const { blob } = serialize_binary_form(convert(data), {
+						remote_refreshes: updates.map((u) => u._key)
+					});
 
-					const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, {
+					const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, {
 						method: 'POST',
-						body: data,
 						headers: {
+							'Content-Type': BINARY_FORM_CONTENT_TYPE,
 							'x-sveltekit-pathname': location.pathname,
 							'x-sveltekit-search': location.search
-						}
+						},
+						body: blob
 					});
 
 					if (!response.ok) {
@@ -539,7 +543,9 @@ export function form(id) {
 					/** @type {InternalRemoteFormIssue[]} */
 					let array = [];
 
-					const validated = await preflight_schema?.['~standard'].validate(convert(form_data));
+					const data = convert(form_data);
+
+					const validated = await preflight_schema?.['~standard'].validate(data);
 
 					if (validate_id !== id) {
 						return;
@@ -548,11 +554,16 @@ export function form(id) {
 					if (validated?.issues) {
 						array = validated.issues.map((issue) => normalize_issue(issue, false));
 					} else if (!preflightOnly) {
-						form_data.set('sveltekit:validate_only', 'true');
-
-						const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, {
+						const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, {
 							method: 'POST',
-							body: form_data
+							headers: {
+								'Content-Type': BINARY_FORM_CONTENT_TYPE,
+								'x-sveltekit-pathname': location.pathname,
+								'x-sveltekit-search': location.search
+							},
+							body: serialize_binary_form(data, {
+								validate_only: true
+							}).blob
 						});
 
 						const result = await response.json();
@@ -644,12 +655,6 @@ function clone(element) {
  */
 function validate_form_data(form_data, enctype) {
 	for (const key of form_data.keys()) {
-		if (key.startsWith('sveltekit:')) {
-			throw new Error(
-				'FormData keys starting with `sveltekit:` are reserved for internal use and should not be set manually'
-			);
-		}
-
 		if (/^\$[.[]?/.test(key)) {
 			throw new Error(
 				'`$` is used to collect all FormData validation issues and cannot be used as the `name` of a form control'