Skip to content

Commit ea05b75

Browse files
sebmarkbagesebmarkbage
authored
Allow Passing Blob/File/MediaSource/MediaStream to src of <img>, <video> and <audio> (facebook#32828)
Behind the `enableSrcObject` flag. This is revisiting a variant of what was discussed in facebook#11163. Instead of supporting the [`srcObject` property](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/srcObject) as a separate name, this adds an overload of `src` to allow objects to be passed. The DOM needs to add separate properties for the object forms since you read back but it doesn't make sense for React's write-only API to do that. Similar to how we'll like add an overload for `popoverTarget` instead of calling it `popoverTargetElement` and how `style` accepts an object and it's not `styleObject={{...}}`. There are a number of reason to revisit this. - It's just way more convenient to have this built-in and it makes conceptual sense. We typically support declarative APIs and polyfill them when necessary. - RSC supports Blobs and by having it built-in you don't need a Client Component wrapper to render it where as doing it with effects would require more complex wrappers. By picking Blobs over base64, client-navigations can use the more optimized binary encoding in the RSC protocol. - The timing aspect of coordinating it with Suspensey images and image decoding is a bit tricky to get right because if you set it in an effect it's too late because you've already rendered it. - SSR gets complicated when done in user space because you have to handle both branches. Likely with `useSyncExternalStore`. - By having it built-in we could optimize the payloads shared between RSC payloads embedded in the HTML and data URLs. This does not support objects for `<source src>` nor `<img srcset>`. Those don't really have equivalents in the DOM neither. They're mainly for picking an option when you don't know programmatically. However, for this use case you're really better off picking a variant before generating the blobs. We may support Response objects in the future too as per whatwg/fetch#49
1 parent 3366146 commit ea05b75

25 files changed

+805
-38
lines changed

‎fixtures/flight/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"browserslist": "^4.18.1",
2424
"busboy": "^1.6.0",
2525
"camelcase": "^6.2.1",
26+
"canvas": "^3.1.0",
2627
"case-sensitive-paths-webpack-plugin": "^2.4.0",
2728
"compression": "^1.7.4",
2829
"concurrently": "^7.3.0",

‎fixtures/flight/src/App.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import {Client} from './Client.js';
1515

1616
import {Note} from './cjs/Note.js';
1717

18+
import {GenerateImage} from './GenerateImage.js';
19+
1820
import {like, greet, increment} from './actions.js';
1921

2022
import {getServerState} from './ServerState.js';
@@ -41,6 +43,7 @@ export default async function App({prerender}) {
4143
const todos = await res.json();
4244

4345
const dedupedChild = <ServerComponent />;
46+
const message = getServerState();
4447
return (
4548
<html lang="en">
4649
<head>
@@ -55,7 +58,7 @@ export default async function App({prerender}) {
5558
) : (
5659
<meta content="when not prerendering we render this meta tag. When prerendering you will expect to see this tag and the one with data-testid=prerendered because we SSR one and hydrate the other" />
5760
)}
58-
<h1>{getServerState()}</h1>
61+
<h1>{message}</h1>
5962
<React.Suspense fallback={null}>
6063
<div data-testid="promise-as-a-child-test">
6164
Promise as a child hydrates without errors: {promisedText}
@@ -79,6 +82,9 @@ export default async function App({prerender}) {
7982
<div>
8083
loaded statically: <Dynamic />
8184
</div>
85+
<div>
86+
<GenerateImage message={message} />
87+
</div>
8288
<Client />
8389
<Note />
8490
<Foo>{dedupedChild}</Foo>

‎fixtures/flight/src/GenerateImage.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import * as React from 'react';
2+
3+
import {createCanvas} from 'canvas';
4+
5+
export async function GenerateImage({message}) {
6+
// Generate an image using an image library
7+
const canvas = createCanvas(200, 70);
8+
const ctx = canvas.getContext('2d');
9+
ctx.font = '20px Impact';
10+
ctx.rotate(-0.1);
11+
ctx.fillText(message, 10, 50);
12+
13+
// Rasterize into a Blob with a mime type
14+
const type = 'image/png';
15+
const blob = new Blob([canvas.toBuffer(type)], {type});
16+
17+
// Just pass it to React
18+
return <img src={blob} />;
19+
}

‎fixtures/flight/yarn.lock‎

Lines changed: 220 additions & 30 deletions
Large diffs are not rendered by default.

‎packages/react-dom-bindings/src/client/ReactDOMComponent.js‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import {
4747
updateTextarea,
4848
restoreControlledTextareaState,
4949
} from './ReactDOMTextarea';
50+
import {setSrcObject} from './ReactDOMSrcObject';
5051
import {validateTextNesting} from './validateDOMNesting';
5152
import {track} from './inputValueTracking';
5253
import setTextContent from './setTextContent';
@@ -67,6 +68,7 @@ import {trackHostMutation} from 'react-reconciler/src/ReactFiberMutationTracking
6768

6869
import {
6970
enableScrollEndPolyfill,
71+
enableSrcObject,
7072
enableTrustedTypesIntegration,
7173
} from 'shared/ReactFeatureFlags';
7274
import {
@@ -402,7 +404,40 @@ function setProp(
402404
break;
403405
}
404406
// fallthrough
405-
case 'src':
407+
case 'src': {
408+
if (enableSrcObject && typeof value === 'object' && value !== null) {
409+
// Some tags support object sources like Blob, File, MediaSource and MediaStream.
410+
if (tag === 'img' || tag === 'video' || tag === 'audio') {
411+
try {
412+
setSrcObject(domElement, tag, value);
413+
break;
414+
} catch (x) {
415+
// If URL.createObjectURL() errors, it was probably some other object type
416+
// that should be toString:ed instead, so we just fall-through to the normal
417+
// path.
418+
}
419+
} else {
420+
if (__DEV__) {
421+
try {
422+
// This should always error.
423+
URL.revokeObjectURL(URL.createObjectURL((value: any)));
424+
if (tag === 'source') {
425+
console.error(
426+
'Passing Blob, MediaSource or MediaStream to <source src> is not supported. ' +
427+
'Pass it directly to <img src>, <video src> or <audio src> instead.',
428+
);
429+
} else {
430+
console.error(
431+
'Passing Blob, MediaSource or MediaStream to <%s src> is not supported.',
432+
tag,
433+
);
434+
}
435+
} catch (x) {}
436+
}
437+
}
438+
}
439+
// Fallthrough
440+
}
406441
case 'href': {
407442
if (
408443
value === '' &&
@@ -2301,6 +2336,39 @@ function hydrateSanitizedAttribute(
23012336
warnForPropDifference(propKey, serverValue, value, serverDifferences);
23022337
}
23032338

2339+
function hydrateSrcObjectAttribute(
2340+
domElement: Element,
2341+
value: Blob,
2342+
extraAttributes: Set<string>,
2343+
serverDifferences: {[propName: string]: mixed},
2344+
): void {
2345+
const attributeName = 'src';
2346+
extraAttributes.delete(attributeName);
2347+
const serverValue = domElement.getAttribute(attributeName);
2348+
if (serverValue != null && value != null) {
2349+
const size = value.size;
2350+
const type = value.type;
2351+
if (typeof size === 'number' && typeof type === 'string') {
2352+
if (serverValue.indexOf('data:' + type + ';base64,') === 0) {
2353+
// For Blobs we don't bother reading the actual data but just diff by checking if
2354+
// the byte length size of the Blob maches the length of the data url.
2355+
const prefixLength = 5 + type.length + 8;
2356+
let byteLength = ((serverValue.length - prefixLength) / 4) * 3;
2357+
if (serverValue[serverValue.length - 1] === '=') {
2358+
byteLength--;
2359+
}
2360+
if (serverValue[serverValue.length - 2] === '=') {
2361+
byteLength--;
2362+
}
2363+
if (byteLength === size) {
2364+
return;
2365+
}
2366+
}
2367+
}
2368+
}
2369+
warnForPropDifference('src', serverValue, value, serverDifferences);
2370+
}
2371+
23042372
function diffHydratedCustomComponent(
23052373
domElement: Element,
23062374
tag: string,
@@ -2547,7 +2615,45 @@ function diffHydratedGenericElement(
25472615
continue;
25482616
}
25492617
// fallthrough
2550-
case 'src':
2618+
case 'src': {
2619+
if (enableSrcObject && typeof value === 'object' && value !== null) {
2620+
// Some tags support object sources like Blob, File, MediaSource and MediaStream.
2621+
if (tag === 'img' || tag === 'video' || tag === 'audio') {
2622+
try {
2623+
// Test if this is a compatible object
2624+
URL.revokeObjectURL(URL.createObjectURL((value: any)));
2625+
hydrateSrcObjectAttribute(
2626+
domElement,
2627+
value,
2628+
extraAttributes,
2629+
serverDifferences,
2630+
);
2631+
continue;
2632+
} catch (x) {
2633+
// If not, just fall through to the normal toString flow.
2634+
}
2635+
} else {
2636+
if (__DEV__) {
2637+
try {
2638+
// This should always error.
2639+
URL.revokeObjectURL(URL.createObjectURL((value: any)));
2640+
if (tag === 'source') {
2641+
console.error(
2642+
'Passing Blob, MediaSource or MediaStream to <source src> is not supported. ' +
2643+
'Pass it directly to <img src>, <video src> or <audio src> instead.',
2644+
);
2645+
} else {
2646+
console.error(
2647+
'Passing Blob, MediaSource or MediaStream to <%s src> is not supported.',
2648+
tag,
2649+
);
2650+
}
2651+
} catch (x) {}
2652+
}
2653+
}
2654+
}
2655+
// Fallthrough
2656+
}
25512657
case 'href':
25522658
if (
25532659
value === '' &&

‎packages/react-dom-bindings/src/client/ReactDOMSrcObject.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/**
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*
7+
* @flow
8+
*/
9+
10+
export function setSrcObject(domElement: Element, tag: string, value: any) {
11+
// We optimistically create the URL regardless of object type. This lets us
12+
// support cross-realms and any type that the browser supports like new types.
13+
const url = URL.createObjectURL((value: any));
14+
const loadEvent = tag === 'img' ? 'load' : 'loadstart';
15+
const cleanUp = () => {
16+
// Once the object has started loading, then it's already collected by the
17+
// browser and it won't refer to it by the URL anymore so we can now revoke it.
18+
URL.revokeObjectURL(url);
19+
domElement.removeEventListener(loadEvent, cleanUp);
20+
domElement.removeEventListener('error', cleanUp);
21+
};
22+
domElement.addEventListener(loadEvent, cleanUp);
23+
domElement.addEventListener('error', cleanUp);
24+
domElement.setAttribute('src', url);
25+
}

‎packages/react-dom-bindings/src/client/ReactFiberConfigDOM.js‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import type {TransitionTypes} from 'react/src/ReactTransitionType';
2929

3030
import {NotPending} from '../shared/ReactDOMFormActions';
3131

32+
import {setSrcObject} from './ReactDOMSrcObject';
33+
3234
import {getCurrentRootHostContainer} from 'react-reconciler/src/ReactFiberHostContext';
3335
import {runWithFiberInDEV} from 'react-reconciler/src/ReactCurrentFiber';
3436

@@ -104,6 +106,7 @@ import {
104106
enableMoveBefore,
105107
disableCommentsAsDOMContainers,
106108
enableSuspenseyImages,
109+
enableSrcObject,
107110
} from 'shared/ReactFeatureFlags';
108111
import {
109112
HostComponent,
@@ -151,7 +154,7 @@ export type Props = {
151154
is?: string,
152155
size?: number,
153156
multiple?: boolean,
154-
src?: string,
157+
src?: string | Blob | MediaSource | MediaStream, // TODO: Response
155158
srcSet?: string,
156159
loading?: 'eager' | 'lazy',
157160
onLoad?: (event: any) => void,
@@ -780,7 +783,23 @@ export function commitMount(
780783
// is already a noop regardless of which properties are assigned. We should revisit if browsers update
781784
// this heuristic in the future.
782785
if (newProps.src) {
783-
((domElement: any): HTMLImageElement).src = (newProps: any).src;
786+
const src = (newProps: any).src;
787+
if (enableSrcObject && typeof src === 'object') {
788+
// For object src, we can't just set the src again to the same blob URL because it might have
789+
// already revoked if it loaded before this. However, we can create a new blob URL and set that.
790+
// This is relatively cheap since the blob is already in memory but this might cause some
791+
// duplicated work.
792+
// TODO: We could maybe detect if load hasn't fired yet and if so reuse the URL.
793+
try {
794+
setSrcObject(domElement, type, src);
795+
return;
796+
} catch (x) {
797+
// If URL.createObjectURL() errors, it was probably some other object type
798+
// that should be toString:ed instead, so we just fall-through to the normal
799+
// path.
800+
}
801+
}
802+
((domElement: any): HTMLImageElement).src = src;
784803
} else if (newProps.srcSet) {
785804
((domElement: any): HTMLImageElement).srcset = (newProps: any).srcSet;
786805
}

‎packages/react-dom-bindings/src/server/ReactDOMLegacyServerStreamConfig.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,14 @@ export function closeWithError(destination: Destination, error: mixed): void {
8080
}
8181

8282
export {createFastHashJS as createFastHash} from 'react-server/src/createFastHashJS';
83+
84+
export function readAsDataURL(blob: Blob): Promise<string> {
85+
return blob.arrayBuffer().then(arrayBuffer => {
86+
const encoded =
87+
typeof Buffer === 'function' && typeof Buffer.from === 'function'
88+
? Buffer.from(arrayBuffer).toString('base64')
89+
: btoa(String.fromCharCode.apply(String, new Uint8Array(arrayBuffer)));
90+
const mimeType = blob.type || 'application/octet-stream';
91+
return 'data:' + mimeType + ';base64,' + encoded;
92+
});
93+
}

‎packages/react-dom-bindings/src/server/ReactFizzConfigDOM.js‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
* @flow
88
*/
99

10-
import type {ReactNodeList, ReactCustomFormAction} from 'shared/ReactTypes';
10+
import type {
11+
ReactNodeList,
12+
ReactCustomFormAction,
13+
Thenable,
14+
} from 'shared/ReactTypes';
1115
import type {
1216
CrossOriginEnum,
1317
PreloadImplOptions,
@@ -27,7 +31,10 @@ import {
2731

2832
import {Children} from 'react';
2933

30-
import {enableFizzExternalRuntime} from 'shared/ReactFeatureFlags';
34+
import {
35+
enableFizzExternalRuntime,
36+
enableSrcObject,
37+
} from 'shared/ReactFeatureFlags';
3138

3239
import type {
3340
Destination,
@@ -42,6 +49,7 @@ import {
4249
writeChunkAndReturn,
4350
stringToChunk,
4451
stringToPrecomputedChunk,
52+
readAsDataURL,
4553
} from 'react-server/src/ReactServerStreamConfig';
4654
import {
4755
resolveRequest,
@@ -1214,6 +1222,47 @@ function pushFormActionAttribute(
12141222
return formData;
12151223
}
12161224

1225+
let blobCache: null | WeakMap<Blob, Thenable<string>> = null;
1226+
1227+
function pushSrcObjectAttribute(
1228+
target: Array<Chunk | PrecomputedChunk>,
1229+
blob: Blob,
1230+
): void {
1231+
// Throwing a Promise style suspense read of the Blob content.
1232+
if (blobCache === null) {
1233+
blobCache = new WeakMap();
1234+
}
1235+
const suspenseCache: WeakMap<Blob, Thenable<string>> = blobCache;
1236+
let thenable = suspenseCache.get(blob);
1237+
if (thenable === undefined) {
1238+
thenable = ((readAsDataURL(blob): any): Thenable<string>);
1239+
thenable.then(
1240+
result => {
1241+
(thenable: any).status = 'fulfilled';
1242+
(thenable: any).value = result;
1243+
},
1244+
error => {
1245+
(thenable: any).status = 'rejected';
1246+
(thenable: any).reason = error;
1247+
},
1248+
);
1249+
suspenseCache.set(blob, thenable);
1250+
}
1251+
if (thenable.status === 'rejected') {
1252+
throw thenable.reason;
1253+
} else if (thenable.status !== 'fulfilled') {
1254+
throw thenable;
1255+
}
1256+
const url = thenable.value;
1257+
target.push(
1258+
attributeSeparator,
1259+
stringToChunk('src'),
1260+
attributeAssign,
1261+
stringToChunk(escapeTextForBrowser(url)),
1262+
attributeEnd,
1263+
);
1264+
}
1265+
12171266
function pushAttribute(
12181267
target: Array<Chunk | PrecomputedChunk>,
12191268
name: string,
@@ -1243,7 +1292,15 @@ function pushAttribute(
12431292
pushStyleAttribute(target, value);
12441293
return;
12451294
}
1246-
case 'src':
1295+
case 'src': {
1296+
if (enableSrcObject && typeof value === 'object' && value !== null) {
1297+
if (typeof Blob === 'function' && value instanceof Blob) {
1298+
pushSrcObjectAttribute(target, value);
1299+
return;
1300+
}
1301+
}
1302+
// Fallthrough to general urls
1303+
}
12471304
case 'href': {
12481305
if (value === '') {
12491306
if (__DEV__) {

0 commit comments

Comments
 (0)