diff --git a/packages/react-native-web/src/exports/Image/__tests__/__snapshots__/index-test.js.snap b/packages/react-native-web/src/exports/Image/__tests__/__snapshots__/index-test.js.snap
index 55e2d30ac5..b3e04407ac 100644
--- a/packages/react-native-web/src/exports/Image/__tests__/__snapshots__/index-test.js.snap
+++ b/packages/react-native-web/src/exports/Image/__tests__/__snapshots__/index-test.js.snap
@@ -255,7 +255,7 @@ exports[`components/Image prop "source" is correctly updated when missing in ini
 </div>
 `;
 
-exports[`components/Image prop "source" is not set immediately if the image has not already been loaded 1`] = `
+exports[`components/Image prop "source" is set immediately if the image has already been loaded 1`] = `
 <div
   class="css-view-175oi2r r-flexBasis-1mlwlqe r-overflow-1udh08x r-zIndex-417010"
 >
@@ -272,53 +272,53 @@ exports[`components/Image prop "source" is not set immediately if the image has
 </div>
 `;
 
-exports[`components/Image prop "source" is set immediately if the image has already been loaded 1`] = `
+exports[`components/Image prop "source" is set immediately if the image has already been loaded 2`] = `
 <div
   class="css-view-175oi2r r-flexBasis-1mlwlqe r-overflow-1udh08x r-zIndex-417010"
 >
   <div
     class="css-view-175oi2r r-backgroundColor-1niwhzg r-backgroundPosition-vvn4in r-backgroundRepeat-u6sd8q r-bottom-1p0dtai r-height-1pi2tsx r-left-1d2f490 r-position-u8s1d r-right-zchlnj r-top-ipm5af r-width-13qz1uu r-zIndex-1wyyakw r-backgroundSize-4gszlv"
-    style="background-image: url(https://google.com/favicon.ico);"
+    style="background-image: url(https://twitter.com/favicon.ico);"
   />
   <img
     alt=""
     class="css-accessibilityImage-9pa8cd"
     draggable="false"
-    src="https://google.com/favicon.ico"
+    src="https://twitter.com/favicon.ico"
   />
 </div>
 `;
 
-exports[`components/Image prop "source" is set immediately if the image has already been loaded 2`] = `
+exports[`components/Image prop "source" is set immediately if the image was preloaded 1`] = `
 <div
   class="css-view-175oi2r r-flexBasis-1mlwlqe r-overflow-1udh08x r-zIndex-417010"
 >
   <div
     class="css-view-175oi2r r-backgroundColor-1niwhzg r-backgroundPosition-vvn4in r-backgroundRepeat-u6sd8q r-bottom-1p0dtai r-height-1pi2tsx r-left-1d2f490 r-position-u8s1d r-right-zchlnj r-top-ipm5af r-width-13qz1uu r-zIndex-1wyyakw r-backgroundSize-4gszlv"
-    style="background-image: url(https://twitter.com/favicon.ico);"
+    style="background-image: url(https://yahoo.com/favicon.ico);"
   />
   <img
     alt=""
     class="css-accessibilityImage-9pa8cd"
     draggable="false"
-    src="https://twitter.com/favicon.ico"
+    src="https://yahoo.com/favicon.ico"
   />
 </div>
 `;
 
-exports[`components/Image prop "source" is set immediately if the image was preloaded 1`] = `
+exports[`components/Image prop "source" is set immediately while image is loading and there is no default source 1`] = `
 <div
   class="css-view-175oi2r r-flexBasis-1mlwlqe r-overflow-1udh08x r-zIndex-417010"
 >
   <div
     class="css-view-175oi2r r-backgroundColor-1niwhzg r-backgroundPosition-vvn4in r-backgroundRepeat-u6sd8q r-bottom-1p0dtai r-height-1pi2tsx r-left-1d2f490 r-position-u8s1d r-right-zchlnj r-top-ipm5af r-width-13qz1uu r-zIndex-1wyyakw r-backgroundSize-4gszlv"
-    style="background-image: url(https://yahoo.com/favicon.ico);"
+    style="background-image: url(https://google.com/not-yet-loaded-image.ico);"
   />
   <img
     alt=""
     class="css-accessibilityImage-9pa8cd"
     draggable="false"
-    src="https://yahoo.com/favicon.ico"
+    src="https://google.com/not-yet-loaded-image.ico"
   />
 </div>
 `;
diff --git a/packages/react-native-web/src/exports/Image/__tests__/index-test.js b/packages/react-native-web/src/exports/Image/__tests__/index-test.js
index 42392d973c..eba876b929 100644
--- a/packages/react-native-web/src/exports/Image/__tests__/index-test.js
+++ b/packages/react-native-web/src/exports/Image/__tests__/index-test.js
@@ -248,8 +248,8 @@ describe('components/Image', () => {
       });
     });
 
-    test('is not set immediately if the image has not already been loaded', () => {
-      const uri = 'https://google.com/favicon.ico';
+    test('is set immediately while image is loading and there is no default source', () => {
+      const uri = 'https://google.com/not-yet-loaded-image.ico';
       const source = { uri };
       const { container } = render(<Image source={source} />);
       expect(container.firstChild).toMatchSnapshot();
diff --git a/packages/react-native-web/src/exports/Image/index.js b/packages/react-native-web/src/exports/Image/index.js
index bd69e5e844..d84eb2a7a4 100644
--- a/packages/react-native-web/src/exports/Image/index.js
+++ b/packages/react-native-web/src/exports/Image/index.js
@@ -205,24 +205,18 @@ const Image: React.AbstractComponent<
     }
   }
 
-  const [state, updateState] = React.useState(() => {
-    const uri = resolveAssetUri(source);
-    if (uri != null) {
-      const isLoaded = ImageLoader.has(uri);
-      if (isLoaded) {
-        return LOADED;
-      }
-    }
-    return IDLE;
-  });
-
+  const [state, updateState] = React.useState(IDLE);
   const [layout, updateLayout] = React.useState({});
   const hasTextAncestor = React.useContext(TextAncestorContext);
   const hiddenImageRef = React.useRef(null);
   const filterRef = React.useRef(_filterId++);
   const requestRef = React.useRef(null);
+  const uri = resolveAssetUri(source);
+  const isCached = uri != null && ImageLoader.has(uri);
   const shouldDisplaySource =
-    state === LOADED || (state === LOADING && defaultSource == null);
+    state === LOADED ||
+    isCached ||
+    (state === LOADING && defaultSource == null);
   const [flatStyle, _resizeMode, filter, _tintColor] = getFlatStyle(
     style,
     blurRadius,
@@ -277,7 +271,6 @@ const Image: React.AbstractComponent<
   }
 
   // Image loading
-  const uri = resolveAssetUri(source);
   React.useEffect(() => {
     abortPendingRequest();
 
@@ -316,7 +309,7 @@ const Image: React.AbstractComponent<
 
     function abortPendingRequest() {
       if (requestRef.current != null) {
-        ImageLoader.abort(requestRef.current);
+        ImageLoader.clear(requestRef.current);
         requestRef.current = null;
       }
     }
diff --git a/packages/react-native-web/src/modules/ImageLoader/index.js b/packages/react-native-web/src/modules/ImageLoader/index.js
index 892db99292..5671cb3456 100644
--- a/packages/react-native-web/src/modules/ImageLoader/index.js
+++ b/packages/react-native-web/src/modules/ImageLoader/index.js
@@ -74,12 +74,13 @@ let id = 0;
 const requests = {};
 
 const ImageLoader = {
-  abort(requestId: number) {
-    let image = requests[`${requestId}`];
+  clear(requestId: number) {
+    const image = requests[`${requestId}`];
     if (image) {
       image.onerror = null;
       image.onload = null;
-      image = null;
+      ImageUriCache.remove(image.src);
+      image.src = '';
       delete requests[`${requestId}`];
     }
   },
@@ -102,7 +103,7 @@ const ImageLoader = {
         }
       }
       if (complete) {
-        ImageLoader.abort(requestId);
+        ImageLoader.clear(requestId);
         clearInterval(interval);
       }
     }
@@ -111,7 +112,7 @@ const ImageLoader = {
       if (typeof failure === 'function') {
         failure();
       }
-      ImageLoader.abort(requestId);
+      ImageLoader.clear(requestId);
       clearInterval(interval);
     }
   },
@@ -123,6 +124,7 @@ const ImageLoader = {
     const image = new window.Image();
     image.onerror = onError;
     image.onload = (e) => {
+      ImageUriCache.add(uri);
       // avoid blocking the main thread
       const onDecode = () => onLoad({ nativeEvent: e });
       if (typeof image.decode === 'function') {
@@ -143,9 +145,8 @@ const ImageLoader = {
       ImageLoader.load(
         uri,
         () => {
-          // Add the uri to the cache so it can be immediately displayed when used
-          // but also immediately remove it to correctly reflect that it has no active references
-          ImageUriCache.add(uri);
+          // load() adds the uri to the cache so it can be immediately displayed when used,
+          // but we also immediately remove it to correctly reflect that it has no active references
           ImageUriCache.remove(uri);
           resolve();
         },