diff --git a/packages/plugin-react/CHANGELOG.md b/packages/plugin-react/CHANGELOG.md
index 138dfe07b..e2f7e5240 100644
--- a/packages/plugin-react/CHANGELOG.md
+++ b/packages/plugin-react/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+### Perf: skip `babel-plugin-react-compiler` if code has no `"use memo"` when `{ compilationMode: "annotation" }` ([#734](https://github.com/vitejs/vite-plugin-react/pull/734))
+
 ### Respect tsconfig `jsxImportSource` ([#726](https://github.com/vitejs/vite-plugin-react/pull/726))
 
 ### Fix `reactRefreshHost` option on rolldown-vite ([#716](https://github.com/vitejs/vite-plugin-react/pull/716))
diff --git a/packages/plugin-react/src/index.ts b/packages/plugin-react/src/index.ts
index bb2b224fa..9538a13a3 100644
--- a/packages/plugin-react/src/index.ts
+++ b/packages/plugin-react/src/index.ts
@@ -102,6 +102,7 @@ export type ViteReactPluginApi = {
 const defaultIncludeRE = /\.[tj]sx?$/
 const defaultExcludeRE = /\/node_modules\//
 const tsRE = /\.tsx?$/
+const compilerAnnotationRE = /['"]use memo['"]/
 
 export default function viteReact(opts: Options = {}): Plugin[] {
   const include = opts.include ?? defaultIncludeRE
@@ -247,11 +248,21 @@ export default function viteReact(opts: Options = {}): Plugin[] {
         const plugins = [...babelOptions.plugins]
 
         // remove react-compiler plugin on non client environment
-        if (ssr) {
-          const reactCompilerPlugin = getReactCompilerPlugin(plugins)
-          if (reactCompilerPlugin) {
-            plugins.splice(plugins.indexOf(reactCompilerPlugin), 1)
-          }
+        let reactCompilerPlugin = getReactCompilerPlugin(plugins)
+        if (reactCompilerPlugin && ssr) {
+          plugins.splice(plugins.indexOf(reactCompilerPlugin), 1)
+          reactCompilerPlugin = undefined
+        }
+
+        // filter by "use memo" when react-compiler { compilationMode: "annotation" }
+        // https://react.dev/learn/react-compiler/incremental-adoption#annotation-mode-configuration
+        if (
+          Array.isArray(reactCompilerPlugin) &&
+          reactCompilerPlugin[1]?.compilationMode === 'annotation' &&
+          !compilerAnnotationRE.test(code)
+        ) {
+          plugins.splice(plugins.indexOf(reactCompilerPlugin), 1)
+          reactCompilerPlugin = undefined
         }
 
         const isJSX = filepath.endsWith('x')
@@ -305,10 +316,9 @@ export default function viteReact(opts: Options = {}): Plugin[] {
           // Required for esbuild.jsxDev to provide correct line numbers
           // This creates issues the react compiler because the re-order is too important
           // People should use @babel/plugin-transform-react-jsx-development to get back good line numbers
-          retainLines:
-            getReactCompilerPlugin(plugins) != null
-              ? false
-              : !isProduction && isJSX && opts.jsxRuntime !== 'classic',
+          retainLines: reactCompilerPlugin
+            ? false
+            : !isProduction && isJSX && opts.jsxRuntime !== 'classic',
           parserOpts: {
             ...babelOptions.parserOpts,
             sourceType: 'module',