diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8f095326f6..1fa369e821 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,15 +18,16 @@ should change the heading of the (upcoming) version to include a major version b
 
 # 6.0.0-beta.8
 
-## @rjsf/util
-
-- Fixed form data propagation with `patternProperties` [#4617](https://github.com/rjsf-team/react-jsonschema-form/pull/4617)
-
 ## @rjsf/chakra-ui
 
 - Added `getChakra` to package exports
 - Restored the `ui:options` customization
 
+## @rjsf/util
+
+- Fixed form data propagation with `patternProperties` [#4617](https://github.com/rjsf-team/react-jsonschema-form/pull/4617)
+- Fixed issue where oneOf schema references could not be modified when defaults were set, fixing [#4580](https://github.com/rjsf-team/react-jsonschema-form/issues/4580).
+
 ## Dev / docs / playground
 
 - Updated precompiled schemas documentation in `validation.md` based on v6 changes, addressingg [#4618](https://github.com/rjsf-team/react-jsonschema-form/issues/4618)
diff --git a/packages/core/test/Form.test.jsx b/packages/core/test/Form.test.jsx
index 5f8709e3dc..96acdc3d98 100644
--- a/packages/core/test/Form.test.jsx
+++ b/packages/core/test/Form.test.jsx
@@ -1493,6 +1493,60 @@ describeRepeated('Form common', (createFormComponent) => {
 
       expect(node.querySelector(secondInputID).value).to.equal('changed!');
     });
+    it('Should modify oneOf object with references when the defaults are set.', () => {
+      const schema = {
+        type: 'object',
+        $defs: {
+          protocol: {
+            type: 'string',
+            enum: ['fast', 'balanced', 'stringent'],
+            default: 'fast',
+          },
+        },
+        oneOf: [
+          {
+            properties: {
+              protocol: {
+                $ref: '#/$defs/protocol',
+              },
+            },
+          },
+          {
+            properties: {
+              something: {
+                type: 'number',
+              },
+            },
+          },
+        ],
+      };
+
+      const { node, onChange } = createFormComponent({
+        schema,
+      });
+
+      const protocolInputID = '#root_protocol';
+      expect(node.querySelector(protocolInputID).value).to.equal('0');
+
+      act(() => {
+        fireEvent.change(node.querySelector(protocolInputID), {
+          target: { value: '1' },
+        });
+      });
+
+      sinon.assert.calledWithMatch(
+        onChange.lastCall,
+        {
+          formData: {
+            protocol: 'balanced',
+          },
+          schema,
+        },
+        'root_protocol',
+      );
+
+      expect(node.querySelector(protocolInputID).value).to.equal('1');
+    });
   });
 
   describe('Blur handler', () => {
diff --git a/packages/utils/src/schema/getDefaultFormState.ts b/packages/utils/src/schema/getDefaultFormState.ts
index 079f11992a..a803648c96 100644
--- a/packages/utils/src/schema/getDefaultFormState.ts
+++ b/packages/utils/src/schema/getDefaultFormState.ts
@@ -204,7 +204,7 @@ export function computeDefaults<T = any, S extends StrictRJSFSchema = RJSFSchema
     required,
     shouldMergeDefaultsIntoFormData = false,
   } = computeDefaultsProps;
-  const formData: T = (isObject(rawFormData) ? rawFormData : {}) as T;
+  let formData: T = (isObject(rawFormData) ? rawFormData : {}) as T;
   const schema: S = isObject(rawSchema) ? rawSchema : ({} as S);
   // Compute the defaults recursively: give highest priority to deepest nodes.
   let defaults: T | T[] | undefined = parentDefaults;
@@ -212,7 +212,6 @@ export function computeDefaults<T = any, S extends StrictRJSFSchema = RJSFSchema
   let schemaToCompute: S | null = null;
   let experimental_dfsb_to_compute = experimental_defaultFormStateBehavior;
   let updatedRecurseList = _recurseList;
-
   if (
     schema[CONST_KEY] &&
     experimental_defaultFormStateBehavior?.constAsDefaults !== 'never' &&
@@ -240,6 +239,13 @@ export function computeDefaults<T = any, S extends StrictRJSFSchema = RJSFSchema
     if (schemaToCompute && !defaults) {
       defaults = schema.default as T | undefined;
     }
+
+    // If shouldMergeDefaultsIntoFormData is true
+    // And the schemaToCompute is set and the rawFormData is not an object
+    // Then set the formData to the rawFormData
+    if (shouldMergeDefaultsIntoFormData && schemaToCompute && !isObject(rawFormData)) {
+      formData = rawFormData as T;
+    }
   } else if (DEPENDENCIES_KEY in schema) {
     // Get the default if set from properties to ensure the dependencies conditions are resolved based on it
     const defaultFormData: T = {