diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java
index e9225d47a6b8b..529a6c5eaac7c 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java
@@ -42,8 +42,17 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) {
         if (this == other) {
             return this;
         }
-        Set<String> mergedFields = new HashSet<>(this.requiredStoredFields);
-        mergedFields.addAll(other.requiredStoredFields);
+        Set<String> mergedFields;
+        if (other.requiredStoredFields.isEmpty()) {
+            /*
+             * In the very very common case that we don't need new stored fields
+             * let's not clone the existing array.
+             */
+            mergedFields = this.requiredStoredFields;
+        } else {
+            mergedFields = new HashSet<>(this.requiredStoredFields);
+            mergedFields.addAll(other.requiredStoredFields);
+        }
         return new StoredFieldsSpec(
             this.requiresSource || other.requiresSource,
             this.requiresMetadata || other.requiresMetadata,
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java
index feb7da38f4bfa..37fd33cf302b2 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java
@@ -18,7 +18,10 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Set;
+
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -67,6 +70,17 @@ public void testScriptFieldsEnableMetadata() {
         assertTrue(spec.requiresMetadata());
     }
 
+    public void testNoCloneOnMerge() {
+        StoredFieldsSpec spec = StoredFieldsSpec.NO_REQUIREMENTS;
+        spec = spec.merge(StoredFieldsSpec.NEEDS_SOURCE);
+        assertThat(spec.requiredStoredFields(), sameInstance(StoredFieldsSpec.NO_REQUIREMENTS.requiredStoredFields()));
+
+        StoredFieldsSpec needsCat = new StoredFieldsSpec(false, false, Set.of("cat"));
+        StoredFieldsSpec withCat = spec.merge(needsCat);
+        spec = withCat.merge(StoredFieldsSpec.NO_REQUIREMENTS);
+        assertThat(spec.requiredStoredFields(), sameInstance(withCat.requiredStoredFields()));
+    }
+
     private static SearchContext searchContext(SearchSourceBuilder sourceBuilder) {
         SearchContext sc = mock(SearchContext.class);
         when(sc.fetchSourceContext()).thenReturn(sourceBuilder.fetchSource());