diff --git a/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributeLoadingInterceptor.java b/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributeLoadingInterceptor.java
index 2d284258cb42..901e8ea42bff 100644
--- a/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributeLoadingInterceptor.java
+++ b/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributeLoadingInterceptor.java
@@ -42,8 +42,7 @@ public LazyAttributeLoadingInterceptor(
 			SharedSessionContractImplementor session) {
 		super( entityName, session );
 		this.identifier = identifier;
-		//Important optimisation to not actually do a Map lookup for entities which don't have any lazy fields at all:
-		this.lazyFields = org.hibernate.internal.util.collections.CollectionHelper.toSmallSet( lazyFields );
+		this.lazyFields = lazyFields;
 	}
 
 	@Override
diff --git a/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributesMetadata.java b/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributesMetadata.java
index 6239ece9b512..d32c3a598b77 100644
--- a/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributesMetadata.java
+++ b/hibernate-core/src/main/java/org/hibernate/bytecode/enhance/spi/interceptor/LazyAttributesMetadata.java
@@ -20,6 +20,7 @@
 
 import static java.util.Collections.unmodifiableMap;
 import static java.util.Collections.unmodifiableSet;
+import static org.hibernate.internal.util.collections.CollectionHelper.toSmallSet;
 
 /**
  * Information about the bytecode lazy attributes for an entity
@@ -103,8 +104,8 @@ public LazyAttributesMetadata(
 		this.entityName = entityName;
 		this.lazyAttributeDescriptorMap = lazyAttributeDescriptorMap;
 		this.fetchGroupToAttributeMap = fetchGroupToAttributeMap;
-		this.fetchGroupNames = unmodifiableSet( fetchGroupToAttributeMap.keySet() );
-		this.lazyAttributeNames = unmodifiableSet( lazyAttributeDescriptorMap.keySet() );
+		this.fetchGroupNames = toSmallSet( unmodifiableSet( fetchGroupToAttributeMap.keySet() ) );
+		this.lazyAttributeNames = toSmallSet( unmodifiableSet( lazyAttributeDescriptorMap.keySet() ) );
 	}
 
 	public String getEntityName() {