Merge pull request #657 from bci-oss/bugfix/605-error-creating-delta-table-with-the-generated-schema

Error creating delta table with the generated schema

Author: Yauhenikapl

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java

@@ -257,13 +257,13 @@ public String visitCollection( final Collection collection, final Context contex
             .getDescription( config.commentLanguage() ) )
             : Optional.empty();
       final String typeDef = type.isComplexType()
-            ? entityToStruct( type.as( ComplexType.class ) ).toString()
+            ? entityToStruct( type.as( ComplexType.class ), false ).toString()
             : type.accept( this, context );
       return column( context.prefix(), "ARRAY<" + typeDef + ">", property.isOptional() || context.forceOptional(),
             comment );
    }
 
-   private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity ) {
+   private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity, final boolean isInsideNestedType ) {
       return new DatabricksType.DatabricksStruct( entity.getAllProperties().stream()
             .flatMap( property -> {
                if ( property.getDataType().isEmpty() || property.isNotInPayload() ) {
@@ -274,12 +274,15 @@ private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity
                if ( type instanceof final Scalar scalar ) {
                   databricksType = databricksTypeMap.get( scalar.getUrn() );
                } else if ( type instanceof final Entity entityType ) {
-                  databricksType = entityToStruct( entityType );
+                  databricksType = entityToStruct( entityType, true );
                } else {
                   return Stream.empty();
                }
+
+               boolean isOptional = isInsideNestedType || property.isOptional();
+
                return Stream.of( new DatabricksType.DatabricksStructEntry( columnName( property ), databricksType,
-                     property.isOptional(), Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) );
+                     isOptional, Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) );
             } )
             .toList() );
    }

core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java

@@ -24,7 +24,7 @@
 import org.junit.jupiter.api.Test;
 
 @SuppressWarnings( "checkstyle:LineLength" )
-public class AspectModelDatabricksDenormalizedSqlVisitorTest extends DatabricksTestBase {
+class AspectModelDatabricksDenormalizedSqlVisitorTest extends DatabricksTestBase {
    @Test
    void testAspectWithAbstractEntity() {
       assertThat( sql( TestAspect.ASPECT_WITH_ABSTRACT_ENTITY ) ).isEqualTo( """
@@ -319,7 +319,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections (
    void testAspectWithNestedEntityList() {
       assertThat( sql( TestAspect.ASPECT_WITH_NESTED_ENTITY_LIST ) ).isEqualTo( """
             CREATE TABLE IF NOT EXISTS aspect_with_nested_entity_list (
-              test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_second_list: STRUCT<test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL> NOT NULL>> NOT NULL
+              test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_second_list: STRUCT<test_local_date_time: TIMESTAMP, random_value: STRING> NOT NULL>> NOT NULL
             )
             TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithNestedEntityList');
             """ );