refactor: migration validation

src/DependencyInjection/migration.xml

@@ -426,6 +426,7 @@
             <argument type="service" id="Symfony\Component\EventDispatcher\EventDispatcherInterface"/>
             <argument type="service" id="SwagMigrationAssistant\Migration\Logging\LoggingService"/>
             <argument type="service" id="SwagMigrationAssistant\Migration\Mapping\MappingService"/>
+            <argument type="service" id="Doctrine\DBAL\Connection"/>
         </service>
     </services>
 </container>

src/Exception/MigrationException.php

@@ -95,7 +95,7 @@ class MigrationException extends HttpException
 
     final public const INVALID_ID = 'SWAG_MIGRATION__INVALID_ID';
 
-    public const DUPLICATE_SOURCE_CONNECTION = 'SWAG_MIGRATION__DUPLICATE_SOURCE_CONNECTION';
+    final public const DUPLICATE_SOURCE_CONNECTION = 'SWAG_MIGRATION__DUPLICATE_SOURCE_CONNECTION';
 
     public static function associationEntityRequiredMissing(string $entity, string $missingEntity): self
     {

src/Migration/Validation/MigrationValidationService.php

@@ -7,11 +7,13 @@
 
 namespace SwagMigrationAssistant\Migration\Validation;
 
+use Doctrine\DBAL\Connection;
 use Shopware\Core\Framework\Context;
+use Shopware\Core\Framework\DataAbstractionLayer\CompiledFieldCollection;
 use Shopware\Core\Framework\DataAbstractionLayer\DefinitionInstanceRegistry;
-use Shopware\Core\Framework\DataAbstractionLayer\Field\Field;
 use Shopware\Core\Framework\DataAbstractionLayer\Field\FkField;
 use Shopware\Core\Framework\DataAbstractionLayer\Field\Flag\Required;
+use Shopware\Core\Framework\DataAbstractionLayer\Field\StorageAware;
 use Shopware\Core\Framework\DataAbstractionLayer\Write\Command\WriteCommandQueue;
 use Shopware\Core\Framework\DataAbstractionLayer\Write\DataStack\KeyValuePair;
 use Shopware\Core\Framework\DataAbstractionLayer\Write\EntityExistence;
@@ -37,13 +39,19 @@
  * @internal
  */
 #[Package('fundamentals@after-sales')]
-readonly class MigrationValidationService
+class MigrationValidationService
 {
+    /**
+     * @var array<string, list<string>>
+     */
+    private array $requiredColumnsCache = [];
+
     public function __construct(
-        private DefinitionInstanceRegistry $definitionRegistry,
-        private EventDispatcherInterface $eventDispatcher,
-        private LoggingServiceInterface $loggingService,
-        private MappingServiceInterface $mappingService,
+        private readonly DefinitionInstanceRegistry $definitionRegistry,
+        private readonly EventDispatcherInterface $eventDispatcher,
+        private readonly LoggingServiceInterface $loggingService,
+        private readonly MappingServiceInterface $mappingService,
+        private readonly Connection $connection,
     ) {
     }
 
@@ -83,7 +91,7 @@ public function validate(
         } catch (\Throwable $exception) {
             $validationContext->getValidationResult()->addLog(
                 MigrationLogBuilder::fromMigrationContext($validationContext->getMigrationContext())
-                    ->withEntityName($validationContext->getEntityDefinition()->getEntityName())
+                    ->withEntityName($entityDefinition->getEntityName())
                     ->withSourceData($validationContext->getSourceData())
                     ->withConvertedData($validationContext->getConvertedData())
                     ->withExceptionMessage($exception->getMessage())
@@ -106,68 +114,91 @@ public function validate(
         return $validationContext->getValidationResult();
     }
 
+    /**
+     * Validates that all required fields are present and that no unexpected fields exist.
+     * Required fields are determined by checking which database columns are non-nullable without a default value
+     */
     private function validateEntityStructure(MigrationValidationContext $validationContext): void
     {
-        $fields = $validationContext->getEntityDefinition()->getFields();
+        $entityDefinition = $validationContext->getEntityDefinition();
+
+        $fields = $entityDefinition->getFields();
+        $entityName = $entityDefinition->getEntityName();
+
+        $convertedData = $validationContext->getConvertedData();
+        $validationResult = $validationContext->getValidationResult();
 
-        $requiredFields = array_values(array_map(
-            static fn (Field $field) => $field->getPropertyName(),
-            $fields->filterByFlag(Required::class)->getElements()
-        ));
+        $requiredDatabaseColumns = $this->getRequiredDatabaseColumns($entityName);
+        $requiredFields = $this->filterRequiredFields(
+            $fields,
+            $requiredDatabaseColumns
+        );
 
-        $convertedFieldNames = array_keys($validationContext->getConvertedData());
-        $missingRequiredFields = array_diff($requiredFields, $convertedFieldNames);
+        $convertedFieldNames = array_keys($convertedData);
+        $missingRequiredFields = array_diff(
+            $requiredFields,
+            $convertedFieldNames
+        );
 
         foreach ($missingRequiredFields as $missingField) {
-            $validationContext->getValidationResult()->addLog(
+            $validationResult->addLog(
                 MigrationLogBuilder::fromMigrationContext($validationContext->getMigrationContext())
-                    ->withEntityName($validationContext->getEntityDefinition()->getEntityName())
+                    ->withEntityName($entityName)
                     ->withFieldName($missingField)
-                    ->withConvertedData($validationContext->getConvertedData())
-                    ->withEntityId($validationContext->getConvertedData()['id'] ?? null)
+                    ->withConvertedData($convertedData)
+                    ->withEntityId($convertedData['id'] ?? null)
                     ->build(MigrationValidationMissingRequiredFieldLog::class)
             );
         }
 
         $unexpectedFields = array_diff($convertedFieldNames, array_keys($fields->getElements()));
 
         foreach ($unexpectedFields as $unexpectedField) {
-            $validationContext->getValidationResult()->addLog(
+            $validationResult->addLog(
                 MigrationLogBuilder::fromMigrationContext($validationContext->getMigrationContext())
-                    ->withEntityName($validationContext->getEntityDefinition()->getEntityName())
+                    ->withEntityName($entityName)
                     ->withFieldName($unexpectedField)
-                    ->withConvertedData($validationContext->getConvertedData())
-                    ->withEntityId($validationContext->getConvertedData()['id'] ?? null)
+                    ->withConvertedData($convertedData)
+                    ->withEntityId($convertedData['id'] ?? null)
                     ->build(MigrationValidationUnexpectedFieldLog::class)
             );
         }
     }
 
+    /**
+     * Validates that all field values conform to their field definitions by attempting to serialize them.
+     */
     private function validateFields(MigrationValidationContext $validationContext): void
     {
-        $fields = $validationContext->getEntityDefinition()->getFields();
+        $entityDefinition = $validationContext->getEntityDefinition();
+        $fields = $entityDefinition->getFields();
+
+        $convertedData = $validationContext->getConvertedData();
+        $validationResult = $validationContext->getValidationResult();
 
-        if (!isset($validationContext->getConvertedData()['id'])) {
+        $id = $convertedData['id'] ?? null;
+
+        if ($id === null) {
             throw MigrationException::unexpectedNullValue('id');
         }
 
-        if (!Uuid::isValid($validationContext->getConvertedData()['id'])) {
-            throw MigrationException::invalidId($validationContext->getConvertedData()['id'], $validationContext->getEntityDefinition()->getEntityName());
+        if (!Uuid::isValid($id)) {
+            throw MigrationException::invalidId($id, $entityDefinition->getEntityName());
         }
 
         $entityExistence = EntityExistence::createForEntity(
-            $validationContext->getEntityDefinition()->getEntityName(),
-            ['id' => $validationContext->getConvertedData()['id']],
+            $entityDefinition->getEntityName(),
+            ['id' => $id],
         );
 
         $parameters = new WriteParameterBag(
-            $validationContext->getEntityDefinition(),
+            $entityDefinition,
             WriteContext::createFromContext($validationContext->getContext()),
             '',
             new WriteCommandQueue(),
         );
 
-        foreach ($validationContext->getConvertedData() as $fieldName => $value) {
+        foreach ($convertedData as $fieldName => $value) {
             if (!$fields->has($fieldName)) {
                 continue;
             }
@@ -185,47 +216,41 @@ private function validateFields(MigrationValidationContext $validationContext):
                 $serializer = $field->getSerializer();
                 \iterator_to_array($serializer->encode($field, $entityExistence, $keyValue, $parameters), false);
             } catch (\Throwable $e) {
-                $validationContext->getValidationResult()->addLog(
+                $validationResult->addLog(
                     MigrationLogBuilder::fromMigrationContext($validationContext->getMigrationContext())
-                        ->withEntityName($validationContext->getEntityDefinition()->getEntityName())
+                        ->withEntityName($entityDefinition->getEntityName())
                         ->withFieldName($fieldName)
                         ->withConvertedData([$fieldName => $value])
                         ->withSourceData($validationContext->getSourceData())
                         ->withExceptionMessage($e->getMessage())
                         ->withExceptionTrace($e->getTrace())
-                        ->withEntityId($validationContext->getConvertedData()['id'] ?? null)
+                        ->withEntityId($id)
                         ->build(MigrationValidationInvalidFieldValueLog::class)
                 );
             }
         }
     }
 
+    /**
+     * Validates that all foreign key fields reference existing entities by checking the mapping service.
+     */
     private function validateAssociations(MigrationValidationContext $validationContext): void
     {
-        $fields = $validationContext->getEntityDefinition()->getFields();
+        $entityDefinition = $validationContext->getEntityDefinition();
+        $fkFields = $entityDefinition->getFields()->filterInstance(FkField::class);
 
-        $fkFields = array_values(array_map(
-            static fn (Field $field) => $field->getPropertyName(),
-            $fields->filterInstance(FkField::class)->getElements()
-        ));
+        $convertedData = $validationContext->getConvertedData();
+        $validationResult = $validationContext->getValidationResult();
 
-        foreach ($fkFields as $fkFieldName) {
-            if (!isset($validationContext->getConvertedData()[$fkFieldName])) {
-                continue;
-            }
-
-            $fkValue = $validationContext->getConvertedData()[$fkFieldName];
+        /** @var FkField $fkField */
+        foreach ($fkFields as $fkField) {
+            $fkFieldName = $fkField->getPropertyName();
+            $fkValue = $convertedData[$fkFieldName] ?? null;
 
-            if ($fkValue === '') {
+            if ($fkValue === null || $fkValue === '') {
                 continue;
             }
 
-            $fkField = $fields->get($fkFieldName);
-
-            if (!$fkField instanceof FkField) {
-                throw MigrationException::unexpectedNullValue($fkFieldName);
-            }
-
             $referenceEntity = $fkField->getReferenceEntity();
 
             if (!$referenceEntity) {
@@ -240,16 +265,69 @@ private function validateAssociations(MigrationValidationContext $validationCont
             );
 
             if (!$hasMapping) {
-                $validationContext->getValidationResult()->addLog(
+                $validationResult->addLog(
                     MigrationLogBuilder::fromMigrationContext($validationContext->getMigrationContext())
-                        ->withEntityName($validationContext->getEntityDefinition()->getEntityName())
+                        ->withEntityName($entityDefinition->getEntityName())
                         ->withFieldName($fkFieldName)
                         ->withConvertedData([$fkFieldName => $fkValue])
                         ->withSourceData($validationContext->getSourceData())
-                        ->withEntityId($validationContext->getConvertedData()['id'] ?? null)
+                        ->withEntityId($convertedData['id'] ?? null)
                         ->build(MigrationValidationInvalidForeignKeyLog::class)
                 );
             }
         }
     }
+
+    /**
+     * @param array<string> $requiredDbColumns
+     *
+     * @return array<string>
+     */
+    private function filterRequiredFields(CompiledFieldCollection $fields, array $requiredDbColumns): array
+    {
+        $requiredFields = [];
+
+        foreach ($fields->filterByFlag(Required::class) as $field) {
+            if (!($field instanceof StorageAware)) {
+                $requiredFields[] = $field->getPropertyName();
+
+                continue;
+            }
+
+            if (!\in_array($field->getStorageName(), $requiredDbColumns, true)) {
+                continue;
+            }
+
+            $requiredFields[] = $field->getPropertyName();
+        }
+
+        return $requiredFields;
+    }
+
+    /**
+     * Gets the list of required database columns for the given entity and caches the result for future calls.
+     * A required database column is defined as a column that is non-nullable, has no default value, and is not auto-incrementing.
+     *
+     * @return list<string>
+     */
+    private function getRequiredDatabaseColumns(string $entityName): array
+    {
+        if (isset($this->requiredColumnsCache[$entityName])) {
+            return $this->requiredColumnsCache[$entityName];
+        }
+
+        $this->requiredColumnsCache[$entityName] = [];
+
+        $columns = $this->connection
+            ->createSchemaManager()
+            ->listTableColumns($entityName);
+
+        foreach ($columns as $column) {
+            if ($column->getNotnull() && $column->getDefault() === null && !$column->getAutoincrement()) {
+                $this->requiredColumnsCache[$entityName][] = $column->getName();
+            }
+        }
+
+        return $this->requiredColumnsCache[$entityName];
+    }
 }

tests/MigrationServicesTrait.php

@@ -7,6 +7,7 @@
 
 namespace SwagMigrationAssistant\Test;
 
+use Doctrine\DBAL\Connection;
 use Psr\Log\NullLogger;
 use Shopware\Core\Checkout\Cart\Tax\TaxCalculator;
 use Shopware\Core\Checkout\Order\Aggregate\OrderDelivery\OrderDeliveryStates;
@@ -212,6 +213,7 @@ protected function getMigrationDataConverter(
             $this->getContainer()->get('event_dispatcher'),
             $loggingService,
             $mappingService,
+            $this->getContainer()->get(Connection::class),
         );
 
         return new MigrationDataConverter(
@@ -221,7 +223,7 @@ protected function getMigrationDataConverter(
             $loggingService,
             $dataDefinition,
             new DummyMappingService(),
-            $validationService
+            $validationService,
         );
     }