Add makefile to help with development setup, change part_ids in bulk import jobs to junction table and implement filtering based on bulk import jobs status and its associated parts' statuses.

Author: barisgit

Makefile

@@ -0,0 +1,99 @@
+# PartDB Makefile for Test Environment Management
+
+.PHONY: help test-setup test-clean test-db-create test-db-migrate test-cache-clear test-fixtures test-run dev-setup dev-clean dev-db-create dev-db-migrate dev-cache-clear dev-warmup dev-reset
+
+# Default target
+help:
+	@echo "PartDB Test Environment Management"
+	@echo "=================================="
+	@echo ""
+	@echo "Available targets:"
+	@echo "  test-setup      - Complete test environment setup (clean, create DB, migrate, load fixtures)"
+	@echo "  test-clean      - Clean test cache and database files"
+	@echo "  test-db-create  - Create test database (if not exists)"
+	@echo "  test-db-migrate - Run database migrations for test environment"
+	@echo "  test-cache-clear- Clear test cache"
+	@echo "  test-fixtures   - Load test fixtures"
+	@echo "  test-run        - Run PHPUnit tests"
+	@echo ""
+	@echo "Development Environment:"
+	@echo "  dev-setup       - Complete development environment setup (clean, create DB, migrate, warmup)"
+	@echo "  dev-clean       - Clean development cache and database files"
+	@echo "  dev-db-create   - Create development database (if not exists)"
+	@echo "  dev-db-migrate  - Run database migrations for development environment"
+	@echo "  dev-cache-clear - Clear development cache"
+	@echo "  dev-warmup      - Warm up development cache"
+	@echo "  dev-reset       - Quick development reset (clean + migrate)"
+	@echo ""
+	@echo "  help           - Show this help message"
+
+# Complete test environment setup
+test-setup: test-clean test-db-create test-db-migrate test-fixtures
+	@echo "✅ Test environment setup complete!"
+
+# Clean test environment
+test-clean:
+	@echo "🧹 Cleaning test environment..."
+	rm -rf var/cache/test
+	rm -f var/app_test.db
+	@echo "✅ Test environment cleaned"
+
+# Create test database
+test-db-create:
+	@echo "🗄️  Creating test database..."
+	-php bin/console doctrine:database:create --if-not-exists --env test || echo "⚠️  Database creation failed (expected for SQLite) - continuing..."
+
+# Run database migrations for test environment
+test-db-migrate:
+	@echo "🔄 Running database migrations..."
+	COMPOSER_MEMORY_LIMIT=-1 php bin/console doctrine:migrations:migrate -n --env test
+
+# Clear test cache
+test-cache-clear:
+	@echo "🗑️  Clearing test cache..."
+	rm -rf var/cache/test
+	@echo "✅ Test cache cleared"
+
+# Load test fixtures
+test-fixtures:
+	@echo "📦 Loading test fixtures..."
+	php bin/console partdb:fixtures:load -n --env test
+
+# Run PHPUnit tests
+test-run:
+	@echo "🧪 Running tests..."
+	php bin/phpunit
+
+# Quick test reset (clean + migrate + fixtures, skip DB creation)
+test-reset: test-cache-clear test-db-migrate test-fixtures
+	@echo "✅ Test environment reset complete!"
+
+# Development helpers
+dev-setup: dev-clean dev-db-create dev-db-migrate dev-warmup
+	@echo "✅ Development environment setup complete!"
+
+dev-clean:
+	@echo "🧹 Cleaning development environment..."
+	rm -rf var/cache/dev
+	rm -f var/app_dev.db
+	@echo "✅ Development environment cleaned"
+
+dev-db-create:
+	@echo "🗄️  Creating development database..."
+	-php bin/console doctrine:database:create --if-not-exists --env dev || echo "⚠️  Database creation failed (expected for SQLite) - continuing..."
+
+dev-db-migrate:
+	@echo "🔄 Running database migrations..."
+	COMPOSER_MEMORY_LIMIT=-1 php bin/console doctrine:migrations:migrate -n --env dev
+
+dev-cache-clear:
+	@echo "🗑️  Clearing development cache..."
+	rm -rf var/cache/dev
+	@echo "✅ Development cache cleared"
+
+dev-warmup:
+	@echo "🔥 Warming up development cache..."
+	COMPOSER_MEMORY_LIMIT=-1 php bin/console cache:warmup --env dev -n --memory-limit=1G
+
+dev-reset: dev-cache-clear dev-db-migrate
+	@echo "✅ Development environment reset complete!" 

migrations/Version20250802153643.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use App\Migration\AbstractMultiPlatformMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20250802205143 extends AbstractMultiPlatformMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add bulk info provider import jobs and job parts tables';
+    }
+
+    public function mySQLUp(Schema $schema): void
+    {
+        $this->addSql('CREATE TABLE bulk_info_provider_import_jobs (id INT AUTO_INCREMENT NOT NULL, name LONGTEXT NOT NULL, field_mappings LONGTEXT NOT NULL, search_results LONGTEXT NOT NULL, status VARCHAR(20) NOT NULL, created_at DATETIME NOT NULL, completed_at DATETIME DEFAULT NULL, prefetch_details TINYINT(1) NOT NULL, created_by_id INT NOT NULL, CONSTRAINT FK_7F58C1EDB03A8386 FOREIGN KEY (created_by_id) REFERENCES `users` (id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ENGINE = InnoDB');
+        $this->addSql('CREATE INDEX IDX_7F58C1EDB03A8386 ON bulk_info_provider_import_jobs (created_by_id)');
+
+        $this->addSql('CREATE TABLE bulk_info_provider_import_job_parts (id INT AUTO_INCREMENT NOT NULL, status VARCHAR(20) NOT NULL, reason LONGTEXT DEFAULT NULL, completed_at DATETIME DEFAULT NULL, job_id INT NOT NULL, part_id INT NOT NULL, CONSTRAINT FK_CD93F28FBE04EA9 FOREIGN KEY (job_id) REFERENCES bulk_info_provider_import_jobs (id), CONSTRAINT FK_CD93F28F4CE34BEC FOREIGN KEY (part_id) REFERENCES `parts` (id), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ENGINE = InnoDB');
+        $this->addSql('CREATE INDEX IDX_CD93F28FBE04EA9 ON bulk_info_provider_import_job_parts (job_id)');
+        $this->addSql('CREATE INDEX IDX_CD93F28F4CE34BEC ON bulk_info_provider_import_job_parts (part_id)');
+        $this->addSql('CREATE UNIQUE INDEX unique_job_part ON bulk_info_provider_import_job_parts (job_id, part_id)');
+    }
+
+    public function mySQLDown(Schema $schema): void
+    {
+        $this->addSql('DROP TABLE bulk_info_provider_import_job_parts');
+        $this->addSql('DROP TABLE bulk_info_provider_import_jobs');
+    }
+
+    public function sqLiteUp(Schema $schema): void
+    {
+        $this->addSql('CREATE TABLE bulk_info_provider_import_jobs (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name CLOB NOT NULL, field_mappings CLOB NOT NULL, search_results CLOB NOT NULL, status VARCHAR(20) NOT NULL, created_at DATETIME NOT NULL, completed_at DATETIME DEFAULT NULL, prefetch_details BOOLEAN NOT NULL, created_by_id INTEGER NOT NULL, CONSTRAINT FK_7F58C1EDB03A8386 FOREIGN KEY (created_by_id) REFERENCES "users" (id) NOT DEFERRABLE INITIALLY IMMEDIATE)');
+        $this->addSql('CREATE INDEX IDX_7F58C1EDB03A8386 ON bulk_info_provider_import_jobs (created_by_id)');
+
+        $this->addSql('CREATE TABLE bulk_info_provider_import_job_parts (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, status VARCHAR(20) NOT NULL, reason CLOB DEFAULT NULL, completed_at DATETIME DEFAULT NULL, job_id INTEGER NOT NULL, part_id INTEGER NOT NULL, CONSTRAINT FK_CD93F28FBE04EA9 FOREIGN KEY (job_id) REFERENCES bulk_info_provider_import_jobs (id) NOT DEFERRABLE INITIALLY IMMEDIATE, CONSTRAINT FK_CD93F28F4CE34BEC FOREIGN KEY (part_id) REFERENCES "parts" (id) NOT DEFERRABLE INITIALLY IMMEDIATE)');
+        $this->addSql('CREATE INDEX IDX_CD93F28FBE04EA9 ON bulk_info_provider_import_job_parts (job_id)');
+        $this->addSql('CREATE INDEX IDX_CD93F28F4CE34BEC ON bulk_info_provider_import_job_parts (part_id)');
+        $this->addSql('CREATE UNIQUE INDEX unique_job_part ON bulk_info_provider_import_job_parts (job_id, part_id)');
+    }
+
+    public function sqLiteDown(Schema $schema): void
+    {
+        $this->addSql('DROP TABLE bulk_info_provider_import_job_parts');
+        $this->addSql('DROP TABLE bulk_info_provider_import_jobs');
+    }
+
+    public function postgreSQLUp(Schema $schema): void
+    {
+        $this->addSql('CREATE TABLE bulk_info_provider_import_jobs (id SERIAL PRIMARY KEY NOT NULL, name TEXT NOT NULL, field_mappings TEXT NOT NULL, search_results TEXT NOT NULL, status VARCHAR(20) NOT NULL, created_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, completed_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, prefetch_details BOOLEAN NOT NULL, created_by_id INT NOT NULL, CONSTRAINT FK_7F58C1EDB03A8386 FOREIGN KEY (created_by_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE)');
+        $this->addSql('CREATE INDEX IDX_7F58C1EDB03A8386 ON bulk_info_provider_import_jobs (created_by_id)');
+
+        $this->addSql('CREATE TABLE bulk_info_provider_import_job_parts (id SERIAL PRIMARY KEY NOT NULL, status VARCHAR(20) NOT NULL, reason TEXT DEFAULT NULL, completed_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, job_id INT NOT NULL, part_id INT NOT NULL, CONSTRAINT FK_CD93F28FBE04EA9 FOREIGN KEY (job_id) REFERENCES bulk_info_provider_import_jobs (id) NOT DEFERRABLE INITIALLY IMMEDIATE, CONSTRAINT FK_CD93F28F4CE34BEC FOREIGN KEY (part_id) REFERENCES parts (id) NOT DEFERRABLE INITIALLY IMMEDIATE)');
+        $this->addSql('CREATE INDEX IDX_CD93F28FBE04EA9 ON bulk_info_provider_import_job_parts (job_id)');
+        $this->addSql('CREATE INDEX IDX_CD93F28F4CE34BEC ON bulk_info_provider_import_job_parts (part_id)');
+        $this->addSql('CREATE UNIQUE INDEX unique_job_part ON bulk_info_provider_import_job_parts (job_id, part_id)');
+    }
+
+    public function postgreSQLDown(Schema $schema): void
+    {
+        $this->addSql('DROP TABLE bulk_info_provider_import_job_parts');
+        $this->addSql('DROP TABLE bulk_info_provider_import_jobs');
+    }
+}

migrations/Version20250802205143.php

@@ -23,6 +23,7 @@
 namespace App\Controller;
 
 use App\Entity\BulkInfoProviderImportJob;
+use App\Entity\BulkInfoProviderImportJobPart;
 use App\Entity\BulkImportJobStatus;
 use App\Entity\Parts\Part;
 use App\Entity\Parts\Supplier;
@@ -104,7 +105,6 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
 
             // Create and save the job
             $job = new BulkInfoProviderImportJob();
-            $job->setPartIds(array_map(fn($part) => $part->getId(), $parts));
             $job->setFieldMappings($fieldMappings);
             $job->setPrefetchDetails($prefetchDetails);
             $user = $this->getUser();
@@ -113,6 +113,12 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
             }
             $job->setCreatedBy($user);
 
+            // Create job parts for each part
+            foreach ($parts as $part) {
+                $jobPart = new BulkInfoProviderImportJobPart($job, $part);
+                $job->addJobPart($jobPart);
+            }
+
             $this->entityManager->persist($job);
             $this->entityManager->flush();
 
@@ -179,7 +185,7 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
 
                 // Convert DTOs to result format with metadata
                 $partResult['search_results'] = array_map(
-                    function($dto) use ($dtoMetadata) {
+                    function ($dto) use ($dtoMetadata) {
                         $dtoKey = $dto->provider_key . '|' . $dto->provider_id;
                         $metadata = $dtoMetadata[$dtoKey] ?? [];
                         return [
@@ -372,8 +378,7 @@ public function step2(int $jobId): Response
         }
 
         // Get the parts and deserialize search results
-        $partRepository = $this->entityManager->getRepository(Part::class);
-        $parts = $partRepository->getElementsFromIDArray($job->getPartIds());
+        $parts = $job->getJobParts()->map(fn($jobPart) => $jobPart->getPart())->toArray();
         $searchResults = $this->deserializeSearchResults($job->getSearchResults(), $parts);
 
         return $this->render('info_providers/bulk_import/step2.html.twig', [

src/Controller/BulkInfoProviderImportController.php

@@ -0,0 +1,82 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace App\DataTables\Filters\Constraints\Part;
+
+use App\DataTables\Filters\Constraints\AbstractConstraint;
+use App\Entity\BulkInfoProviderImportJobPart;
+use Doctrine\ORM\QueryBuilder;
+
+class BulkImportJobExistsConstraint extends AbstractConstraint
+{
+    /** @var bool|null The value of our constraint */
+    protected ?bool $value = null;
+
+    public function __construct()
+    {
+        parent::__construct('bulk_import_job_exists');
+    }
+
+    /**
+     * Gets the value of this constraint. Null means "don't filter", true means "filter for parts in bulk import jobs", false means "filter for parts not in bulk import jobs".
+     */
+    public function getValue(): ?bool
+    {
+        return $this->value;
+    }
+
+    /**
+     * Sets the value of this constraint. Null means "don't filter", true means "filter for parts in bulk import jobs", false means "filter for parts not in bulk import jobs".
+     */
+    public function setValue(?bool $value): void
+    {
+        $this->value = $value;
+    }
+
+    public function isEnabled(): bool
+    {
+        return $this->value !== null;
+    }
+
+    public function apply(QueryBuilder $queryBuilder): void
+    {
+        // Do not apply a filter if value is null (filter is set to ignore)
+        if (!$this->isEnabled()) {
+            return;
+        }
+
+        // Use EXISTS subquery to avoid join conflicts
+        $existsSubquery = $queryBuilder->getEntityManager()->createQueryBuilder();
+        $existsSubquery->select('1')
+            ->from(BulkInfoProviderImportJobPart::class, 'bip_exists')
+            ->where('bip_exists.part = part.id');
+
+        if ($this->value === true) {
+            // Filter for parts that ARE in bulk import jobs
+            $queryBuilder->andWhere('EXISTS (' . $existsSubquery->getDQL() . ')');
+        } else {
+            // Filter for parts that are NOT in bulk import jobs
+            $queryBuilder->andWhere('NOT EXISTS (' . $existsSubquery->getDQL() . ')');
+        }
+    }
+}