Optimized LCSC batch search calls and extracted it into interface for potential general use in the future

Author: jbtronics

src/Controller/BulkInfoProviderImportController.php

@@ -30,6 +30,8 @@
 use App\Form\InfoProviderSystem\GlobalFieldMappingType;
 use App\Services\InfoProviderSystem\PartInfoRetriever;
 use App\Services\InfoProviderSystem\ExistingPartFinder;
+use App\Services\InfoProviderSystem\ProviderRegistry;
+use App\Services\InfoProviderSystem\Providers\LCSCProvider;
 use Doctrine\ORM\EntityManagerInterface;
 use Psr\Log\LoggerInterface;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
@@ -44,6 +46,7 @@ class BulkInfoProviderImportController extends AbstractController
 {
     public function __construct(
         private readonly PartInfoRetriever $infoRetriever,
+        private readonly LCSCProvider $LCSCProvider,
         private readonly ExistingPartFinder $existingPartFinder,
         private readonly EntityManagerInterface $entityManager
     ) {
@@ -53,7 +56,7 @@ public function __construct(
     public function step1(Request $request, LoggerInterface $exceptionLogger): Response
     {
         $this->denyAccessUnlessGranted('@info_providers.create_parts');
-        
+
         // Increase execution time for bulk operations
         set_time_limit(600); // 10 minutes for large batches
 
@@ -72,7 +75,7 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
             $this->addFlash('error', 'No valid parts found for bulk import');
             return $this->redirectToRoute('homepage');
         }
-        
+
         // Warn about large batches
         if (count($parts) > 50) {
             $this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
@@ -110,7 +113,7 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
             $formData = $form->getData();
             $fieldMappings = $formData['field_mappings'];
             $prefetchDetails = $formData['prefetch_details'] ?? false;
-            
+
             // Debug logging
             $exceptionLogger->info('Form data received', [
                 'prefetch_details' => $prefetchDetails,
@@ -143,13 +146,13 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
                 // Optimize: Use batch async requests for LCSC provider
                 $lcscKeywords = [];
                 $keywordToPartField = [];
-                
+
                 // First, collect all LCSC keywords for batch processing
                 foreach ($parts as $part) {
                     foreach ($fieldMappings as $mapping) {
                         $field = $mapping['field'];
                         $providers = $mapping['providers'] ?? [];
-                        
+
                         if (in_array('lcsc', $providers, true)) {
                             $keyword = $this->getKeywordFromField($part, $field);
                             if ($keyword) {
@@ -291,11 +294,11 @@ function ($dto) use ($dtoMetadata) {
                         'job_id' => $job->getId(),
                         'parts_count' => count($parts)
                     ]);
-                    
+
                     // Delete the job since it has no useful results
                     $this->entityManager->remove($job);
                     $this->entityManager->flush();
-                    
+
                     $this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.');
                     return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
                 }
@@ -304,18 +307,18 @@ function ($dto) use ($dtoMetadata) {
                 $job->setSearchResults($this->serializeSearchResults($searchResults));
                 $job->markAsInProgress();
                 $this->entityManager->flush();
-                
+
             } catch (\Exception $e) {
                 $exceptionLogger->error('Critical error during bulk import search', [
                     'job_id' => $job->getId(),
                     'error' => $e->getMessage(),
                     'exception' => $e
                 ]);
-                
+
                 // Delete the job on critical failure
                 $this->entityManager->remove($job);
                 $this->entityManager->flush();
-                
+
                 $this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
                 return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
             }
@@ -356,19 +359,19 @@ public function manageBulkJobs(): Response
         // Also clean up jobs with no results (failed searches)
         $updatedJobs = false;
         $jobsToDelete = [];
-        
+
         foreach ($allJobs as $job) {
             if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
                 $job->markAsCompleted();
                 $updatedJobs = true;
             }
-            
+
             // Mark jobs with no results for deletion (failed searches)
             if ($job->getResultCount() === 0 && $job->isInProgress()) {
                 $jobsToDelete[] = $job;
             }
         }
-        
+
         // Delete failed jobs
         foreach ($jobsToDelete as $job) {
             $this->entityManager->remove($job);
@@ -378,7 +381,7 @@ public function manageBulkJobs(): Response
         // Flush changes if any jobs were updated
         if ($updatedJobs) {
             $this->entityManager->flush();
-            
+
             if (!empty($jobsToDelete)) {
                 $this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.');
             }
@@ -619,18 +622,7 @@ private function deserializeSearchResults(array $serializedResults, array $parts
      */
     private function searchLcscBatch(array $keywords): array
     {
-        // Get LCSC provider through reflection since PartInfoRetriever doesn't expose it
-        $reflection = new \ReflectionClass($this->infoRetriever);
-        $registryProp = $reflection->getProperty('provider_registry');
-        $registryProp->setAccessible(true);
-        $registry = $registryProp->getValue($this->infoRetriever);
-        
-        $lcscProvider = $registry->getProviderByKey('lcsc');
-        if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) {
-            return $lcscProvider->searchByKeywordsBatch($keywords);
-        }
-        
-        return [];
+        return $this->LCSCProvider->searchByKeywordsBatch($keywords);
     }
 
     #[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
@@ -710,4 +702,4 @@ public function markPartPending(int $jobId, int $partId): Response
             'job_completed' => $job->isCompleted()
         ]);
     }
-}
+}

src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php

@@ -0,0 +1,40 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\Services\InfoProviderSystem\Providers;
+
+use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
+
+/**
+ * This interface marks a provider as a info provider which can provide information directly in batch operations
+ */
+interface BatchInfoProviderInterface extends InfoProviderInterface
+{
+    /**
+     * Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
+     * This allows for a more efficient search compared to running multiple single searches.
+     * @param  string[]  $keywords
+     * @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
+     */
+    public function searchByKeywordsBatch(array $keywords): array;
+}

src/Services/InfoProviderSystem/Providers/LCSCProvider.php

@@ -33,7 +33,7 @@
 use Symfony\Component\HttpFoundation\Cookie;
 use Symfony\Contracts\HttpClient\HttpClientInterface;
 
-class LCSCProvider implements InfoProviderInterface
+class LCSCProvider implements BatchInfoProviderInterface
 {
 
     private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';