Skip to content

Commit 504ff98

Browse files
npuchkonpuchko
committed
ACP2E-1930: Import/Export memory overflow issue
1 parent 827073a commit 504ff98

File tree

19 files changed

+843
-175
lines changed

19 files changed

+843
-175
lines changed

‎app/code/Magento/CatalogImportExport/Model/Import/Product.php

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use Magento\CatalogImportExport\Model\Import\Product\MediaGalleryProcessor;
1717
use Magento\CatalogImportExport\Model\Import\Product\RowValidatorInterface as ValidatorInterface;
1818
use Magento\CatalogImportExport\Model\Import\Product\Skip;
19+
use Magento\CatalogImportExport\Model\Import\Product\SkuStorage;
1920
use Magento\CatalogImportExport\Model\Import\Product\StatusProcessor;
2021
use Magento\CatalogImportExport\Model\Import\Product\StockProcessor;
2122
use Magento\CatalogImportExport\Model\Import\Product\Type\AbstractType;
@@ -768,6 +769,11 @@ class Product extends AbstractEntity
768769
*/
769770
private $stockItemProcessor;
770771

772+
/**
773+
* @var SkuStorage|null
774+
*/
775+
private ?SkuStorage $skuStorage;
776+
771777
/**
772778
* @param \Magento\Framework\Json\Helper\Data $jsonHelper
773779
* @param \Magento\ImportExport\Helper\Data $importExportData
@@ -818,6 +824,7 @@ class Product extends AbstractEntity
818824
* @param LinkProcessor|null $linkProcessor
819825
* @param File|null $fileDriver
820826
* @param StockItemProcessorInterface|null $stockItemProcessor
827+
* @param SkuStorage|null $skuStorage
821828
* @throws LocalizedException
822829
* @throws \Magento\Framework\Exception\FileSystemException
823830
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
@@ -873,7 +880,8 @@ public function __construct(
873880
StockProcessor $stockProcessor = null,
874881
LinkProcessor $linkProcessor = null,
875882
?File $fileDriver = null,
876-
?StockItemProcessorInterface $stockItemProcessor = null
883+
?StockItemProcessorInterface $stockItemProcessor = null,
884+
?SkuStorage $skuStorage = null
877885
) {
878886
$this->_eventManager = $eventManager;
879887
$this->stockRegistry = $stockRegistry;
@@ -929,6 +937,8 @@ public function __construct(
929937
);
930938
$this->_optionEntity = $data['option_entity'] ??
931939
$optionFactory->create(['data' => ['product_entity' => $this]]);
940+
$this->skuStorage = $skuStorage ?? ObjectManager::getInstance()
941+
->get(SkuStorage::class);
932942
$this->_initAttributeSets()
933943
->_initTypeModels()
934944
->_initSkus()
@@ -1141,7 +1151,7 @@ protected function _importData()
11411151
protected function _replaceProducts()
11421152
{
11431153
$this->deleteProductsForReplacement();
1144-
$this->_oldSku = $this->skuProcessor->reloadOldSkus()->getOldSkus();
1154+
$this->skuStorage->reset();
11451155
$this->_validatedRows = null;
11461156
$this->setParameters(
11471157
array_merge(
@@ -1203,7 +1213,7 @@ protected function _initAttributeSets()
12031213
protected function _initSkus()
12041214
{
12051215
$this->skuProcessor->setTypeModels($this->_productTypeModels);
1206-
$this->_oldSku = $this->skuProcessor->reloadOldSkus()->getOldSkus();
1216+
$this->skuStorage->reset();
12071217
return $this;
12081218
}
12091219

@@ -1336,7 +1346,7 @@ protected function _saveProductAttributes(array $attributesData)
13361346
$linkIdBySkuForStatusChanged = [];
13371347
$tableData = [];
13381348
foreach ($skuData as $sku => $attributes) {
1339-
$linkId = $this->_oldSku[strtolower($sku)][$linkField];
1349+
$linkId = $this->skuStorage->get((string)$sku)[$linkField];
13401350
foreach ($attributes as $attributeId => $storeValues) {
13411351
foreach ($storeValues as $storeId => $storeValue) {
13421352
if ($attributeId === $statusAttributeId) {
@@ -1459,7 +1469,7 @@ public function saveProductEntity(array $entityRowsIn, array $entityRowsUp)
14591469
$this->skuProcessor->setNewSkuData($sku, $key, $value);
14601470
}
14611471
}
1462-
$this->updateOldSku($newProducts);
1472+
$this->updateSkuStorage($newProducts);
14631473
}
14641474
return $this;
14651475
}
@@ -1480,22 +1490,11 @@ private function getOldSkuFieldsForSelect()
14801490
* @param array $newProducts
14811491
* @return void
14821492
*/
1483-
private function updateOldSku(array $newProducts)
1493+
private function updateSkuStorage(array $newProducts): void
14841494
{
1485-
$oldSkus = [];
14861495
foreach ($newProducts as $info) {
1487-
$typeId = $info['type_id'];
1488-
$sku = strtolower($info['sku']);
1489-
$oldSkus[$sku] = [
1490-
'type_id' => $typeId,
1491-
'attr_set_id' => $info['attribute_set_id'],
1492-
$this->getProductIdentifierField() => $info[$this->getProductIdentifierField()],
1493-
'supported_type' => isset($this->_productTypeModels[$typeId]),
1494-
$this->getProductEntityLinkField() => $info[$this->getProductEntityLinkField()],
1495-
];
1496+
$this->skuStorage->set($info);
14961497
}
1497-
1498-
$this->_oldSku = array_replace($this->_oldSku, $oldSkus);
14991498
}
15001499

15011500
/**
@@ -2566,10 +2565,19 @@ public function getNextBunch()
25662565
* new products with the same SKU in different letter cases.
25672566
*
25682567
* @return array
2568+
* @deprecated This method is deprecated due to high memory consumption.
2569+
* @see SkuStorage
25692570
*/
25702571
public function getOldSku()
25712572
{
2572-
return $this->_oldSku;
2573+
// For backward compatibility get all data from storage
2574+
$oldSkus = [];
2575+
foreach ($this->skuStorage->iterate() as $sku => $value) {
2576+
$oldSkus[$sku] = $value;
2577+
$oldSkus[$sku]['supported_type'] = isset($this->_productTypeModels[$value['type_id']]);
2578+
}
2579+
2580+
return $oldSkus;
25732581
}
25742582

25752583
/**
@@ -3217,8 +3225,7 @@ private function parseMultipleValues($labelRow)
32173225
private function isSkuExist($sku)
32183226
{
32193227
if ($sku !== null) {
3220-
$sku = strtolower($sku);
3221-
return isset($this->_oldSku[$sku]);
3228+
return $this->skuStorage->has($sku);
32223229
}
32233230
return false;
32243231
}
@@ -3231,7 +3238,7 @@ private function isSkuExist($sku)
32313238
*/
32323239
private function getExistingSku($sku)
32333240
{
3234-
return $this->_oldSku[strtolower($sku)];
3241+
return $this->skuStorage->get((string)$sku);
32353242
}
32363243

32373244
/**

‎app/code/Magento/CatalogImportExport/Model/Import/Product/LinkProcessor.php

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ class LinkProcessor
4646
*/
4747
private $logger;
4848

49+
/**
50+
* @var SkuStorage
51+
*/
52+
private SkuStorage $skuStorage;
53+
4954
/**
5055
* LinkProcessor constructor.
5156
*
@@ -54,20 +59,23 @@ class LinkProcessor
5459
* @param SkuProcessor $skuProcessor
5560
* @param LoggerInterface $logger
5661
* @param array $linkNameToId
62+
* @param SkuStorage $skuStorage
5763
*/
5864
public function __construct(
5965
LinkFactory $linkFactory,
6066
Helper $resourceHelper,
6167
SkuProcessor $skuProcessor,
6268
LoggerInterface $logger,
63-
array $linkNameToId
69+
array $linkNameToId,
70+
SkuStorage $skuStorage
6471
) {
6572
$this->linkFactory = $linkFactory;
6673
$this->resourceHelper = $resourceHelper;
6774
$this->skuProcessor = $skuProcessor;
6875
$this->logger = $logger;
6976

7077
$this->linkNameToId = $linkNameToId;
78+
$this->skuStorage = $skuStorage;
7179
}
7280

7381
/**
@@ -171,10 +179,10 @@ private function processLinkBunches(
171179
? explode($importEntity->getMultipleValueSeparator(), $rowData[$linkName . 'position'])
172180
: [];
173181

174-
$linkSkus = $this->filterValidLinks($importEntity, $sku, $linkSkus);
182+
$linkSkus = $this->filterValidLinks($sku, $linkSkus);
175183

176184
foreach ($linkSkus as $linkedKey => $linkedSku) {
177-
$linkedId = $this->getProductLinkedId($importEntity, $linkedSku);
185+
$linkedId = $this->getProductLinkedId($linkedSku);
178186
if ($linkedId == null) {
179187
// Import file links to a SKU which is skipped for some reason, which leads to a "NULL"
180188
// link causing fatal errors.
@@ -222,7 +230,7 @@ private function deleteProductsLinks(
222230
Product $importEntity,
223231
Link $resource,
224232
array $linksToDelete
225-
) {
233+
): void {
226234
if (!empty($linksToDelete) && Import::BEHAVIOR_APPEND === $importEntity->getBehavior()) {
227235
foreach ($linksToDelete as $linkTypeId => $productIds) {
228236
if (!empty($productIds)) {
@@ -243,27 +251,23 @@ private function deleteProductsLinks(
243251
/**
244252
* Check if product exists for specified SKU
245253
*
246-
* @param Product $importEntity
247254
* @param string $sku
248255
* @return bool
249256
*/
250-
private function isSkuExist(Product $importEntity, string $sku): bool
257+
private function isSkuExist(string $sku): bool
251258
{
252-
$sku = strtolower($sku);
253-
return isset($importEntity->getOldSku()[$sku]);
259+
return $this->skuStorage->has($sku);
254260
}
255261

256262
/**
257263
* Get existing SKU record
258264
*
259-
* @param Product $importEntity
260265
* @param string $sku
261-
* @return mixed
266+
* @return array|null
262267
*/
263-
private function getExistingSku(Product $importEntity, string $sku)
268+
private function getExistingSku(string $sku): ?array
264269
{
265-
$sku = strtolower($sku);
266-
return $importEntity->getOldSku()[$sku];
270+
return $this->skuStorage->get($sku);
267271
}
268272

269273
/**
@@ -296,20 +300,17 @@ private function fetchProductLinks(Product $importEntity, Link $resource, int $p
296300
/**
297301
* Gets the Id of the Sku
298302
*
299-
* @param Product $importEntity
300303
* @param string $linkedSku
301304
* @return int|null
302305
*/
303-
private function getProductLinkedId(Product $importEntity, string $linkedSku): ?int
306+
private function getProductLinkedId(string $linkedSku): ?int
304307
{
305308
$linkedSku = trim($linkedSku);
306309
$newSku = $this->skuProcessor->getNewSku($linkedSku);
307310

308-
$linkedId = ! empty($newSku) ?
311+
return !empty($newSku) ?
309312
$newSku['entity_id'] :
310-
$this->getExistingSku($importEntity, $linkedSku)['entity_id'];
311-
312-
return $linkedId;
313+
$this->getExistingSku($linkedSku)['entity_id'];
313314
}
314315

315316
/**
@@ -329,7 +330,7 @@ private function saveLinksData(
329330
array $productIds,
330331
array $linkRows,
331332
array $positionRows
332-
) {
333+
): void {
333334
$mainTable = $resource->getMainTable();
334335
if (Import::BEHAVIOR_APPEND != $importEntity->getBehavior() && $productIds) {
335336
$importEntity->getConnection()->delete(
@@ -370,7 +371,7 @@ private function composeLinkKey(int $productId, int $linkedId, int $linkTypeId):
370371
* @param array $rowData
371372
* @return array
372373
*/
373-
private function filterProvidedLinkTypes(array $rowData)
374+
private function filterProvidedLinkTypes(array $rowData): array
374375
{
375376
return array_filter(
376377
$this->linkNameToId,
@@ -384,21 +385,20 @@ function ($linkName) use ($rowData) {
384385
/**
385386
* Filter out invalid links
386387
*
387-
* @param Product $importEntity
388388
* @param string $sku
389389
* @param array $linkSkus
390390
* @return array
391391
*/
392-
private function filterValidLinks(Product $importEntity, string $sku, array $linkSkus)
392+
private function filterValidLinks(string $sku, array $linkSkus): array
393393
{
394394
return array_filter(
395395
$linkSkus,
396-
function ($linkedSku) use ($sku, $importEntity) {
396+
function ($linkedSku) use ($sku) {
397397
$linkedSku = $linkedSku !== null ? trim($linkedSku) : '';
398398

399399
return (
400400
$this->skuProcessor->getNewSku($linkedSku) !== null
401-
|| $this->isSkuExist($importEntity, $linkedSku)
401+
|| $this->isSkuExist($linkedSku)
402402
)
403403
&& strcasecmp($linkedSku, $sku) !== 0;
404404
}

0 commit comments

Comments
 (0)