Skip to content

Commit 7a2437f

Browse files
bubasumabubasuma
committed
ACP2E-1675: [Cloud] Deployment issue due to exceeded memory and large tables
1 parent 647b34e commit 7a2437f

File tree

2 files changed

+157
-43
lines changed

2 files changed

+157
-43
lines changed

‎app/code/Magento/Catalog/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypes.php

Lines changed: 50 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,53 @@
99
use Magento\Catalog\Model\Product;
1010
use Magento\Eav\Setup\EavSetup;
1111
use Magento\Eav\Setup\EavSetupFactory;
12-
use Magento\Framework\Exception\LocalizedException;
12+
use Magento\Framework\DB\Query\Generator;
13+
use Magento\Framework\DB\Select;
14+
use Magento\Framework\DB\Sql\Expression;
1315
use Magento\Framework\Setup\ModuleDataSetupInterface;
1416
use Magento\Framework\Setup\Patch\DataPatchInterface;
1517

1618
class UpdateMultiselectAttributesBackendTypes implements DataPatchInterface
1719
{
20+
private const BATCH_SIZE = 10000;
21+
1822
/**
1923
* @var ModuleDataSetupInterface
2024
*/
2125
private $dataSetup;
26+
2227
/**
2328
* @var EavSetupFactory
2429
*/
2530
private $eavSetupFactory;
2631

32+
/**
33+
* @var Generator
34+
*/
35+
private Generator $batchQueryGenerator;
36+
37+
/**
38+
* @var int
39+
*/
40+
private int $batchSize;
41+
2742
/**
2843
* MigrateMultiselectAttributesData constructor.
2944
* @param ModuleDataSetupInterface $dataSetup
3045
* @param EavSetupFactory $eavSetupFactory
46+
* @param Generator $batchQueryGenerator
47+
* @param int $batchSize
3148
*/
3249
public function __construct(
3350
ModuleDataSetupInterface $dataSetup,
34-
EavSetupFactory $eavSetupFactory
51+
EavSetupFactory $eavSetupFactory,
52+
Generator $batchQueryGenerator,
53+
int $batchSize = self::BATCH_SIZE
3554
) {
3655
$this->dataSetup = $dataSetup;
3756
$this->eavSetupFactory = $eavSetupFactory;
57+
$this->batchQueryGenerator = $batchQueryGenerator;
58+
$this->batchSize = $batchSize;
3859
}
3960

4061
/**
@@ -77,23 +98,36 @@ public function apply()
7798

7899
$varcharTable = $setup->getTable('catalog_product_entity_varchar');
79100
$textTable = $setup->getTable('catalog_product_entity_text');
80-
$varcharTableDataSql = $connection
81-
->select()
82-
->from($varcharTable)
83-
->where('attribute_id in (?)', $attributesToMigrate);
84101

85102
$columns = $connection->describeTable($varcharTable);
86-
unset($columns['value_id']);
87-
$connection->query(
88-
$connection->insertFromSelect(
89-
$connection->select()
90-
->from($varcharTable, array_keys($columns))
91-
->where('attribute_id in (?)', $attributesToMigrate),
92-
$textTable,
93-
array_keys($columns)
94-
)
103+
$primaryKey = 'value_id';
104+
unset($columns[$primaryKey]);
105+
$columnNames = array_keys($columns);
106+
$batchIterator = $this->batchQueryGenerator->generate(
107+
$primaryKey,
108+
$connection->select()
109+
->from($varcharTable)
110+
->where('attribute_id in (?)', $attributesToMigrate),
111+
$this->batchSize
95112
);
96-
$connection->query($connection->deleteFromSelect($varcharTableDataSql, $varcharTable));
113+
foreach ($batchIterator as $select) {
114+
$selectForInsert = clone $select;
115+
$selectForInsert->reset(Select::COLUMNS);
116+
$selectForInsert->columns($columnNames);
117+
$connection->query(
118+
$connection->insertFromSelect($selectForInsert, $textTable, $columnNames)
119+
);
120+
$selectForDelete = clone $select;
121+
$selectForDelete->reset(Select::COLUMNS);
122+
$selectForDelete->columns($primaryKey);
123+
$selectForDelete = $connection->select()
124+
->from($selectForDelete, $primaryKey);
125+
126+
$connection->delete(
127+
$varcharTable,
128+
new Expression($primaryKey . ' IN (' . $selectForDelete->assemble() . ')')
129+
);
130+
}
97131

98132
foreach ($attributesToMigrate as $attributeId) {
99133
$eavSetup->updateAttribute($entityTypeId, $attributeId, 'backend_type', 'text');

‎app/code/Magento/Catalog/Test/Unit/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypesTest.php

Lines changed: 107 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,30 @@
1111
use Magento\Eav\Setup\EavSetup;
1212
use Magento\Eav\Setup\EavSetupFactory;
1313
use Magento\Framework\DB\Adapter\AdapterInterface;
14+
use Magento\Framework\DB\Query\BatchIteratorInterface;
15+
use Magento\Framework\DB\Query\Generator;
1416
use Magento\Framework\DB\Select;
1517
use Magento\Framework\Setup\ModuleDataSetupInterface;
18+
use PHPUnit\Framework\MockObject\MockObject;
1619
use PHPUnit\Framework\TestCase;
1720

1821
class UpdateMultiselectAttributesBackendTypesTest extends TestCase
1922
{
2023
/**
21-
* @var ModuleDataSetupInterface|\PHPUnit\Framework\MockObject\MockObject
24+
* @var ModuleDataSetupInterface|MockObject
2225
*/
2326
private $dataSetup;
2427

2528
/**
26-
* @var EavSetupFactory|\PHPUnit\Framework\MockObject\MockObject
29+
* @var EavSetupFactory|MockObject
2730
*/
2831
private $eavSetupFactory;
2932

33+
/**
34+
* @var Generator|MockObject
35+
*/
36+
private $batchQueryGenerator;
37+
3038
/**
3139
* @var UpdateMultiselectAttributesBackendTypes
3240
*/
@@ -40,35 +48,64 @@ protected function setUp(): void
4048
parent::setUp();
4149
$this->dataSetup = $this->createMock(ModuleDataSetupInterface::class);
4250
$this->eavSetupFactory = $this->createMock(EavSetupFactory::class);
43-
$this->model = new UpdateMultiselectAttributesBackendTypes($this->dataSetup, $this->eavSetupFactory);
51+
$this->batchQueryGenerator = $this->createMock(Generator::class);
52+
$this->model = new UpdateMultiselectAttributesBackendTypes(
53+
$this->dataSetup,
54+
$this->eavSetupFactory,
55+
$this->batchQueryGenerator
56+
);
4457
}
4558

59+
/**
60+
* @SuppressWarnings(PHPMD.ExcessiveMethodLength)
61+
* @return void
62+
*/
4663
public function testApply(): void
4764
{
4865
$attributeIds = [3, 7];
4966
$entityTypeId = 4;
5067
$eavSetup = $this->createMock(EavSetup::class);
5168
$connection = $this->createMock(AdapterInterface::class);
52-
$select1 = $this->createMock(Select::class);
53-
$select2 = $this->createMock(Select::class);
54-
$select3 = $this->createMock(Select::class);
55-
$this->eavSetupFactory->method('create')
69+
$selectAttributes = $this->createMock(Select::class);
70+
$selectAttributesValues = $this->createMock(Select::class);
71+
$selectForInsert1 = $this->createMock(Select::class);
72+
$selectForInsert2 = $this->createMock(Select::class);
73+
$selectForDelete1 = $this->createMock(Select::class);
74+
$selectForDelete2 = $this->createMock(Select::class);
75+
$batchIterator = $this->getMockForAbstractClass(BatchIteratorInterface::class);
76+
$batchIterator->method('current')
77+
->willReturnOnConsecutiveCalls($selectForInsert1, $selectForInsert2);
78+
$batchIterator->method('valid')
79+
->willReturnOnConsecutiveCalls(true, true, false);
80+
$this->eavSetupFactory->expects($this->once())
81+
->method('create')
5682
->willReturn($eavSetup);
57-
$this->dataSetup->method('getConnection')
83+
$this->dataSetup->expects($this->once())
84+
->method('getConnection')
5885
->willReturn($connection);
5986
$this->dataSetup->method('getTable')
6087
->willReturnArgument(0);
88+
$this->batchQueryGenerator->expects($this->once())
89+
->method('generate')
90+
->willReturn($batchIterator);
6191
$eavSetup->method('getEntityTypeId')
6292
->willReturn(4);
63-
$eavSetup->method('updateAttribute')
93+
$eavSetup->expects($this->exactly(2))
94+
->method('updateAttribute')
6495
->withConsecutive(
6596
[$entityTypeId, 3, 'backend_type', 'text'],
6697
[$entityTypeId, 7, 'backend_type', 'text']
6798
);
68-
$connection->expects($this->exactly(3))
99+
$connection->expects($this->exactly(4))
69100
->method('select')
70-
->willReturnOnConsecutiveCalls($select1, $select2, $select3);
71-
$connection->method('describeTable')
101+
->willReturnOnConsecutiveCalls(
102+
$selectAttributes,
103+
$selectAttributesValues,
104+
$selectForDelete1,
105+
$selectForDelete2
106+
);
107+
$connection->expects($this->once())
108+
->method('describeTable')
72109
->willReturn(
73110
[
74111
'value_id' => [],
@@ -78,35 +115,78 @@ public function testApply(): void
78115
'row_id' => [],
79116
]
80117
);
81-
$connection->method('fetchCol')
82-
->with($select1)
118+
$connection->expects($this->once())
119+
->method('fetchCol')
120+
->with($selectAttributes)
83121
->willReturn($attributeIds);
84-
$connection->method('insertFromSelect')
85-
->with($select3, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id'])
122+
$connection->expects($this->exactly(2))
123+
->method('insertFromSelect')
124+
->withConsecutive(
125+
[$selectForInsert1, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id']],
126+
[$selectForInsert2, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id']],
127+
)
86128
->willReturn('');
87-
$connection->method('deleteFromSelect')
88-
->with($select2, 'catalog_product_entity_varchar')
129+
$connection->expects($this->exactly(2))
130+
->method('delete')
131+
->withConsecutive(
132+
['catalog_product_entity_varchar', 'value_id IN (select1)'],
133+
['catalog_product_entity_varchar', 'value_id IN (select2)'],
134+
)
89135
->willReturn('');
90-
$select1->method('from')
136+
$selectAttributes->expects($this->once())
137+
->method('from')
91138
->with('eav_attribute', ['attribute_id'])
92139
->willReturnSelf();
93-
$select1->method('where')
140+
$selectAttributes->expects($this->exactly(3))
141+
->method('where')
94142
->withConsecutive(
95143
['entity_type_id = ?', $entityTypeId],
96144
['backend_type = ?', 'varchar'],
97145
['frontend_input = ?', 'multiselect']
98146
)
99147
->willReturnSelf();
100-
$select2->method('from')
101-
->with('catalog_product_entity_varchar')
148+
$selectForInsert1->expects($this->exactly(2))
149+
->method('reset')
150+
->with(Select::COLUMNS)
102151
->willReturnSelf();
103-
$select2->method('where')
104-
->with('attribute_id in (?)', $attributeIds)
152+
$selectForInsert1->expects($this->exactly(2))
153+
->method('columns')
154+
->withConsecutive(
155+
[['attribute_id', 'store_id', 'value', 'row_id']],
156+
['value_id']
157+
)
158+
->willReturnSelf();
159+
$selectForInsert2->expects($this->exactly(2))
160+
->method('reset')
161+
->with(Select::COLUMNS)
162+
->willReturnSelf();
163+
$selectForInsert2->expects($this->exactly(2))
164+
->method('columns')
165+
->withConsecutive(
166+
[['attribute_id', 'store_id', 'value', 'row_id']],
167+
['value_id']
168+
)
169+
->willReturnSelf();
170+
$selectForDelete1->expects($this->once())
171+
->method('from')
172+
->with($selectForInsert1, 'value_id')
173+
->willReturnSelf();
174+
$selectForDelete1->expects($this->once())
175+
->method('assemble')
176+
->willReturn('select1');
177+
$selectForDelete2->expects($this->once())
178+
->method('from')
179+
->with($selectForInsert2, 'value_id')
105180
->willReturnSelf();
106-
$select3->method('from')
107-
->with('catalog_product_entity_varchar', ['attribute_id', 'store_id', 'value', 'row_id'])
181+
$selectForDelete2->expects($this->once())
182+
->method('assemble')
183+
->willReturn('select2');
184+
$selectAttributesValues->expects($this->once())
185+
->method('from')
186+
->with('catalog_product_entity_varchar', '*')
108187
->willReturnSelf();
109-
$select3->method('where')
188+
$selectAttributesValues->expects($this->once())
189+
->method('where')
110190
->with('attribute_id in (?)', $attributeIds)
111191
->willReturnSelf();
112192
$this->model->apply();

0 commit comments

Comments
 (0)