ACP2E-4303: Reindexing stuck due to high memory usage

Author: alzota

app/code/Magento/CatalogRule/Model/Indexer/IndexBuilder.php

@@ -20,6 +20,7 @@
 use Magento\CatalogRule\Model\ResourceModel\Rule\RuleIdProvider;
 use Magento\CatalogRule\Model\Rule;
 use Magento\CatalogRule\Model\RuleFactory;
+use Magento\Customer\Api\GroupExcludedWebsiteRepositoryInterface;
 use Magento\Eav\Model\Config;
 use Magento\Framework\App\ObjectManager;
 use Magento\Framework\App\ResourceConnection;
@@ -214,6 +215,11 @@ class IndexBuilder
      */
     private $ruleFactory;
 
+    /**
+     * @var GroupExcludedWebsiteRepositoryInterface
+     */
+    private $groupExcludedWebsiteRepository;
+
     /**
      * @param RuleCollectionFactory $ruleCollectionFactory
      * @param PriceCurrencyInterface $priceCurrency
@@ -243,6 +249,7 @@ class IndexBuilder
      * @param CatalogRuleInsertBatchSizeCalculator|null $insertBatchSizeCalculator
      * @param RuleIdProvider|null $ruleIdProvider
      * @param RuleFactory|null $ruleFactory
+     * @param GroupExcludedWebsiteRepositoryInterface|null $groupExcludedWebsiteRepository
      * @SuppressWarnings(PHPMD.ExcessiveParameterList)
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
@@ -274,7 +281,8 @@ public function __construct(
         ?DynamicBatchSizeCalculator $batchSizeCalculator = null,
         ?CatalogRuleInsertBatchSizeCalculator $insertBatchSizeCalculator = null,
         ?RuleIdProvider $ruleIdProvider = null,
-        ?RuleFactory $ruleFactory = null
+        ?RuleFactory $ruleFactory = null,
+        ?GroupExcludedWebsiteRepositoryInterface $groupExcludedWebsiteRepository = null
     ) {
         $this->resource = $resource;
         $this->connection = $resource->getConnection();
@@ -331,6 +339,8 @@ public function __construct(
             ObjectManager::getInstance()->get(RuleIdProvider::class);
         $this->ruleFactory = $ruleFactory ??
             ObjectManager::getInstance()->get(RuleFactory::class);
+        $this->groupExcludedWebsiteRepository = $groupExcludedWebsiteRepository ??
+            ObjectManager::getInstance()->get(GroupExcludedWebsiteRepositoryInterface::class);
     }
 
     /**
@@ -443,13 +453,20 @@ protected function doReindexFull()
     {
         $dynamicBatchCount = $this->insertBatchSizeCalculator->getInsertBatchSize($this->connection);
         $ruleIds = $this->getActiveRuleIds();
+
+        $allExcludedWebsites = $this->groupExcludedWebsiteRepository->getAllExcludedWebsites();
+
         foreach ($ruleIds as $ruleId) {
 
             $rule = $this->loadRuleById($ruleId);
             if (!$rule) {
                 $this->logger->warning("Rule ID {$ruleId} not found, skipping");
                 continue;
             }
+
+            $ruleExcludedWebsites = $this->filterExcludedWebsitesForRule($rule, $allExcludedWebsites);
+            $rule->setData('excluded_website_ids', $ruleExcludedWebsites);
+
             $this->reindexRuleProduct->execute($rule, $dynamicBatchCount, true);
 
             $rule->clearInstance();
@@ -780,6 +797,33 @@ protected function loadRuleById($ruleId)
         return $rule->getId() ? $rule : null;
     }
 
+    /**
+     * Filter excluded websites for a specific rule based on its customer groups
+     *
+     * @param Rule $rule
+     * @param array $allExcludedWebsites
+     * @return array
+     */
+    private function filterExcludedWebsitesForRule(Rule $rule, array $allExcludedWebsites): array
+    {
+        $ruleExcludedWebsites = [];
+        $customerGroupIds = $rule->getCustomerGroupIds();
+
+        if (empty($customerGroupIds) || empty($allExcludedWebsites)) {
+            return $ruleExcludedWebsites;
+        }
+
+        foreach ($customerGroupIds as $customerGroupId) {
+            if (isset($allExcludedWebsites[$customerGroupId])) {
+                foreach ($allExcludedWebsites[$customerGroupId] as $websiteId) {
+                    $ruleExcludedWebsites[$websiteId] = $websiteId;
+                }
+            }
+        }
+
+        return array_values($ruleExcludedWebsites);
+    }
+
     /**
      * Get active rules
      *

app/code/Magento/CatalogRule/Model/Indexer/ReindexRuleProduct.php

@@ -173,9 +173,12 @@ private function prepareRuleData(Rule $rule): array
         $toTimeInAdminTz = $this->parseDateByWebsiteTz((string)$rule->getToDate(), self::ADMIN_WEBSITE_ID);
 
         $excludedWebsites = [];
+
         $ruleExtensionAttributes = $rule->getExtensionAttributes();
         if ($ruleExtensionAttributes && $ruleExtensionAttributes->getExcludeWebsiteIds()) {
             $excludedWebsites = $ruleExtensionAttributes->getExcludeWebsiteIds();
+        } elseif ($rule->hasData('excluded_website_ids')) {
+            $excludedWebsites = $rule->getData('excluded_website_ids') ?: [];
         }
 
         return [

app/code/Magento/CatalogRule/Model/ResourceModel/Product/AttributeValuesLoader.php

@@ -232,4 +232,17 @@ private function evictOldBatchesIfNeeded(): void
             }
         }
     }
+
+    /**
+     * Reset all loaded data and clear cache
+     *
+     * @return void
+     */
+    public function resetCache(): void
+    {
+        $this->loadedData = [];
+        $this->loadedBatches = [];
+        $this->batchQueue = [];
+        $this->totalCount = null;
+    }
 }

app/code/Magento/CatalogRule/Model/ResourceModel/Product/Collection.php

@@ -131,4 +131,17 @@ public function getAllAttributeValues($attribute)
 
         return $this->loaderCache[$attributeId];
     }
+
+    /**
+     * Clear attribute loader cache
+     *
+     * @return void
+     */
+    public function __destruct()
+    {
+        foreach ($this->loaderCache as $loader) {
+            $loader->resetCache();
+        }
+        $this->loaderCache = [];
+    }
 }