Skip to content

Commit e1441f3

Browse files
Add dynamic family filter option based on updated products to optimize large catalog imports
1 parent c83961b commit e1441f3

File tree

6 files changed

+203
-2
lines changed

6 files changed

+203
-2
lines changed

FEATURES.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ This document provides detailed information about all features available in the
1111
- [Important Attributes](#important-attributes)
1212
- [Default Store Values for Required Attributes](#default-store-values)
1313
- [Exclude Families from Import](#exclude-families)
14+
- [Dynamic Family Filtering](#dynamic-family-filtering)
1415
- [Remove Redundant EAV Attributes](#remove-redundant-eav)
1516
- [Category Features](#category-features)
1617
- [Category Exist - Skip URL Path Regeneration](#category-exist)
@@ -91,6 +92,24 @@ Prevents specific product families from being imported. Products belonging to ex
9192

9293
---
9394

95+
### Dynamic Family Filtering
96+
<a id="dynamic-family-filtering"></a>
97+
98+
Speeds up product imports for large catalogs by only processing families that have updated products within the configured "Updated Mode" period.
99+
100+
**How it works:**
101+
1. Before import starts, queries Akeneo API for products updated within the configured period
102+
2. Extracts unique family codes from those products
103+
3. Only imports families that have updates, skipping all others
104+
105+
**Configuration:** `Stores > Configuration > Catalog > Akeneo Connector > Products Filters > Dynamic Family Filtering`
106+
107+
**Example:** Catalog has 936 families. With "Updated Mode" set to "Since Last 2 Days" and only 5 products updated, only 5 families are processed instead of all 936.
108+
109+
**Important:** This feature uses the existing "Updated Mode" configuration from the Akeneo Connector. Make sure your update filter is properly configured.
110+
111+
---
112+
94113
### Remove Redundant EAV Attributes
95114

96115
<a id="remove-redundant-eav"></a>

Plugin/Job/Product.php

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
<?php
2+
23
declare(strict_types=1);
34

45
namespace JustBetter\AkeneoBundle\Plugin\Job;
56

67
use Akeneo\Connector\Job\Product as AkeneoProduct;
78
use Akeneo\Connector\Model\Source\Filters\Family;
9+
use JustBetter\AkeneoBundle\Service\DynamicFamilyFilter;
810
use Magento\Framework\App\Config\ScopeConfigInterface;
911

1012
class Product
@@ -13,7 +15,8 @@ class Product
1315

1416
public function __construct(
1517
protected ScopeConfigInterface $scopeConfig,
16-
protected Family $familyFilter
18+
protected Family $familyFilter,
19+
protected DynamicFamilyFilter $dynamicFamilyFilter
1720
) {
1821
}
1922

@@ -35,6 +38,13 @@ public function afterGetFamiliesToImport(AkeneoProduct $subject, ?array $familie
3538
$families = is_array($allFamilies) ? array_values($allFamilies) : [];
3639
}
3740

38-
return array_diff($families, $familiesToExclude);
41+
$families = array_diff($families, $familiesToExclude);
42+
43+
$dynamicFamilies = $this->dynamicFamilyFilter->getFamiliesWithUpdatedProducts();
44+
if ($dynamicFamilies !== null) {
45+
$families = array_intersect($families, $dynamicFamilies);
46+
}
47+
48+
return array_values($families);
3949
}
4050
}

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ For configuration instructions and best practices, see **[Configuration Guide](F
110110
| **<a href="FEATURES.md#product-import-features">Product Import</a>** | <a href="FEATURES.md#important-attributes">Important Attributes</a> | JustBetter Akeneo > Important Attributes |
111111
| | <a href="FEATURES.md#default-store-values">Default Store Values</a> | JustBetter Akeneo > Default Store Values |
112112
| | <a href="FEATURES.md#exclude-families">Exclude Families from Import</a> | Products Filters > Excluded Families |
113+
| | <a href="FEATURES.md#dynamic-family-filtering">Dynamic Family Filtering</a> | Products Filters > Dynamic Family Filtering |
113114
| | <a href="FEATURES.md#remove-redundant-eav">Remove Redundant EAV</a> | JustBetter Akeneo > Remove Redundant EAV |
114115
| **<a href="FEATURES.md#category-features">Category</a>** | <a href="FEATURES.md#category-exist">Category Exist - Skip URL Regeneration</a> | JustBetter Akeneo > Category Exist |
115116
| **<a href="FEATURES.md#tax--pricing-features">Tax & Pricing</a>** | <a href="FEATURES.md#set-tax-class">Set Tax Class</a> | JustBetter Akeneo > Set Tax Class |

Service/DynamicFamilyFilter.php

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace JustBetter\AkeneoBundle\Service;
6+
7+
use Akeneo\Connector\Helper\Authenticator;
8+
use Akeneo\Connector\Helper\Config as ConfigHelper;
9+
use Akeneo\Connector\Model\Source\Filters\Update;
10+
use Magento\Framework\App\Config\ScopeConfigInterface;
11+
use Magento\Framework\App\ResourceConnection;
12+
use Magento\Framework\Stdlib\DateTime\TimezoneInterface;
13+
14+
class DynamicFamilyFilter
15+
{
16+
public const CONFIG_PATH_ENABLED = 'akeneo_connector/products_filters/dynamic_families_enabled';
17+
18+
public function __construct(
19+
protected ScopeConfigInterface $scopeConfig,
20+
protected Authenticator $authenticator,
21+
protected ConfigHelper $configHelper,
22+
protected ResourceConnection $resourceConnection,
23+
protected TimezoneInterface $timezone
24+
) {
25+
}
26+
27+
public function isEnabled(): bool
28+
{
29+
return $this->scopeConfig->isSetFlag(self::CONFIG_PATH_ENABLED);
30+
}
31+
32+
/**
33+
* @return array<string>|null
34+
*/
35+
public function getFamiliesWithUpdatedProducts(): ?array
36+
{
37+
if (!$this->isEnabled()) {
38+
return null;
39+
}
40+
41+
$client = $this->authenticator->getAkeneoApiClient();
42+
if ($client === null) {
43+
return null;
44+
}
45+
46+
$filter = $this->buildUpdateFilter();
47+
if (empty($filter)) {
48+
return null;
49+
}
50+
51+
$families = [];
52+
$searchFilters = ['updated' => [$filter]];
53+
$pageSize = $this->configHelper->getPaginationSize();
54+
55+
foreach ($client->getProductApi()->all($pageSize, ['search' => $searchFilters]) as $product) {
56+
if (!empty($product['family'])) {
57+
$families[] = $product['family'];
58+
}
59+
}
60+
61+
foreach ($client->getProductModelApi()->all($pageSize, ['search' => $searchFilters]) as $model) {
62+
if (!empty($model['family'])) {
63+
$families[] = $model['family'];
64+
}
65+
}
66+
67+
return array_values(array_unique($families));
68+
}
69+
70+
/**
71+
* @return array<string, mixed>
72+
*/
73+
protected function buildUpdateFilter(): array
74+
{
75+
return match ($this->configHelper->getUpdatedMode()) {
76+
Update::GREATER_THAN => $this->greaterThan($this->configHelper->getUpdatedGreaterFilter()),
77+
Update::LOWER_THAN => $this->lowerThan($this->configHelper->getUpdatedLowerFilter()),
78+
Update::BETWEEN => $this->between(
79+
$this->configHelper->getUpdatedBetweenAfterFilter(),
80+
$this->configHelper->getUpdatedBetweenBeforeFilter()
81+
),
82+
Update::SINCE_LAST_N_DAYS => $this->sinceLastNDays($this->configHelper->getUpdatedSinceFilter()),
83+
Update::SINCE_LAST_N_HOURS => $this->sinceLastNHours($this->configHelper->getUpdatedSinceLastHoursFilter()),
84+
Update::SINCE_LAST_IMPORT => $this->sinceLastImport(),
85+
default => [],
86+
};
87+
}
88+
89+
/**
90+
* @return array<string, mixed>
91+
*/
92+
protected function greaterThan(?string $date): array
93+
{
94+
return $date ? ['operator' => '>', 'value' => "$date 00:00:00"] : [];
95+
}
96+
97+
/**
98+
* @return array<string, mixed>
99+
*/
100+
protected function lowerThan(?string $date): array
101+
{
102+
return $date ? ['operator' => '<', 'value' => "$date 23:59:59"] : [];
103+
}
104+
105+
/**
106+
* @return array<string, mixed>
107+
*/
108+
protected function between(?string $after, ?string $before): array
109+
{
110+
return ($after && $before)
111+
? ['operator' => 'BETWEEN', 'value' => ["$after 00:00:00", "$before 23:59:59"]]
112+
: [];
113+
}
114+
115+
/**
116+
* @return array<string, mixed>
117+
*/
118+
protected function sinceLastNDays(?string $days): array
119+
{
120+
if (!$days || !is_numeric($days)) {
121+
return [];
122+
}
123+
$date = $this->timezone->date()->modify("-$days days");
124+
125+
return ['operator' => '>', 'value' => $date->format('Y-m-d H:i:s')];
126+
}
127+
128+
/**
129+
* @return array<string, mixed>
130+
*/
131+
protected function sinceLastNHours(?string $hours): array
132+
{
133+
if (!$hours || !is_numeric($hours)) {
134+
return [];
135+
}
136+
$date = $this->timezone->date()->modify("-$hours hours");
137+
138+
return ['operator' => '>', 'value' => $date->format('Y-m-d H:i:s')];
139+
}
140+
141+
/**
142+
* @return array<string, mixed>
143+
*/
144+
protected function sinceLastImport(): array
145+
{
146+
$connection = $this->resourceConnection->getConnection();
147+
$date = $connection->fetchOne(
148+
$connection->select()
149+
->from($this->resourceConnection->getTableName('akeneo_connector_job'), ['last_success_date'])
150+
->where('code = ?', 'product')
151+
);
152+
153+
return $date ? ['operator' => '>', 'value' => $date] : [];
154+
}
155+
}

etc/adminhtml/system.xml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,19 @@
164164
</depends>
165165
<can_be_empty>1</can_be_empty>
166166
</field>
167+
<field id="dynamic_families_enabled" translate="label" type="select" sortOrder="150" showInDefault="1" showInWebsite="0" showInStore="0">
168+
<label>Dynamic Family Filtering</label>
169+
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
170+
<comment>
171+
<![CDATA[
172+
<b>Speeds up product imports for large catalogs.</b>
173+
<br/><br/>
174+
First fetches updated products based on "Updated Mode"; then imports only the families found in those results.
175+
<br/><br/>
176+
Example: 936 families total, but only 5 with updates ⇒ only those 5 are processed.
177+
]]>
178+
</comment>
179+
</field>
167180
</group>
168181
</section>
169182
</system>

etc/config.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
<api>https://slack.com/api/chat.postMessage</api>
1818
</slack>
1919
</justbetter>
20+
<products_filters>
21+
<dynamic_families_enabled>0</dynamic_families_enabled>
22+
</products_filters>
2023
</akeneo_connector>
2124
</default>
2225
</config>

0 commit comments

Comments
 (0)