Skip to content

Commit b231d91

Browse files
Add dynamic family filter option (#59)
* Add dynamic family filter option based on updated products to optimize large catalog imports * Refactor DynamicFamilyFilter to use SearchBuilder and improve null handling
1 parent c83961b commit b231d91

File tree

7 files changed

+192
-3
lines changed

7 files changed

+192
-3
lines changed

FEATURES.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ This document provides detailed information about all features available in the
1111
- [Important Attributes](#important-attributes)
1212
- [Default Store Values for Required Attributes](#default-store-values)
1313
- [Exclude Families from Import](#exclude-families)
14+
- [Dynamic Family Filtering](#dynamic-family-filtering)
1415
- [Remove Redundant EAV Attributes](#remove-redundant-eav)
1516
- [Category Features](#category-features)
1617
- [Category Exist - Skip URL Path Regeneration](#category-exist)
@@ -91,6 +92,24 @@ Prevents specific product families from being imported. Products belonging to ex
9192

9293
---
9394

95+
### Dynamic Family Filtering
96+
<a id="dynamic-family-filtering"></a>
97+
98+
Speeds up product imports for large catalogs by only processing families that have updated products within the configured "Updated Mode" period.
99+
100+
**How it works:**
101+
1. Before import starts, queries Akeneo API for products updated within the configured period
102+
2. Extracts unique family codes from those products
103+
3. Only imports families that have updates, skipping all others
104+
105+
**Configuration:** `Stores > Configuration > Catalog > Akeneo Connector > Products Filters > Dynamic Family Filtering`
106+
107+
**Example:** Catalog has 936 families. With "Updated Mode" set to "Since Last 2 Days" and only 5 products updated, only 5 families are processed instead of all 936.
108+
109+
**Important:** This feature uses the existing "Updated Mode" configuration from the Akeneo Connector. Make sure your update filter is properly configured.
110+
111+
---
112+
94113
### Remove Redundant EAV Attributes
95114

96115
<a id="remove-redundant-eav"></a>

Plugin/Job/Product.php

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
<?php
2+
23
declare(strict_types=1);
34

45
namespace JustBetter\AkeneoBundle\Plugin\Job;
56

67
use Akeneo\Connector\Job\Product as AkeneoProduct;
78
use Akeneo\Connector\Model\Source\Filters\Family;
9+
use JustBetter\AkeneoBundle\Service\DynamicFamilyFilter;
810
use Magento\Framework\App\Config\ScopeConfigInterface;
911

1012
class Product
@@ -13,7 +15,8 @@ class Product
1315

1416
public function __construct(
1517
protected ScopeConfigInterface $scopeConfig,
16-
protected Family $familyFilter
18+
protected Family $familyFilter,
19+
protected DynamicFamilyFilter $dynamicFamilyFilter
1720
) {
1821
}
1922

@@ -35,6 +38,13 @@ public function afterGetFamiliesToImport(AkeneoProduct $subject, ?array $familie
3538
$families = is_array($allFamilies) ? array_values($allFamilies) : [];
3639
}
3740

38-
return array_diff($families, $familiesToExclude);
41+
$families = array_diff($families, $familiesToExclude);
42+
43+
$dynamicFamilies = $this->dynamicFamilyFilter->getFamiliesWithUpdatedProducts();
44+
if ($dynamicFamilies !== null) {
45+
$families = array_intersect($families, $dynamicFamilies);
46+
}
47+
48+
return array_values($families);
3949
}
4050
}

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ For configuration instructions and best practices, see **[Configuration Guide](F
110110
| **<a href="FEATURES.md#product-import-features">Product Import</a>** | <a href="FEATURES.md#important-attributes">Important Attributes</a> | JustBetter Akeneo > Important Attributes |
111111
| | <a href="FEATURES.md#default-store-values">Default Store Values</a> | JustBetter Akeneo > Default Store Values |
112112
| | <a href="FEATURES.md#exclude-families">Exclude Families from Import</a> | Products Filters > Excluded Families |
113+
| | <a href="FEATURES.md#dynamic-family-filtering">Dynamic Family Filtering</a> | Products Filters > Dynamic Family Filtering |
113114
| | <a href="FEATURES.md#remove-redundant-eav">Remove Redundant EAV</a> | JustBetter Akeneo > Remove Redundant EAV |
114115
| **<a href="FEATURES.md#category-features">Category</a>** | <a href="FEATURES.md#category-exist">Category Exist - Skip URL Regeneration</a> | JustBetter Akeneo > Category Exist |
115116
| **<a href="FEATURES.md#tax--pricing-features">Tax & Pricing</a>** | <a href="FEATURES.md#set-tax-class">Set Tax Class</a> | JustBetter Akeneo > Set Tax Class |

Service/DynamicFamilyFilter.php

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace JustBetter\AkeneoBundle\Service;
6+
7+
use Akeneo\Connector\Helper\Authenticator;
8+
use Akeneo\Connector\Helper\Config as ConfigHelper;
9+
use Akeneo\Connector\Model\Source\Filters\Update;
10+
use Akeneo\Pim\ApiClient\Search\SearchBuilder;
11+
use Akeneo\Pim\ApiClient\Search\SearchBuilderFactory;
12+
use Magento\Framework\App\Config\ScopeConfigInterface;
13+
use Magento\Framework\App\ResourceConnection;
14+
use Magento\Framework\Stdlib\DateTime\TimezoneInterface;
15+
16+
class DynamicFamilyFilter
17+
{
18+
public const CONFIG_PATH_ENABLED = 'akeneo_connector/products_filters/dynamic_families_enabled';
19+
20+
public function __construct(
21+
protected ScopeConfigInterface $scopeConfig,
22+
protected Authenticator $authenticator,
23+
protected ConfigHelper $configHelper,
24+
protected ResourceConnection $resourceConnection,
25+
protected TimezoneInterface $timezone,
26+
protected SearchBuilderFactory $searchBuilderFactory
27+
) {
28+
}
29+
30+
public function isEnabled(): bool
31+
{
32+
return $this->scopeConfig->isSetFlag(self::CONFIG_PATH_ENABLED);
33+
}
34+
35+
/**
36+
* @return array<string>|null
37+
*/
38+
public function getFamiliesWithUpdatedProducts(): ?array
39+
{
40+
if (!$this->isEnabled()) {
41+
return null;
42+
}
43+
44+
$client = $this->authenticator->getAkeneoApiClient();
45+
if ($client === null) {
46+
return null;
47+
}
48+
49+
$searchBuilder = $this->buildUpdateFilter();
50+
if ($searchBuilder === null) {
51+
return null;
52+
}
53+
54+
$families = [];
55+
$filters = ['search' => $searchBuilder->getFilters()];
56+
$pageSize = $this->configHelper->getPaginationSize();
57+
58+
foreach ($client->getProductApi()->all($pageSize, $filters) as $product) {
59+
if (!empty($product['family'])) {
60+
$families[] = $product['family'];
61+
}
62+
}
63+
64+
foreach ($client->getProductModelApi()->all($pageSize, $filters) as $model) {
65+
if (!empty($model['family'])) {
66+
$families[] = $model['family'];
67+
}
68+
}
69+
70+
return array_values(array_unique($families));
71+
}
72+
73+
protected function buildUpdateFilter(): ?SearchBuilder
74+
{
75+
$searchBuilder = $this->searchBuilderFactory->create();
76+
77+
return match ($this->configHelper->getUpdatedMode()) {
78+
Update::GREATER_THAN => $this->greaterThan($searchBuilder, $this->configHelper->getUpdatedGreaterFilter()),
79+
Update::LOWER_THAN => $this->lowerThan($searchBuilder, $this->configHelper->getUpdatedLowerFilter()),
80+
Update::BETWEEN => $this->between(
81+
$searchBuilder,
82+
$this->configHelper->getUpdatedBetweenAfterFilter(),
83+
$this->configHelper->getUpdatedBetweenBeforeFilter()
84+
),
85+
Update::SINCE_LAST_N_DAYS => $this->sinceLastNDays($searchBuilder, $this->configHelper->getUpdatedSinceFilter()),
86+
Update::SINCE_LAST_N_HOURS => $this->sinceLastNHours($searchBuilder, $this->configHelper->getUpdatedSinceLastHoursFilter()),
87+
Update::SINCE_LAST_IMPORT => $this->sinceLastImport($searchBuilder),
88+
default => null,
89+
};
90+
}
91+
92+
protected function greaterThan(SearchBuilder $searchBuilder, ?string $date): ?SearchBuilder
93+
{
94+
return $date ? $searchBuilder->addFilter('updated', '>', "$date 00:00:00") : null;
95+
}
96+
97+
protected function lowerThan(SearchBuilder $searchBuilder, ?string $date): ?SearchBuilder
98+
{
99+
return $date ? $searchBuilder->addFilter('updated', '<', "$date 23:59:59") : null;
100+
}
101+
102+
protected function between(SearchBuilder $searchBuilder, ?string $after, ?string $before): ?SearchBuilder
103+
{
104+
return ($after && $before)
105+
? $searchBuilder->addFilter('updated', 'BETWEEN', ["$after 00:00:00", "$before 23:59:59"])
106+
: null;
107+
}
108+
109+
protected function sinceLastNDays(SearchBuilder $searchBuilder, ?string $days): ?SearchBuilder
110+
{
111+
if (!$days || !is_numeric($days)) {
112+
return null;
113+
}
114+
$date = $this->timezone->date()->modify("-$days days");
115+
116+
return $searchBuilder->addFilter('updated', '>', $date->format('Y-m-d H:i:s'));
117+
}
118+
119+
protected function sinceLastNHours(SearchBuilder $searchBuilder, ?string $hours): ?SearchBuilder
120+
{
121+
if (!$hours || !is_numeric($hours)) {
122+
return null;
123+
}
124+
$date = $this->timezone->date()->modify("-$hours hours");
125+
126+
return $searchBuilder->addFilter('updated', '>', $date->format('Y-m-d H:i:s'));
127+
}
128+
129+
protected function sinceLastImport(SearchBuilder $searchBuilder): ?SearchBuilder
130+
{
131+
$connection = $this->resourceConnection->getConnection();
132+
$date = $connection->fetchOne(
133+
$connection->select()
134+
->from($this->resourceConnection->getTableName('akeneo_connector_job'), ['last_success_date'])
135+
->where('code = ?', 'product')
136+
);
137+
138+
return $date ? $searchBuilder->addFilter('updated', '>', $date) : null;
139+
}
140+
}

etc/adminhtml/system.xml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,19 @@
164164
</depends>
165165
<can_be_empty>1</can_be_empty>
166166
</field>
167+
<field id="dynamic_families_enabled" translate="label" type="select" sortOrder="150" showInDefault="1" showInWebsite="0" showInStore="0">
168+
<label>Dynamic Family Filtering</label>
169+
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
170+
<comment>
171+
<![CDATA[
172+
<b>Speeds up product imports for large catalogs.</b>
173+
<br/><br/>
174+
First fetches updated products based on "Updated Mode"; then imports only the families found in those results.
175+
<br/><br/>
176+
Example: 936 families total, but only 5 with updates ⇒ only those 5 are processed.
177+
]]>
178+
</comment>
179+
</field>
167180
</group>
168181
</section>
169182
</system>

etc/config.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
<api>https://slack.com/api/chat.postMessage</api>
1818
</slack>
1919
</justbetter>
20+
<products_filters>
21+
<dynamic_families_enabled>0</dynamic_families_enabled>
22+
</products_filters>
2023
</akeneo_connector>
2124
</default>
2225
</config>

phpstan.neon

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,7 @@ parameters:
55
- .
66
excludePaths:
77
- vendor
8-
level: 7
8+
level: 7
9+
ignoreErrors:
10+
- message: '#\w+Factory#'
11+
reportUnmatched: false

0 commit comments

Comments
 (0)