Skip to content

Commit 5e25b78

Browse files
marcelogpinheiromarcelogpinheiro
committed
feat: logs
1 parent 0151f2a commit 5e25b78

File tree

5 files changed

+117
-10
lines changed

5 files changed

+117
-10
lines changed

‎src/main/java/com/commercetools/sync/products/ProductSync.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@
6262
import java.util.stream.Collectors;
6363
import javax.annotation.Nonnull;
6464
import javax.annotation.Nullable;
65+
import org.slf4j.Logger;
66+
import org.slf4j.LoggerFactory;
6567
import org.apache.commons.lang3.tuple.ImmutablePair;
6668

6769
public class ProductSync
@@ -90,6 +92,7 @@ public class ProductSync
9092
private final ProductBatchValidator batchValidator;
9193

9294
private ConcurrentHashMap.KeySetView<String, Boolean> readyToResolve;
95+
private static final Logger LOGGER = LoggerFactory.getLogger(ProductSync.class);
9396

9497
/**
9598
* Takes a {@link ProductSyncOptions} instance to instantiate a new {@link ProductSync} instance
@@ -177,6 +180,12 @@ protected CompletionStage<ProductSyncStatistics> processBatch(
177180
return CompletableFuture.completedFuture(statistics);
178181
}
179182

183+
LOGGER.info(
184+
"Processing batch: size={}, distinctDraftKeys={}, referencedProductKeys={}",
185+
batch.size(),
186+
validDrafts.stream().map(ProductDraft::getKey).filter(Objects::nonNull).count(),
187+
result.getRight().getProductKeys().size());
188+
180189
return productReferenceResolver
181190
.populateKeyToIdCachesForReferencedKeys(result.getRight())
182191
.handle(ImmutablePair::new)
@@ -195,6 +204,11 @@ protected CompletionStage<ProductSyncStatistics> processBatch(
195204
}
196205

197206
final Map<String, String> productKeyToIdCache = cachingResponse.getKey();
207+
LOGGER.info(
208+
"Cache primed: productKeysCached={}, productTypeKeysCached={}, categoryKeysCached={}",
209+
result.getRight().getProductKeys().size(),
210+
result.getRight().getProductTypeKeys().size(),
211+
result.getRight().getCategoryKeys().size());
198212
return syncBatch(validDrafts, productKeyToIdCache);
199213
})
200214
.thenApply(
@@ -295,6 +309,11 @@ private CompletionStage<Optional<WaitingToBeResolvedProducts>> keepTrackOfMissin
295309

296310
missingReferencedProductKeys.forEach(
297311
missingParentKey -> statistics.addMissingDependency(missingParentKey, newProduct.getKey()));
312+
LOGGER.warn(
313+
"Missing referenced product keys for draft. draftKey={}, missingKeysCount={}, sampleKeys={}",
314+
newProduct.getKey(),
315+
missingReferencedProductKeys.size(),
316+
missingReferencedProductKeys.stream().limit(5).collect(Collectors.toSet()));
298317
return unresolvedReferencesService.save(
299318
new WaitingToBeResolvedProducts(newProduct, missingReferencedProductKeys),
300319
UnresolvedReferencesServiceImpl.CUSTOM_OBJECT_PRODUCT_CONTAINER_KEY,
@@ -399,10 +418,12 @@ private CompletionStage<Void> syncDraft(
399418
final Map<String, ProductProjection> oldProductMap =
400419
oldProducts.stream().collect(toMap(ProductProjection::getKey, identity()));
401420

421+
LOGGER.debug("Resolving references for draftKey={}", newProductDraft.getKey());
402422
return productReferenceResolver
403423
.resolveReferences(newProductDraft)
404424
.thenCompose(
405425
resolvedDraft -> {
426+
LOGGER.debug("Resolved references for draftKey={}", newProductDraft.getKey());
406427
final ProductProjection oldProduct = oldProductMap.get(newProductDraft.getKey());
407428

408429
return ofNullable(oldProduct)

‎src/main/java/com/commercetools/sync/products/helpers/ProductReferenceResolver.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,12 @@
5151
import java.util.concurrent.CompletableFuture;
5252
import java.util.concurrent.CompletionStage;
5353
import javax.annotation.Nonnull;
54+
import org.slf4j.Logger;
55+
import org.slf4j.LoggerFactory;
5456

5557
public final class ProductReferenceResolver
5658
extends BaseReferenceResolver<ProductDraft, ProductSyncOptions> {
59+
private static final Logger LOGGER = LoggerFactory.getLogger(ProductReferenceResolver.class);
5760
private final ProductTypeService productTypeService;
5861
private final CategoryService categoryService;
5962
private final VariantReferenceResolver variantReferenceResolver;
@@ -547,6 +550,19 @@ public CompletableFuture<Map<String, String>> populateKeyToIdCachesForReferenced
547550

548551
return collectionOfFuturesToFutureOfCollection(futures, toList())
549552
.thenCompose(
550-
ignored -> productService.cacheKeysToIds(Collections.emptySet())); // return all cache.
553+
ignored -> {
554+
LOGGER.info(
555+
"populateKeyToIdCaches: products={}, productTypes={}, categories={}, taxCategories={}, types={}, channels={}, states={}, customerGroups={}, customObjects={}",
556+
referencedKeys.getProductKeys().size(),
557+
referencedKeys.getProductTypeKeys().size(),
558+
referencedKeys.getCategoryKeys().size(),
559+
referencedKeys.getTaxCategoryKeys().size(),
560+
referencedKeys.getTypeKeys().size(),
561+
referencedKeys.getChannelKeys().size(),
562+
referencedKeys.getStateKeys().size(),
563+
referencedKeys.getCustomerGroupKeys().size(),
564+
referencedKeys.getCustomObjectCompositeIdentifiers().size());
565+
return productService.cacheKeysToIds(Collections.emptySet());
566+
}); // return all cache.
551567
}
552568
}

‎src/main/java/com/commercetools/sync/products/helpers/VariantReferenceResolver.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,12 @@
4040
import java.util.concurrent.CompletionStage;
4141
import java.util.function.Function;
4242
import javax.annotation.Nonnull;
43+
import org.slf4j.Logger;
44+
import org.slf4j.LoggerFactory;
4345

4446
public final class VariantReferenceResolver
4547
extends BaseReferenceResolver<ProductVariantDraft, ProductSyncOptions> {
48+
private static final Logger LOGGER = LoggerFactory.getLogger(VariantReferenceResolver.class);
4649
private final PriceReferenceResolver priceReferenceResolver;
4750
private final AssetReferenceResolver assetReferenceResolver;
4851
private final ProductService productService;
@@ -174,7 +177,9 @@ private CompletionStage<Attribute> resolveAttributeReference(
174177

175178
if (!allAttributeReferences.isEmpty()) {
176179
return CompletableFutureUtils.mapValuesToFutureOfCompletedValues(
177-
allAttributeReferences, this::resolveReference, toList())
180+
allAttributeReferences,
181+
referenceNode -> resolveReference(referenceNode, attributeDraft.getName()),
182+
toList())
178183
.thenApply(
179184
ignoredResult -> {
180185
if (cleanupAttributeValue(attributeDraftValueAsJson, attributeDraft).isEmpty()) {
@@ -192,12 +197,33 @@ private CompletionStage<Attribute> resolveAttributeReference(
192197
}
193198

194199
@Nonnull
195-
private CompletionStage<Void> resolveReference(@Nonnull final JsonNode referenceValue) {
200+
private CompletionStage<Void> resolveReference(
201+
@Nonnull final JsonNode referenceValue, @Nonnull final String attributeName) {
202+
final String typeId =
203+
referenceValue.has(REFERENCE_TYPE_ID_FIELD) ? referenceValue.get(REFERENCE_TYPE_ID_FIELD).asText() : "";
204+
205+
final JsonNode idFieldNode = referenceValue.get(REFERENCE_ID_FIELD);
206+
if (idFieldNode == null || Objects.equals(idFieldNode, NullNode.getInstance())) {
207+
LOGGER.warn(
208+
"Attribute reference has empty id field. attribute='{}', typeId='{}', value={}",
209+
attributeName,
210+
typeId,
211+
referenceValue.toString());
212+
}
213+
196214
return getResolvedId(referenceValue)
197215
.thenAccept(
198-
optionalId ->
199-
optionalId.ifPresent(
200-
id -> ((ObjectNode) referenceValue).put(REFERENCE_ID_FIELD, id)));
216+
optionalId -> {
217+
if (optionalId.isPresent()) {
218+
((ObjectNode) referenceValue).put(REFERENCE_ID_FIELD, optionalId.get());
219+
} else {
220+
LOGGER.warn(
221+
"Failed to resolve reference id. attribute='{}', typeId='{}', refPayload={}",
222+
attributeName,
223+
typeId,
224+
referenceValue.toString());
225+
}
226+
});
201227
}
202228

203229
@Nonnull

‎src/main/java/com/commercetools/sync/products/utils/ProductTransformUtils.java‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,13 @@
3131
import java.util.stream.Collectors;
3232
import javax.annotation.Nonnull;
3333
import javax.annotation.Nullable;
34+
import org.slf4j.Logger;
35+
import org.slf4j.LoggerFactory;
3436

3537
public final class ProductTransformUtils {
3638

39+
private static final Logger LOGGER = LoggerFactory.getLogger(ProductTransformUtils.class);
40+
3741
/**
3842
* Transforms products by resolving the references and map them to ProductDrafts.
3943
*
@@ -113,9 +117,14 @@ private CompletionStage<List<ProductDraft>> transformReferencesAndMapToProductDr
113117
return CompletableFuture.allOf(
114118
transformReferencesToRunParallel.stream().toArray(CompletableFuture[]::new))
115119
.thenApply(
116-
ignore ->
117-
ProductReferenceResolutionUtils.mapToProductDrafts(
118-
products, this.referenceIdToKeyCache));
120+
ignore -> {
121+
LOGGER.info(
122+
"Transform references complete: products={}, cacheSize(id->key)~{}",
123+
products.size(),
124+
this.referenceIdToKeyCache.size());
125+
return ProductReferenceResolutionUtils.mapToProductDrafts(
126+
products, this.referenceIdToKeyCache);
127+
});
119128
}
120129

121130
@Nonnull
@@ -335,8 +344,19 @@ private List<JsonNode> getAttributeReferences(@Nonnull final List<ProductVariant
335344
private void replaceReferences(@Nonnull final List<JsonNode> allAttributeReferences) {
336345
allAttributeReferences.forEach(
337346
reference -> {
338-
final String id = reference.get(REFERENCE_ID_FIELD).asText();
347+
final String typeId =
348+
reference.has(REFERENCE_TYPE_ID_FIELD)
349+
? reference.get(REFERENCE_TYPE_ID_FIELD).asText()
350+
: "";
351+
final JsonNode idNode = reference.get(REFERENCE_ID_FIELD);
352+
final String id = idNode != null ? idNode.asText(null) : null;
339353
final String key = referenceIdToKeyCache.get(id);
354+
if (key == null) {
355+
LOGGER.warn(
356+
"Attribute reference id->key not found in source cache. typeId='{}', id='{}' -> key=null",
357+
typeId,
358+
id);
359+
}
340360
((ObjectNode) reference).put(REFERENCE_ID_FIELD, key);
341361
});
342362
}
@@ -363,6 +383,16 @@ CompletableFuture<Void> getIdToKeys(@Nonnull final List<JsonNode> allAttributeRe
363383
final Set<JsonNode> nonCachedReferences = getNonCachedReferences(allAttributeReferences);
364384
final Map<GraphQlQueryResource, Set<String>> map =
365385
buildMapOfRequestTypeToReferencedIds(nonCachedReferences);
386+
LOGGER.info(
387+
"Collect non-cached ids by resource: PRODUCTS={}, CATEGORIES={}, PRODUCT_TYPES={}, STATES={}, CHANNELS={}, CUSTOMER_GROUPS={}, TYPES={}, CUSTOM_OBJECTS={}",
388+
map.getOrDefault(GraphQlQueryResource.PRODUCTS, Collections.emptySet()).size(),
389+
map.getOrDefault(GraphQlQueryResource.CATEGORIES, Collections.emptySet()).size(),
390+
map.getOrDefault(GraphQlQueryResource.PRODUCT_TYPES, Collections.emptySet()).size(),
391+
map.getOrDefault(GraphQlQueryResource.STATES, Collections.emptySet()).size(),
392+
map.getOrDefault(GraphQlQueryResource.CHANNELS, Collections.emptySet()).size(),
393+
map.getOrDefault(GraphQlQueryResource.CUSTOMER_GROUPS, Collections.emptySet()).size(),
394+
map.getOrDefault(GraphQlQueryResource.TYPES, Collections.emptySet()).size(),
395+
map.getOrDefault(GraphQlQueryResource.CUSTOM_OBJECTS, Collections.emptySet()).size());
366396

367397
final Set<String> nonCachedCustomObjectIds = map.remove(GraphQlQueryResource.CUSTOM_OBJECTS);
368398

@@ -376,6 +406,11 @@ CompletableFuture<Void> getIdToKeys(@Nonnull final List<JsonNode> allAttributeRe
376406
.map(
377407
resource -> {
378408
List<List<String>> chunk = ChunkUtils.chunk(map.get(resource), CHUNK_SIZE);
409+
LOGGER.info(
410+
"Preparing GraphQL fetch for {} ids: chunks={} chunkSize={}",
411+
resource,
412+
chunk.size(),
413+
CHUNK_SIZE);
379414
return createGraphQLRequests(chunk, resource);
380415
})
381416
.flatMap(Collection::stream)

‎src/main/resources/it.properties‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# rename to ctp.credentials.properties and fill the values to run the job locally without env variables
2+
# see CtpClientUtils for more info
3+
source.projectKey=java-sync-source-marcelo
4+
source.clientId=8Cqap5NACcvYybY4QkmWGype
5+
source.clientSecret=fVutrn2qwFmvwaUz0qfk1NazGQZX_Ep_
6+
7+
target.projectKey=java-sync-target-marcelo
8+
target.clientId=FjqdGB40QJLl8SfP8pk9dZ_O
9+
target.clientSecret=rKB7RYZe7naBXIbuLnZbL8G58KUWDuse

0 commit comments

Comments
 (0)