diff --git a/cla-backend-go/cmd/signatures_timestamp_backfill/main.go b/cla-backend-go/cmd/signatures_timestamp_backfill/main.go index b0966b4f8..d44b911ee 100644 --- a/cla-backend-go/cmd/signatures_timestamp_backfill/main.go +++ b/cla-backend-go/cmd/signatures_timestamp_backfill/main.go @@ -408,10 +408,7 @@ func firstPassScanAndUpdate( updateExpr := setPrefix vals := map[string]*dynamodb.AttributeValue{":empty": {S: aws.String("")}} - names := map[string]*string{ - "#date_created": aws.String(attrDateCreated), - "#date_modified": aws.String(attrDateModified), - } + names := map[string]*string{} first := true if setCreated { if !first { @@ -419,6 +416,7 @@ func firstPassScanAndUpdate( } updateExpr += exprSetDateCreated vals[":date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)} + names["#date_created"] = aws.String(attrDateCreated) first = false } if setModified { @@ -427,6 +425,7 @@ func firstPassScanAndUpdate( } updateExpr += exprSetDateModified vals[":date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)} + names["#date_modified"] = aws.String(attrDateModified) } if debug { @@ -450,7 +449,8 @@ func firstPassScanAndUpdate( } // Build CLI (always emitted in dry-run; emitted on failure in live-run) - cmd := buildAwsCliUpdate(region, stage, tableName, sig.SignatureID, updateExpr, names, vals, condAnyMissing) + condExpr := buildConditionExpression(names) + cmd := buildAwsCliUpdate(region, stage, tableName, sig.SignatureID, updateExpr, names, vals, condExpr) dbg(" CLI: %s", cmd) if dryRun { @@ -468,7 +468,7 @@ func firstPassScanAndUpdate( UpdateExpression: aws.String(updateExpr), ExpressionAttributeNames: names, ExpressionAttributeValues: vals, - ConditionExpression: aws.String(condAnyMissing), + ConditionExpression: aws.String(condExpr), }) if uerr != nil { log.Printf("Update failed %s: %v", sig.SignatureID, uerr) @@ -604,12 +604,7 @@ func snowflakeFix( updateExpr := setPrefix vals := map[string]*dynamodb.AttributeValue{":empty": {S: aws.String("")}} - names := map[string]*string{ - "#date_created": aws.String(attrDateCreated), - "#date_modified": aws.String(attrDateModified), - "#approx_date_created": aws.String(attrApproxDateCreated), - "#approx_date_modified": aws.String(attrApproxDateModified), - } + names := map[string]*string{} first := true if setCreated { if !first { @@ -619,9 +614,11 @@ func snowflakeFix( if srcC == labelFivetranSynced { updateExpr += exprSetApproxDateCreated vals[":approx_date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)} + names["#approx_date_created"] = aws.String(attrApproxDateCreated) } else { updateExpr += exprSetDateCreated vals[":date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)} + names["#date_created"] = aws.String(attrDateCreated) } first = false } @@ -633,9 +630,11 @@ func snowflakeFix( if srcM == labelFivetranSynced { updateExpr += exprSetApproxDateModified vals[":approx_date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)} + names["#approx_date_modified"] = aws.String(attrApproxDateModified) } else { updateExpr += exprSetDateModified vals[":date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)} + names["#date_modified"] = aws.String(attrDateModified) } } @@ -659,7 +658,9 @@ func snowflakeFix( } } - cmd := buildAwsCliUpdate(region, stage, tableName, id, updateExpr, names, vals, condAnyMissing) + // Build condition expression with names that are actually defined + condExpr := buildConditionExpression(names) + cmd := buildAwsCliUpdate(region, stage, tableName, id, updateExpr, names, vals, condExpr) dbg(" SF CLI: %s", cmd) if dryRun { @@ -672,13 +673,16 @@ func snowflakeFix( continue } + // Build condition expression with names that are actually defined + condExpr = buildConditionExpression(names) + _, uerr := ddb.UpdateItem(&dynamodb.UpdateItemInput{ TableName: aws.String(tableName), Key: map[string]*dynamodb.AttributeValue{"signature_id": {S: aws.String(id)}}, UpdateExpression: aws.String(updateExpr), ExpressionAttributeNames: names, ExpressionAttributeValues: vals, - ConditionExpression: aws.String(condAnyMissing), + ConditionExpression: aws.String(condExpr), }) if uerr != nil { log.Printf("Update failed (SF) %s: %v", id, uerr) @@ -1058,6 +1062,39 @@ func parseSnowflakeCSV(b []byte) map[string]string { return res } +// buildConditionExpression builds a condition expression using only the attribute names +// that are actually defined in the names map to avoid DynamoDB validation errors +func buildConditionExpression(names map[string]*string) string { + var conditions []string + + // Check for regular date_created field + if _, hasDateCreated := names["#date_created"]; hasDateCreated { + conditions = append(conditions, "attribute_not_exists(#date_created) OR #date_created = :empty") + } + + // Check for regular date_modified field + if _, hasDateModified := names["#date_modified"]; hasDateModified { + conditions = append(conditions, "attribute_not_exists(#date_modified) OR #date_modified = :empty") + } + + // Check for approx_date_created field + if _, hasApproxDateCreated := names["#approx_date_created"]; hasApproxDateCreated { + conditions = append(conditions, "attribute_not_exists(#approx_date_created) OR #approx_date_created = :empty") + } + + // Check for approx_date_modified field + if _, hasApproxDateModified := names["#approx_date_modified"]; hasApproxDateModified { + conditions = append(conditions, "attribute_not_exists(#approx_date_modified) OR #approx_date_modified = :empty") + } + + // If no specific conditions, use a basic condition that should always allow updates + if len(conditions) == 0 { + return "attribute_exists(signature_id)" + } + + return strings.Join(conditions, " OR ") +} + // ----------------------------------------------------------------------------- // AWS CLI builder & stats print // ----------------------------------------------------------------------------- diff --git a/utils/signature_dates_stats.sh b/utils/signature_dates_stats.sh new file mode 100644 index 000000000..da25cce1b --- /dev/null +++ b/utils/signature_dates_stats.sh @@ -0,0 +1,5 @@ +#!/bin/bash +aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_not_exists(#a) OR #a = :nullval" --expression-attribute-names '{"#a":"date_created"}' --expression-attribute-values '{":nullval":{"NULL":true}}' --select "COUNT" +aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_exists(#a)" --expression-attribute-names '{"#a":"approx_date_created"}' --select "COUNT" +aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_not_exists(#a) OR #a = :nullval" --expression-attribute-names '{"#a":"date_modified"}' --expression-attribute-values '{":nullval":{"NULL":true}}' --select "COUNT" +aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_exists(#a)" --expression-attribute-names '{"#a":"approx_date_modified"}' --select "COUNT"