Merge pull request #4875 from linuxfoundation/unicron-fix-backfill-script

Unicron fix backfill script

Author: lukaszgryglicki

cla-backend-go/cmd/signatures_timestamp_backfill/main.go

@@ -408,17 +408,15 @@ func firstPassScanAndUpdate(
 
 			updateExpr := setPrefix
 			vals := map[string]*dynamodb.AttributeValue{":empty": {S: aws.String("")}}
-			names := map[string]*string{
-				"#date_created":  aws.String(attrDateCreated),
-				"#date_modified": aws.String(attrDateModified),
-			}
+			names := map[string]*string{}
 			first := true
 			if setCreated {
 				if !first {
 					updateExpr += commaSep
 				}
 				updateExpr += exprSetDateCreated
 				vals[":date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)}
+				names["#date_created"] = aws.String(attrDateCreated)
 				first = false
 			}
 			if setModified {
@@ -427,6 +425,7 @@ func firstPassScanAndUpdate(
 				}
 				updateExpr += exprSetDateModified
 				vals[":date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)}
+				names["#date_modified"] = aws.String(attrDateModified)
 			}
 
 			if debug {
@@ -450,7 +449,8 @@ func firstPassScanAndUpdate(
 			}
 
 			// Build CLI (always emitted in dry-run; emitted on failure in live-run)
-			cmd := buildAwsCliUpdate(region, stage, tableName, sig.SignatureID, updateExpr, names, vals, condAnyMissing)
+			condExpr := buildConditionExpression(names)
+			cmd := buildAwsCliUpdate(region, stage, tableName, sig.SignatureID, updateExpr, names, vals, condExpr)
 			dbg("  CLI: %s", cmd)
 
 			if dryRun {
@@ -468,7 +468,7 @@ func firstPassScanAndUpdate(
 				UpdateExpression:          aws.String(updateExpr),
 				ExpressionAttributeNames:  names,
 				ExpressionAttributeValues: vals,
-				ConditionExpression:       aws.String(condAnyMissing),
+				ConditionExpression:       aws.String(condExpr),
 			})
 			if uerr != nil {
 				log.Printf("Update failed %s: %v", sig.SignatureID, uerr)
@@ -604,12 +604,7 @@ func snowflakeFix(
 
 			updateExpr := setPrefix
 			vals := map[string]*dynamodb.AttributeValue{":empty": {S: aws.String("")}}
-			names := map[string]*string{
-				"#date_created":         aws.String(attrDateCreated),
-				"#date_modified":        aws.String(attrDateModified),
-				"#approx_date_created":  aws.String(attrApproxDateCreated),
-				"#approx_date_modified": aws.String(attrApproxDateModified),
-			}
+			names := map[string]*string{}
 			first := true
 			if setCreated {
 				if !first {
@@ -619,9 +614,11 @@ func snowflakeFix(
 				if srcC == labelFivetranSynced {
 					updateExpr += exprSetApproxDateCreated
 					vals[":approx_date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)}
+					names["#approx_date_created"] = aws.String(attrApproxDateCreated)
 				} else {
 					updateExpr += exprSetDateCreated
 					vals[":date_created"] = &dynamodb.AttributeValue{S: aws.String(finalC)}
+					names["#date_created"] = aws.String(attrDateCreated)
 				}
 				first = false
 			}
@@ -633,9 +630,11 @@ func snowflakeFix(
 				if srcM == labelFivetranSynced {
 					updateExpr += exprSetApproxDateModified
 					vals[":approx_date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)}
+					names["#approx_date_modified"] = aws.String(attrApproxDateModified)
 				} else {
 					updateExpr += exprSetDateModified
 					vals[":date_modified"] = &dynamodb.AttributeValue{S: aws.String(finalM)}
+					names["#date_modified"] = aws.String(attrDateModified)
 				}
 			}
 
@@ -659,7 +658,9 @@ func snowflakeFix(
 				}
 			}
 
-			cmd := buildAwsCliUpdate(region, stage, tableName, id, updateExpr, names, vals, condAnyMissing)
+			// Build condition expression with names that are actually defined
+			condExpr := buildConditionExpression(names)
+			cmd := buildAwsCliUpdate(region, stage, tableName, id, updateExpr, names, vals, condExpr)
 			dbg("  SF CLI: %s", cmd)
 
 			if dryRun {
@@ -672,13 +673,16 @@ func snowflakeFix(
 				continue
 			}
 
+			// Build condition expression with names that are actually defined
+			condExpr = buildConditionExpression(names)
+
 			_, uerr := ddb.UpdateItem(&dynamodb.UpdateItemInput{
 				TableName:                 aws.String(tableName),
 				Key:                       map[string]*dynamodb.AttributeValue{"signature_id": {S: aws.String(id)}},
 				UpdateExpression:          aws.String(updateExpr),
 				ExpressionAttributeNames:  names,
 				ExpressionAttributeValues: vals,
-				ConditionExpression:       aws.String(condAnyMissing),
+				ConditionExpression:       aws.String(condExpr),
 			})
 			if uerr != nil {
 				log.Printf("Update failed (SF) %s: %v", id, uerr)
@@ -1058,6 +1062,39 @@ func parseSnowflakeCSV(b []byte) map[string]string {
 	return res
 }
 
+// buildConditionExpression builds a condition expression using only the attribute names
+// that are actually defined in the names map to avoid DynamoDB validation errors
+func buildConditionExpression(names map[string]*string) string {
+	var conditions []string
+
+	// Check for regular date_created field
+	if _, hasDateCreated := names["#date_created"]; hasDateCreated {
+		conditions = append(conditions, "attribute_not_exists(#date_created) OR #date_created = :empty")
+	}
+
+	// Check for regular date_modified field
+	if _, hasDateModified := names["#date_modified"]; hasDateModified {
+		conditions = append(conditions, "attribute_not_exists(#date_modified) OR #date_modified = :empty")
+	}
+
+	// Check for approx_date_created field
+	if _, hasApproxDateCreated := names["#approx_date_created"]; hasApproxDateCreated {
+		conditions = append(conditions, "attribute_not_exists(#approx_date_created) OR #approx_date_created = :empty")
+	}
+
+	// Check for approx_date_modified field
+	if _, hasApproxDateModified := names["#approx_date_modified"]; hasApproxDateModified {
+		conditions = append(conditions, "attribute_not_exists(#approx_date_modified) OR #approx_date_modified = :empty")
+	}
+
+	// If no specific conditions, use a basic condition that should always allow updates
+	if len(conditions) == 0 {
+		return "attribute_exists(signature_id)"
+	}
+
+	return strings.Join(conditions, " OR ")
+}
+
 // -----------------------------------------------------------------------------
 // AWS CLI builder & stats print
 // -----------------------------------------------------------------------------

utils/signature_dates_stats.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_not_exists(#a) OR #a = :nullval" --expression-attribute-names '{"#a":"date_created"}' --expression-attribute-values '{":nullval":{"NULL":true}}' --select "COUNT"
+aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_exists(#a)" --expression-attribute-names '{"#a":"approx_date_created"}' --select "COUNT"
+aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_not_exists(#a) OR #a = :nullval" --expression-attribute-names '{"#a":"date_modified"}' --expression-attribute-values '{":nullval":{"NULL":true}}' --select "COUNT"
+aws --profile lfproduct-prod dynamodb scan --table-name cla-prod-signatures --filter-expression "attribute_exists(#a)" --expression-attribute-names '{"#a":"approx_date_modified"}' --select "COUNT"