Merge pull request #4470 from communitybridge/revert-4469-feature/ecla-duplicates

Revert "[#4460] Feature/Corporate contributors dups"

Author: nickmango

cla-backend-go/signatures/repository.go

@@ -4616,32 +4616,6 @@ func (repo repository) addAdditionalICLAMetaData(f logrus.Fields, intermediateRe
 	return finalResults, nil
 }
 
-func deduplicateSignatures(signatures []map[string]*dynamodb.AttributeValue) []map[string]*dynamodb.AttributeValue {
-	f := logrus.Fields{
-		"functionName": "v1.signatures.repository.deduplicateSignatures",
-	}
-	// Map to keep track of the latest unique signatures by SignatureReferenceID
-	uniqueSignatures := make(map[string]map[string]*dynamodb.AttributeValue)
-	for _, sig := range signatures {
-		log.WithFields(f).Debugf("processing signature: %+v", sig)
-		refID := sig["signature_reference_id"].S
-		existingSig, exists := uniqueSignatures[*refID]
-		if !exists || (sig["date_created"] != nil && existingSig["date_created"] != nil && *sig["date_created"].S > *existingSig["date_created"].S) {
-			uniqueSignatures[*refID] = sig
-		} else if !exists || (sig["date_created"] != nil && existingSig["date_created"] == nil) {
-			uniqueSignatures[*refID] = sig
-		} else if !exists || (sig["date_created"] == nil && existingSig["date_created"] != nil) {
-			uniqueSignatures[*refID] = sig
-		}
-	}
-	// Convert the map back to a slice
-	var uniqueSignatureList []map[string]*dynamodb.AttributeValue
-	for _, sig := range uniqueSignatures {
-		uniqueSignatureList = append(uniqueSignatureList, sig)
-	}
-	return uniqueSignatureList
-}
-
 func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, claGroupID string, companyID *string, pageSize *int64, nextKey *string, searchTerm *string) (*models.CorporateContributorList, error) {
 	f := logrus.Fields{
 		"functionName":   "v1.signatures.repository.GetClaGroupCorporateContributors",
@@ -4724,10 +4698,10 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla
 
 	out := &models.CorporateContributorList{List: make([]*models.CorporateContributor, 0)}
 	var lastEvaluatedKey string
-	var allItems []map[string]*dynamodb.AttributeValue
+
 	currentCount := int64(0)
 
-	for ok := true; ok; ok = lastEvaluatedKey != "" {
+	for ok := true; ok; ok = lastEvaluatedKey != "" && currentCount < *pageSize {
 		// Make the DynamoDB Query API call
 		log.WithFields(f).Debug("querying signatures...")
 		results, queryErr := repo.dynamoDBClient.Query(queryInput)
@@ -4736,90 +4710,92 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla
 			return nil, queryErr
 		}
 
-		// Append the results...
-		allItems = append(allItems, results.Items...)
-		currentCount += int64(len(results.Items))
-
-		if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
-			lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
-			queryInput.ExclusiveStartKey = results.LastEvaluatedKey
-		} else {
-			lastEvaluatedKey = ""
-		}
-
-	}
-
-	allItems = deduplicateSignatures(allItems)
-	var dbSignatures []ItemSignature
-	err = dynamodbattribute.UnmarshalListOfMaps(allItems, &dbSignatures)
-	if err != nil {
-		log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
-			claGroupID, err)
-		return nil, err
-	}
+		var dbSignatures []ItemSignature
 
-	log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
-	for _, sig := range dbSignatures {
-		var sigCreatedTime = sig.DateCreated
-		t, err := utils.ParseDateTime(sig.DateCreated)
+		err := dynamodbattribute.UnmarshalListOfMaps(results.Items, &dbSignatures)
 		if err != nil {
-			log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
-		} else {
-			sigCreatedTime = utils.TimeToString(t)
+			log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
+				claGroupID, err)
+			return nil, err
 		}
 
-		// Set the signed date/time
-		var sigSignedTime string
-		// Use the user docusign date signed value if it is present - older signatures do not have this
-		if sig.UserDocusignDateSigned != "" {
-			// Put the date into a standard format
-			t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
+		log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
+		for _, sig := range dbSignatures {
+			var sigCreatedTime = sig.DateCreated
+			t, err := utils.ParseDateTime(sig.DateCreated)
 			if err != nil {
-				log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
+				log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
 			} else {
-				sigSignedTime = utils.TimeToString(t)
+				sigCreatedTime = utils.TimeToString(t)
 			}
-		} else {
-			// Put the date into a standard format
-			t, err = utils.ParseDateTime(sig.DateCreated)
-			if err != nil {
-				log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
+
+			// Set the signed date/time
+			var sigSignedTime string
+			// Use the user docusign date signed value if it is present - older signatures do not have this
+			if sig.UserDocusignDateSigned != "" {
+				// Put the date into a standard format
+				t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
+				if err != nil {
+					log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
+				} else {
+					sigSignedTime = utils.TimeToString(t)
+				}
 			} else {
-				sigSignedTime = utils.TimeToString(t)
+				// Put the date into a standard format
+				t, err = utils.ParseDateTime(sig.DateCreated)
+				if err != nil {
+					log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
+				} else {
+					sigSignedTime = utils.TimeToString(t)
+				}
 			}
-		}
 
-		signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))
+			signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))
+
+			sigName := sig.UserName
+			user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
+			if userErr != nil {
+				log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
+			}
+			if user != nil && sigName == "" {
+				sigName = user.Username
+			}
 
-		sigName := sig.UserName
-		user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
-		if userErr != nil {
-			log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
+			out.List = append(out.List, &models.CorporateContributor{
+				SignatureID:            sig.SignatureID,
+				GithubID:               sig.UserGithubUsername,
+				LinuxFoundationID:      sig.UserLFUsername,
+				Name:                   sigName,
+				SignatureVersion:       signatureVersion,
+				Email:                  sig.UserEmail,
+				Timestamp:              sigCreatedTime,
+				UserDocusignName:       sig.UserDocusignName,
+				UserDocusignDateSigned: sigSignedTime,
+				SignatureModified:      sig.DateModified,
+				SignatureApproved:      sig.SignatureApproved,
+				SignatureSigned:        sig.SignatureSigned,
+			})
+
+			// Increment the current count
+			currentCount++
+			if currentCount >= *pageSize {
+				break
+			}
 		}
-		if user != nil && sigName == "" {
-			sigName = user.Username
+
+		if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
+			lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
+			queryInput.ExclusiveStartKey = results.LastEvaluatedKey
+		} else {
+			lastEvaluatedKey = ""
 		}
 
-		out.List = append(out.List, &models.CorporateContributor{
-			SignatureID:            sig.SignatureID,
-			GithubID:               sig.UserGithubUsername,
-			LinuxFoundationID:      sig.UserLFUsername,
-			Name:                   sigName,
-			SignatureVersion:       signatureVersion,
-			Email:                  sig.UserEmail,
-			Timestamp:              sigCreatedTime,
-			UserDocusignName:       sig.UserDocusignName,
-			UserDocusignDateSigned: sigSignedTime,
-			SignatureModified:      sig.DateModified,
-			SignatureApproved:      sig.SignatureApproved,
-			SignatureSigned:        sig.SignatureSigned,
-		})
 	}
 	sort.Slice(out.List, func(i, j int) bool {
 		return out.List[i].Name < out.List[j].Name
 	})
 
-	out.ResultCount = int64(len(out.List))
+	out.ResultCount = currentCount
 	out.TotalCount = totalCount
 	out.NextKey = lastEvaluatedKey
 
@@ -4875,7 +4851,6 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
 
 	var lastEvaluatedKey string
 	var totalCount int64
-	var allItems []map[string]*dynamodb.AttributeValue
 
 	// Loop until we have all the records - we'll get a nil lastEvaluatedKey when we're done
 	for ok := true; ok; ok = lastEvaluatedKey != "" {
@@ -4886,7 +4861,8 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
 			return
 		}
 
-		allItems = append(allItems, results.Items...)
+		// Add the count to the total
+		totalCount += *results.Count
 
 		// Set the last evaluated key
 		if results.LastEvaluatedKey["signature_id"] != nil {
@@ -4897,13 +4873,10 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
 		}
 	}
 
-	// Deduplicate the signatures
-	// uniqueSignatures := deduplicateSignatures(allItems)
-	totalCount = int64(len(allItems))
-	totalCountChannel <- totalCount
-
 	log.WithFields(f).Debugf("total signature count query took: %s", time.Since(beforeQuery))
 
+	totalCountChannel <- totalCount
+
 }
 
 // EclaAutoCreate this routine updates the CCLA signature record by adjusting the auto_create_ecla column to the specified value