Skip to content

Commit a5c3d85

Browse files
nickmangonickmango
authored
Merge pull request #4469 from nickmango/feature/ecla-duplicates
[#4460] Feature/Corporate contributors dups
2 parents eff666e + 79a9dd1 commit a5c3d85

File tree

1 file changed

+100
-73
lines changed

1 file changed

+100
-73
lines changed

‎cla-backend-go/signatures/repository.go‎

Lines changed: 100 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -4616,6 +4616,32 @@ func (repo repository) addAdditionalICLAMetaData(f logrus.Fields, intermediateRe
46164616
return finalResults, nil
46174617
}
46184618

4619+
func deduplicateSignatures(signatures []map[string]*dynamodb.AttributeValue) []map[string]*dynamodb.AttributeValue {
4620+
f := logrus.Fields{
4621+
"functionName": "v1.signatures.repository.deduplicateSignatures",
4622+
}
4623+
// Map to keep track of the latest unique signatures by SignatureReferenceID
4624+
uniqueSignatures := make(map[string]map[string]*dynamodb.AttributeValue)
4625+
for _, sig := range signatures {
4626+
log.WithFields(f).Debugf("processing signature: %+v", sig)
4627+
refID := sig["signature_reference_id"].S
4628+
existingSig, exists := uniqueSignatures[*refID]
4629+
if !exists || (sig["date_created"] != nil && existingSig["date_created"] != nil && *sig["date_created"].S > *existingSig["date_created"].S) {
4630+
uniqueSignatures[*refID] = sig
4631+
} else if !exists || (sig["date_created"] != nil && existingSig["date_created"] == nil) {
4632+
uniqueSignatures[*refID] = sig
4633+
} else if !exists || (sig["date_created"] == nil && existingSig["date_created"] != nil) {
4634+
uniqueSignatures[*refID] = sig
4635+
}
4636+
}
4637+
// Convert the map back to a slice
4638+
var uniqueSignatureList []map[string]*dynamodb.AttributeValue
4639+
for _, sig := range uniqueSignatures {
4640+
uniqueSignatureList = append(uniqueSignatureList, sig)
4641+
}
4642+
return uniqueSignatureList
4643+
}
4644+
46194645
func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, claGroupID string, companyID *string, pageSize *int64, nextKey *string, searchTerm *string) (*models.CorporateContributorList, error) {
46204646
f := logrus.Fields{
46214647
"functionName": "v1.signatures.repository.GetClaGroupCorporateContributors",
@@ -4698,10 +4724,10 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla
46984724

46994725
out := &models.CorporateContributorList{List: make([]*models.CorporateContributor, 0)}
47004726
var lastEvaluatedKey string
4701-
4727+
var allItems []map[string]*dynamodb.AttributeValue
47024728
currentCount := int64(0)
47034729

4704-
for ok := true; ok; ok = lastEvaluatedKey != "" && currentCount < *pageSize {
4730+
for ok := true; ok; ok = lastEvaluatedKey != "" {
47054731
// Make the DynamoDB Query API call
47064732
log.WithFields(f).Debug("querying signatures...")
47074733
results, queryErr := repo.dynamoDBClient.Query(queryInput)
@@ -4710,92 +4736,90 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla
47104736
return nil, queryErr
47114737
}
47124738

4713-
var dbSignatures []ItemSignature
4739+
// Append the results...
4740+
allItems = append(allItems, results.Items...)
4741+
currentCount += int64(len(results.Items))
47144742

4715-
err := dynamodbattribute.UnmarshalListOfMaps(results.Items, &dbSignatures)
4743+
if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
4744+
lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
4745+
queryInput.ExclusiveStartKey = results.LastEvaluatedKey
4746+
} else {
4747+
lastEvaluatedKey = ""
4748+
}
4749+
4750+
}
4751+
4752+
allItems = deduplicateSignatures(allItems)
4753+
var dbSignatures []ItemSignature
4754+
err = dynamodbattribute.UnmarshalListOfMaps(allItems, &dbSignatures)
4755+
if err != nil {
4756+
log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
4757+
claGroupID, err)
4758+
return nil, err
4759+
}
4760+
4761+
log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
4762+
for _, sig := range dbSignatures {
4763+
var sigCreatedTime = sig.DateCreated
4764+
t, err := utils.ParseDateTime(sig.DateCreated)
47164765
if err != nil {
4717-
log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
4718-
claGroupID, err)
4719-
return nil, err
4766+
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
4767+
} else {
4768+
sigCreatedTime = utils.TimeToString(t)
47204769
}
47214770

4722-
log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
4723-
for _, sig := range dbSignatures {
4724-
var sigCreatedTime = sig.DateCreated
4725-
t, err := utils.ParseDateTime(sig.DateCreated)
4771+
// Set the signed date/time
4772+
var sigSignedTime string
4773+
// Use the user docusign date signed value if it is present - older signatures do not have this
4774+
if sig.UserDocusignDateSigned != "" {
4775+
// Put the date into a standard format
4776+
t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
47264777
if err != nil {
4727-
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
4778+
log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
47284779
} else {
4729-
sigCreatedTime = utils.TimeToString(t)
4780+
sigSignedTime = utils.TimeToString(t)
47304781
}
4731-
4732-
// Set the signed date/time
4733-
var sigSignedTime string
4734-
// Use the user docusign date signed value if it is present - older signatures do not have this
4735-
if sig.UserDocusignDateSigned != "" {
4736-
// Put the date into a standard format
4737-
t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
4738-
if err != nil {
4739-
log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
4740-
} else {
4741-
sigSignedTime = utils.TimeToString(t)
4742-
}
4782+
} else {
4783+
// Put the date into a standard format
4784+
t, err = utils.ParseDateTime(sig.DateCreated)
4785+
if err != nil {
4786+
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
47434787
} else {
4744-
// Put the date into a standard format
4745-
t, err = utils.ParseDateTime(sig.DateCreated)
4746-
if err != nil {
4747-
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
4748-
} else {
4749-
sigSignedTime = utils.TimeToString(t)
4750-
}
4788+
sigSignedTime = utils.TimeToString(t)
47514789
}
4790+
}
47524791

4753-
signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))
4754-
4755-
sigName := sig.UserName
4756-
user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
4757-
if userErr != nil {
4758-
log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
4759-
}
4760-
if user != nil && sigName == "" {
4761-
sigName = user.Username
4762-
}
4792+
signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))
47634793

4764-
out.List = append(out.List, &models.CorporateContributor{
4765-
SignatureID: sig.SignatureID,
4766-
GithubID: sig.UserGithubUsername,
4767-
LinuxFoundationID: sig.UserLFUsername,
4768-
Name: sigName,
4769-
SignatureVersion: signatureVersion,
4770-
Email: sig.UserEmail,
4771-
Timestamp: sigCreatedTime,
4772-
UserDocusignName: sig.UserDocusignName,
4773-
UserDocusignDateSigned: sigSignedTime,
4774-
SignatureModified: sig.DateModified,
4775-
SignatureApproved: sig.SignatureApproved,
4776-
SignatureSigned: sig.SignatureSigned,
4777-
})
4778-
4779-
// Increment the current count
4780-
currentCount++
4781-
if currentCount >= *pageSize {
4782-
break
4783-
}
4794+
sigName := sig.UserName
4795+
user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
4796+
if userErr != nil {
4797+
log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
47844798
}
4785-
4786-
if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
4787-
lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
4788-
queryInput.ExclusiveStartKey = results.LastEvaluatedKey
4789-
} else {
4790-
lastEvaluatedKey = ""
4799+
if user != nil && sigName == "" {
4800+
sigName = user.Username
47914801
}
47924802

4803+
out.List = append(out.List, &models.CorporateContributor{
4804+
SignatureID: sig.SignatureID,
4805+
GithubID: sig.UserGithubUsername,
4806+
LinuxFoundationID: sig.UserLFUsername,
4807+
Name: sigName,
4808+
SignatureVersion: signatureVersion,
4809+
Email: sig.UserEmail,
4810+
Timestamp: sigCreatedTime,
4811+
UserDocusignName: sig.UserDocusignName,
4812+
UserDocusignDateSigned: sigSignedTime,
4813+
SignatureModified: sig.DateModified,
4814+
SignatureApproved: sig.SignatureApproved,
4815+
SignatureSigned: sig.SignatureSigned,
4816+
})
47934817
}
47944818
sort.Slice(out.List, func(i, j int) bool {
47954819
return out.List[i].Name < out.List[j].Name
47964820
})
47974821

4798-
out.ResultCount = currentCount
4822+
out.ResultCount = int64(len(out.List))
47994823
out.TotalCount = totalCount
48004824
out.NextKey = lastEvaluatedKey
48014825

@@ -4851,6 +4875,7 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
48514875

48524876
var lastEvaluatedKey string
48534877
var totalCount int64
4878+
var allItems []map[string]*dynamodb.AttributeValue
48544879

48554880
// Loop until we have all the records - we'll get a nil lastEvaluatedKey when we're done
48564881
for ok := true; ok; ok = lastEvaluatedKey != "" {
@@ -4861,8 +4886,7 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
48614886
return
48624887
}
48634888

4864-
// Add the count to the total
4865-
totalCount += *results.Count
4889+
allItems = append(allItems, results.Items...)
48664890

48674891
// Set the last evaluated key
48684892
if results.LastEvaluatedKey["signature_id"] != nil {
@@ -4873,10 +4897,13 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
48734897
}
48744898
}
48754899

4876-
log.WithFields(f).Debugf("total signature count query took: %s", time.Since(beforeQuery))
4877-
4900+
// Deduplicate the signatures
4901+
// uniqueSignatures := deduplicateSignatures(allItems)
4902+
totalCount = int64(len(allItems))
48784903
totalCountChannel <- totalCount
48794904

4905+
log.WithFields(f).Debugf("total signature count query took: %s", time.Since(beforeQuery))
4906+
48804907
}
48814908

48824909
// EclaAutoCreate this routine updates the CCLA signature record by adjusting the auto_create_ecla column to the specified value

0 commit comments

Comments
 (0)