Skip to content

Commit fe30457

Browse files
lukaszgryglickilukaszgryglicki
authored
Merge pull request #4811 from linuxfoundation/unicron-logging-ecla-signed-events-and-updating-caches
Add logging of ECLA signed/ack events and invatlidate caches on those events
2 parents 4d3744f + 65a3956 commit fe30457

File tree

10 files changed

+183
-14
lines changed

10 files changed

+183
-14
lines changed

‎cla-backend-go/events/event_data.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,24 @@ type CorporateSignatureSignedEventData struct {
457457
SignatoryName string
458458
}
459459

460+
type EmployeeSignatureCreatedEventData struct {
461+
ProjectName string
462+
ProjectID string
463+
CompanyName string
464+
CompanyID string
465+
EmployeeUserID string
466+
EmployeeUserName string
467+
}
468+
469+
type EmployeeSignatureSignedEventData struct {
470+
ProjectName string
471+
ProjectID string
472+
CompanyName string
473+
CompanyID string
474+
EmployeeUserID string
475+
EmployeeUserName string
476+
}
477+
460478
// BypassCLAEventData event data model
461479
type BypassCLAEventData struct {
462480
Repo string
@@ -489,6 +507,16 @@ func (ed *CorporateSignatureSignedEventData) GetEventSummaryString(args *LogEven
489507
return data, true
490508
}
491509

510+
func (ed *EmployeeSignatureCreatedEventData) GetEventSummaryString(args *LogEventArgs) (string, bool) {
511+
data := fmt.Sprintf("The ECLA signature was created for the project %s and company %s by %s.", ed.ProjectName, ed.CompanyName, ed.EmployeeUserName)
512+
return data, true
513+
}
514+
515+
func (ed *EmployeeSignatureSignedEventData) GetEventSummaryString(args *LogEventArgs) (string, bool) {
516+
data := fmt.Sprintf("The ECLA signature was signed for the project %s and company %s by %s.", ed.ProjectName, ed.CompanyName, ed.EmployeeUserName)
517+
return data, true
518+
}
519+
492520
// GetEventDetailsString returns the details string for this event
493521
func (ed *SignatureAutoCreateECLAUpdatedEventData) GetEventDetailsString(args *LogEventArgs) (string, bool) {
494522

@@ -2786,3 +2814,15 @@ func (ed *IndividualSignatureSignedEventData) GetEventDetailsString(args *LogEve
27862814
args.LfUsername, ed.ProjectName)
27872815
return data, false
27882816
}
2817+
2818+
func (ed *EmployeeSignatureCreatedEventData) GetEventDetailsString(args *LogEventArgs) (string, bool) {
2819+
data := fmt.Sprintf("The user %s created an employee (ECLA) signature for project %s, company %s, employee %s.",
2820+
args.LfUsername, ed.ProjectName, ed.CompanyName, ed.EmployeeUserName)
2821+
return data, true
2822+
}
2823+
2824+
func (ed *EmployeeSignatureSignedEventData) GetEventDetailsString(args *LogEventArgs) (string, bool) {
2825+
data := fmt.Sprintf("The user %s signed an employee (ECLA) signature for project %s, company %s, employee %s.",
2826+
args.LfUsername, ed.ProjectName, ed.CompanyName, ed.EmployeeUserName)
2827+
return data, true
2828+
}

‎cla-backend-go/events/event_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,11 @@ const (
9797
ProjectServiceCLADisabled = "project.service.cla.disabled"
9898
SignatureAutoCreateECLAUpdated = "signature.auto_create_ecla.updated"
9999

100+
EmployeeSignatureCreated = "employee.signature.created"
101+
100102
IndividualSignatureSigned = "individual.signature.signed"
101103
CorporateSignatureSigned = "corporate.signature.signed"
104+
EmployeeSignatureSigned = "employee.signature.signed"
102105

103106
BypassCLA = "Bypass CLA"
104107
)

‎cla-backend-go/github/github_repository.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1629,7 +1629,7 @@ func UpdateCacheAfterSignature(ctx context.Context, user *models.User, projectID
16291629
ModelProjectUserCache.Set(projKey, user, true, affiliated)
16301630
}
16311631

1632-
log.WithFields(f).Infof("updated caches for user login=%q (GitHubID=%s), project=%s: marked as authorized for %d email(s)",
1632+
log.WithFields(f).Infof("updated caches for user login=%s (GitHubID=%s), project=%s: marked as authorized for %d email(s)",
16331633
loginLower, githubID, projectID, len(emails))
16341634

16351635
return nil

‎cla-backend-go/signatures/repository.go‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2582,7 +2582,49 @@ func (repo repository) CreateProjectCompanyEmployeeSignature(ctx context.Context
25822582
log.WithFields(f).WithError(putErr).Warn("cannot create new signature record")
25832583
return putErr
25842584
}
2585-
2585+
// Log the event
2586+
eventDataCreated := events.EmployeeSignatureCreatedEventData{
2587+
ProjectName: claGroupModel.ProjectName,
2588+
ProjectID: claGroupModel.ProjectID,
2589+
CompanyName: companyModel.CompanyName,
2590+
CompanyID: companyModel.CompanyID,
2591+
EmployeeUserID: employeeUserModel.UserID,
2592+
EmployeeUserName: employeeUserName,
2593+
}
2594+
log.WithFields(f).Debugf("logging event: %+v", eventDataCreated)
2595+
eventArgs := &events.LogEventArgs{
2596+
EventType: events.EmployeeSignatureCreated,
2597+
ProjectID: claGroupModel.ProjectID,
2598+
UserID: employeeUserModel.UserID,
2599+
LfUsername: employeeUserModel.LfUsername,
2600+
EventData: &eventDataCreated,
2601+
CLAGroupID: claGroupModel.ProjectID,
2602+
}
2603+
log.WithFields(f).Debugf("logging event: %+v", eventArgs)
2604+
repo.eventsService.LogEvent(eventArgs)
2605+
eventDataSigned := events.EmployeeSignatureSignedEventData{
2606+
ProjectName: claGroupModel.ProjectName,
2607+
ProjectID: claGroupModel.ProjectID,
2608+
CompanyName: companyModel.CompanyName,
2609+
CompanyID: companyModel.CompanyID,
2610+
EmployeeUserID: employeeUserModel.UserID,
2611+
EmployeeUserName: employeeUserName,
2612+
}
2613+
log.WithFields(f).Debugf("logging event: %+v", eventDataSigned)
2614+
eventArgs = &events.LogEventArgs{
2615+
EventType: events.EmployeeSignatureSigned,
2616+
ProjectID: claGroupModel.ProjectID,
2617+
UserID: employeeUserModel.UserID,
2618+
LfUsername: employeeUserModel.LfUsername,
2619+
EventData: &eventDataSigned,
2620+
CLAGroupID: claGroupModel.ProjectID,
2621+
}
2622+
log.WithFields(f).Debugf("logging event: %+v", eventArgs)
2623+
repo.eventsService.LogEvent(eventArgs)
2624+
err = github.UpdateCacheAfterSignature(ctx, employeeUserModel, claGroupModel.ProjectID)
2625+
if err != nil {
2626+
log.WithFields(f).WithError(err).Warnf("unable to update cache for user: %s, project ID: %s", employeeUserName, claGroupModel.ProjectID)
2627+
}
25862628
return nil
25872629
}
25882630

‎cla-backend-go/v2/sign/helpers.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
//nolint:gocyclo
2121
func (s service) updateChangeRequest(ctx context.Context, installationID, repositoryID, pullRequestID int64, projectID string) error {
2222
f := logrus.Fields{
23-
"functionName": "v1.signatures.service.updateChangeRequest",
23+
"functionName": "v2.sign.helpers.updateChangeRequest",
2424
"repositoryID": repositoryID,
2525
"pullRequestID": pullRequestID,
2626
"projectID": projectID,
@@ -104,7 +104,7 @@ func (s service) updateChangeRequest(ctx context.Context, installationID, reposi
104104
// true, true, nil if user has an ECLA (authorized, with company affiliation, no error)
105105
func (s service) hasUserSigned(ctx context.Context, user *models.User, projectID string) (*bool, *bool, error) {
106106
f := logrus.Fields{
107-
"functionName": "v1.signatures.service.hasUserSigned",
107+
"functionName": "v2.sign.helpers.hasUserSigned",
108108
"projectID": projectID,
109109
"user": user,
110110
}

‎cla-backend-go/v2/sign/service.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -613,10 +613,9 @@ func (s *service) SignedIndividualCallbackGithub(ctx context.Context, payload []
613613
}
614614
log.WithFields(f).Debugf("logging event: %+v", eventArgs)
615615
s.eventsService.LogEvent(eventArgs)
616-
err = github.UpdateCacheAfterSignature(context.Background(), claUser, signature.ProjectID)
616+
err = github.UpdateCacheAfterSignature(ctx, claUser, signature.ProjectID)
617617
if err != nil {
618618
log.WithFields(f).WithError(err).Warnf("unable to update cache for user: %s, project ID: %s", claUser.Username, signature.ProjectID)
619-
return nil
620619
}
621620

622621
} else {
@@ -1227,10 +1226,9 @@ func (s *service) SignedCorporateCallback(ctx context.Context, payload []byte, c
12271226
CompanyID: companyID,
12281227
CompanySFID: companyModel.CompanyExternalID,
12291228
})
1230-
err = github.UpdateCacheAfterSignature(context.Background(), user, signature.ProjectID)
1229+
err = github.UpdateCacheAfterSignature(ctx, user, signature.ProjectID)
12311230
if err != nil {
12321231
log.WithFields(f).WithError(err).Warnf("unable to update cache for company: %v, user: %s, project ID: %s", companyID, user.Username, signature.ProjectID)
1233-
return err
12341232
}
12351233

12361234
// Check if project is a gerrit instance

‎cla-backend/cla/models/docusign_models.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,19 @@ def request_employee_signature(self, project_id, company_id, user_id, return_url
776776
event_summary=event_summary,
777777
contains_pii=True,
778778
)
779+
Event.create_event(
780+
event_type=EventType.EmployeeSignatureSigned,
781+
event_company_id=company_id,
782+
event_cla_group_id=project_id,
783+
event_user_id=user_id,
784+
event_user_name=user.get_user_name() if user else None,
785+
event_data=event_data,
786+
event_summary=event_summary,
787+
contains_pii=True,
788+
)
789+
if return_url_type.lower() == "github":
790+
# Update cache to mark this user as authorized for the project - we only need this for GitHub as we only use caching in GitHub
791+
update_cache_after_signature(user, project)
779792

780793
# If the project does not require an ICLA to be signed, update the pull request and remove the active
781794
# signature metadata.

‎utils/delete_user_repository_project_signature.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# DRY=1 - dry run
3+
# DRY=1 ./utils/delete_user_repository_project_signature.sh mlehotskylf-org2/easycla-dev lukaszgryglicki
4+
if [ -z "$STAGE" ]
5+
then
6+
export STAGE=dev
7+
fi
8+
if [ -z "$1" ]
9+
then
10+
echo "$0: you need to provide the repository name, for example: 'mlehotskylf-org2/easycla-dev'"
11+
exit 1
12+
fi
13+
REPO_NAME=$1
14+
if [ -z "$2" ]
15+
then
16+
echo "$0: you need to provide the GitHub user name, for example: 'lukaszgryglicki'"
17+
exit 2
18+
fi
19+
GITHUB_USER=$2
20+
PROJECT_IDS=$(./utils/scan.sh repositories repository_name "${REPO_NAME}" | jq -r '.[].repository_project_id.S')
21+
if [ -z "$PROJECT_IDS" ] || [ "$PROJECT_IDS" == "null" ]
22+
then
23+
echo "$0: cannot find project IDs for repository ${REPO_NAME}"
24+
exit 3
25+
fi
26+
echo "Project IDs: ${PROJECT_IDS}"
27+
USER_IDS=$(./utils/scan.sh users user_github_username "${GITHUB_USER}" | jq -r '.[].user_id.S')
28+
if [ -z "$USER_IDS" ] || [ "$USER_IDS" == "null" ]
29+
then
30+
echo "$0: cannot find user ID for GitHub user ${GITHUB_USER}"
31+
exit 4
32+
fi
33+
echo "User IDs: ${USER_IDS}"
34+
for PROJECT_ID in $PROJECT_IDS
35+
do
36+
for USER_ID in $USER_IDS
37+
do
38+
echo "Deleting user repository project signature for user ID ${USER_ID} and project ID ${PROJECT_ID}"
39+
SIGS=$(aws dynamodb query \
40+
--profile "lfproduct-${STAGE}" \
41+
--table-name "cla-${STAGE}-signatures" \
42+
--index-name "signature-project-reference-index" \
43+
--key-condition-expression "signature_project_id = :pid AND signature_reference_id = :uid" \
44+
--filter-expression "signature_reference_type = :rtype AND signature_signed = :true AND signature_approved = :true" \
45+
--expression-attribute-values '{
46+
":pid": {"S":"'"${PROJECT_ID}"'"},
47+
":uid": {"S":"'"${USER_ID}"'"},
48+
":rtype":{"S":"user"},
49+
":true":{"BOOL": true}
50+
}' --output json | jq -r '.Items[].signature_id.S')
51+
for SIG in $SIGS
52+
do
53+
if [ ! -z "$DRY" ]
54+
then
55+
echo "DRY RUN: would delete this signature:"
56+
./utils/scan.sh signatures signature_id "${SIG}"
57+
echo "DRY RUN: would delete this ^ signature."
58+
continue
59+
fi
60+
echo "Deleting signature ID ${SIG}"
61+
aws dynamodb delete-item --profile "lfproduct-${STAGE}" --table-name "cla-${STAGE}-signatures" --key '{"signature_id": {"S":"'"${SIG}"'"}}'
62+
done
63+
done
64+
done

‎utils/search_aws_log_group.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,6 @@ fi
6868
if [ ! -z "${DEBUG}" ]
6969
then
7070
echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${2}\""
71-
aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\""
72-
else
73-
aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\"" | jq -r '.events'
7471
fi
72+
aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\"" | jq -r '.events | sort_by(.timestamp)'
7573

‎utils/search_aws_logs.sh‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
# REGION=us-east-1|us-east-2 STAGE=dev DEBUG=1 DTFROM='3 days ago' DTTO='2 days ago' OUT=logs.json ./utils/search_aws_logs.sh 'error'
77
# DEBUG=1 STAGE=dev REGION=us-east-1 DTFROM='10 days ago' DTTO='1 second ago' OUT=api-logs-dev.json ./utils/search_aws_logs.sh 'LG:api-request-path' && ./utils/count_apis.sh api-logs-dev.json
88
# DEBUG=1 STAGE=prod REGION=us-east-1 NO_ECHO=1 DTFROM='10 days ago' DTTO='1 second ago' OUT=api-logs-prod.json ./utils/search_aws_logs.sh 'LG:api-request-path' && ./utils/count_apis.sh api-logs-prod.json
9+
# REVERSE=1 - reverse logs order - newest on top.
910
# To find distinct log groups: | jq -r 'map(.logGroupName) | unique | .[]'
1011
# in us-east-1 (mostly V1, V2 and V3):
1112
# To see specific log group: | jq 'map(select(.logGroupName == "/aws/lambda/cla-backend-dev-apiv1"))'
@@ -96,7 +97,7 @@ do
9697
--start-time "${DTFROM}" \
9798
--end-time "${DTTO}" \
9899
--filter-pattern "\"${search}\"" | jq --arg logGroupName "$log_group" '
99-
.events[] |
100+
.events[]? |
100101
.logGroupName = $logGroupName |
101102
.dt = ( (.timestamp / 1000) | strftime("%Y-%m-%d %H:%M:%S") ) + "." + ( (.timestamp % 1000 | tostring) | if length == 1 then "00" + . elif length == 2 then "0" + . else . end )
102103
')
@@ -117,9 +118,19 @@ done
117118

118119
if [ ! -z "${OUT}" ]
119120
then
120-
echo "$jsons" | jq -s 'sort_by(.dt) | reverse' > "${OUT}"
121+
if [ ! -z "${REVERSE}" ]
122+
then
123+
echo "$jsons" | jq -s '(. // []) | sort_by(.dt) | reverse' > "${OUT}"
124+
else
125+
echo "$jsons" | jq -s '(. // []) | sort_by(.dt)' > "${OUT}"
126+
fi
121127
fi
122128
if [ -z "${NO_ECHO}" ]
123129
then
124-
echo "$jsons" | jq -s 'sort_by(.dt) | reverse'
130+
if [ ! -z "${REVERSE}" ]
131+
then
132+
echo "$jsons" | jq -s '(. // []) | sort_by(.dt) | reverse'
133+
else
134+
echo "$jsons" | jq -s '(. // []) | sort_by(.dt)'
135+
fi
125136
fi

0 commit comments

Comments
 (0)