Skip to content

Commit c2d23f4

Browse files
lukaszgryglickilukaszgryglicki
authored
Merge pull request #4657 from communitybridge/unicron-templates-analysis
Add utils to get (latest) ICLA/CCLA templates and fix get latest document functions in py and go + add test coverage
2 parents 4cb6d7b + 893a703 commit c2d23f4

File tree

7 files changed

+297
-35
lines changed

7 files changed

+297
-35
lines changed

‎cla-backend-go/project/common/helpers.go‎

Lines changed: 21 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package common
55

66
import (
77
"context"
8-
"fmt"
98
"strconv"
109
"time"
1110

@@ -75,58 +74,50 @@ func GetCurrentDocument(ctx context.Context, docs []models.ClaGroupDocument) (mo
7574
"functionName": "v1.project.helpers.GetCurrentDocument",
7675
utils.XREQUESTID: ctx.Value(utils.XREQUESTID),
7776
}
78-
var currentDoc models.ClaGroupDocument
79-
var currentDocVersion float64
80-
var currentDocDateTime time.Time
77+
var currDoc models.ClaGroupDocument
78+
var lastDateTime time.Time
79+
lastMaj, lastMin := 0, -1
8180
for _, doc := range docs {
82-
maj, err := strconv.Atoi(doc.DocumentMajorVersion)
81+
currMaj, err := strconv.Atoi(doc.DocumentMajorVersion)
8382
if err != nil {
8483
log.WithFields(f).WithError(err).Warnf("invalid major number in cla group: %s", doc.DocumentMajorVersion)
8584
continue
8685
}
8786

88-
min, err := strconv.Atoi(doc.DocumentMinorVersion)
87+
currMin, err := strconv.Atoi(doc.DocumentMinorVersion)
8988
if err != nil {
9089
log.WithFields(f).WithError(err).Warnf("invalid minor number in cla group: %s", doc.DocumentMinorVersion)
9190
continue
9291
}
9392

94-
version, err := strconv.ParseFloat(fmt.Sprintf("%d.%d", maj, min), 32)
93+
currDateTime, err := utils.ParseDateTime(doc.DocumentCreationDate)
9594
if err != nil {
96-
log.WithFields(f).WithError(err).Warnf("invalid major/minor version in cla group: %s.%s", doc.DocumentMajorVersion, doc.DocumentMinorVersion)
95+
log.WithFields(f).WithError(err).Warnf("invalid date time in cla group: %s", doc.DocumentCreationDate)
9796
continue
9897
}
9998

100-
dateTime, err := utils.ParseDateTime(doc.DocumentCreationDate)
101-
if err != nil {
102-
log.WithFields(f).WithError(err).Warnf("invalid date time in cla group: %s", doc.DocumentCreationDate)
99+
if currMaj > lastMaj {
100+
lastMaj = currMaj
101+
lastMin = currMin
102+
lastDateTime = currDateTime
103+
currDoc = doc
103104
continue
104105
}
105106

106-
// // No previous, use the first...
107-
// if currentDoc == (models.ClaGroupDocument{}) {
108-
// currentDoc = doc
109-
// currentDocVersion = version
110-
// currentDocDateTime = dateTime
111-
// continue
112-
// }
113-
114-
// Newer version...
115-
if version > currentDocVersion {
116-
currentDoc = doc
117-
currentDocVersion = version
118-
currentDocDateTime = dateTime
107+
if currMaj == lastMaj && currMin > lastMin {
108+
lastMin = currMin
109+
lastDateTime = currDateTime
110+
currDoc = doc
111+
continue
119112
}
120113

121-
// Same version, but a later date...
122-
if version == currentDocVersion && dateTime.After(currentDocDateTime) {
123-
currentDoc = doc
124-
currentDocVersion = version
125-
currentDocDateTime = dateTime
114+
if currMaj == lastMaj && currMin == lastMin && currDateTime.After(lastDateTime) {
115+
lastDateTime = currDateTime
116+
currDoc = doc
126117
}
127118
}
128119

129-
return currentDoc, nil
120+
return currDoc, nil
130121
}
131122

132123
func AreClaGroupDocumentsEqual(doc1, doc2 models.ClaGroupDocument) bool {

‎cla-backend-go/tests/project_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ package tests
55

66
import (
77
"context"
8+
"strconv"
89
"testing"
10+
"time"
911

1012
"github.com/communitybridge/easycla/cla-backend-go/project/common"
1113

@@ -14,6 +16,84 @@ import (
1416
"github.com/stretchr/testify/assert"
1517
)
1618

19+
func createDocument(major, minor int, date time.Time, name string) models.ClaGroupDocument {
20+
return models.ClaGroupDocument{
21+
DocumentMajorVersion: strconv.Itoa(major),
22+
DocumentMinorVersion: strconv.Itoa(minor),
23+
DocumentCreationDate: utils.TimeToString(date),
24+
DocumentName: name,
25+
}
26+
}
27+
28+
func mustParseTime(t *testing.T, value string) time.Time {
29+
dt, err := time.Parse(time.RFC3339, value)
30+
assert.Nil(t, err)
31+
return dt
32+
}
33+
34+
func TestGetCurrentDocument(t *testing.T) {
35+
// Dates
36+
dt_250217 := mustParseTime(t, "2025-02-17T15:00:13Z")
37+
dt_240217 := mustParseTime(t, "2024-02-17T15:00:13Z")
38+
dt_240218 := mustParseTime(t, "2024-02-18T15:00:13Z")
39+
dt_230217 := mustParseTime(t, "2023-02-17T15:00:13Z")
40+
dt_230218 := mustParseTime(t, "2023-02-18T15:00:13Z")
41+
dt_220218 := mustParseTime(t, "2022-02-18T15:00:13Z")
42+
43+
// Create documents
44+
document_15 := createDocument(1, 5, dt_250217, "document_15")
45+
document_29 := createDocument(2, 9, dt_240217, "document_29")
46+
document_29_newer := createDocument(2, 9, dt_240218, "document_29_newer")
47+
document_210 := createDocument(2, 10, dt_230217, "document_210")
48+
document_210_newer := createDocument(2, 10, dt_230218, "document_210_newer")
49+
document_30 := createDocument(3, 0, dt_220218, "document_30")
50+
document_31 := createDocument(3, 1, dt_220218, "document_31")
51+
document_310 := createDocument(3, 10, dt_220218, "document_310")
52+
53+
// Test cases
54+
tests := []struct {
55+
docs []models.ClaGroupDocument
56+
expectedName string
57+
expectedMajor int
58+
expectedMinor int
59+
}{
60+
{[]models.ClaGroupDocument{document_29}, "document_29", 2, 9},
61+
{[]models.ClaGroupDocument{document_29_newer, document_29}, "document_29_newer", 2, 9},
62+
{[]models.ClaGroupDocument{document_29, document_29_newer}, "document_29_newer", 2, 9},
63+
{[]models.ClaGroupDocument{document_29, document_210}, "document_210", 2, 10},
64+
{[]models.ClaGroupDocument{document_29_newer, document_210}, "document_210", 2, 10},
65+
{[]models.ClaGroupDocument{document_29, document_210_newer}, "document_210_newer", 2, 10},
66+
{[]models.ClaGroupDocument{document_29_newer, document_210_newer}, "document_210_newer", 2, 10},
67+
{[]models.ClaGroupDocument{document_29, document_29_newer, document_210}, "document_210", 2, 10},
68+
{[]models.ClaGroupDocument{document_29, document_210, document_210_newer}, "document_210_newer", 2, 10},
69+
{[]models.ClaGroupDocument{document_29, document_29_newer, document_210, document_210_newer}, "document_210_newer", 2, 10},
70+
{[]models.ClaGroupDocument{document_210, document_210_newer, document_29_newer, document_29}, "document_210_newer", 2, 10},
71+
{[]models.ClaGroupDocument{document_210, document_15, document_210_newer, document_29_newer, document_29}, "document_210_newer", 2, 10},
72+
{[]models.ClaGroupDocument{document_210, document_210_newer, document_29_newer, document_30, document_29}, "document_30", 3, 0},
73+
{[]models.ClaGroupDocument{document_210, document_15, document_210_newer, document_29_newer, document_30, document_29}, "document_30", 3, 0},
74+
{[]models.ClaGroupDocument{document_15, document_30, document_29}, "document_30", 3, 0},
75+
{[]models.ClaGroupDocument{document_31, document_310, document_30}, "document_310", 3, 10},
76+
}
77+
78+
for _, tt := range tests {
79+
doc, err := common.GetCurrentDocument(context.Background(), tt.docs)
80+
assert.Nil(t, err)
81+
82+
// Check the document name
83+
assert.Equal(t, tt.expectedName, doc.DocumentName)
84+
85+
// Check the major version
86+
major, err := strconv.Atoi(doc.DocumentMajorVersion)
87+
assert.Nil(t, err)
88+
assert.Equal(t, tt.expectedMajor, major)
89+
90+
// Check the minor version
91+
minor, err := strconv.Atoi(doc.DocumentMinorVersion)
92+
assert.Nil(t, err)
93+
assert.Equal(t, tt.expectedMinor, minor)
94+
}
95+
}
96+
1797
func TestGetCurrentDocumentVersion(t *testing.T) {
1898
currentTime, _ := utils.CurrentTime()
1999
yesterday := currentTime.AddDate(0, 0, -1)

‎cla-backend/cla/models/dynamo_models.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1409,13 +1409,16 @@ def _get_latest_version(self, documents):
14091409
if current_major > last_major:
14101410
last_major = current_major
14111411
last_minor = current_minor
1412+
latest_date = document.get_document_creation_date()
14121413
current_document = document
14131414
continue
14141415
if current_major == last_major and current_minor > last_minor:
14151416
last_minor = current_minor
1417+
latest_date = document.get_document_creation_date()
14161418
current_document = document
1419+
continue
14171420
# Retrieve document that has the latest date
1418-
if not latest_date or document.get_document_creation_date() > latest_date:
1421+
if current_major == last_major and current_minor == last_minor and (not latest_date or document.get_document_creation_date() > latest_date):
14191422
latest_date = document.get_document_creation_date()
14201423
current_document = document
14211424
return (last_major, last_minor, current_document)

‎cla-backend/cla/tests/unit/test_dynamo_models.py‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,78 @@
55

66
import pytest
77
from cla import utils
8-
from cla.models.dynamo_models import Company, User
8+
from cla.models.dynamo_models import Company, User, Project, Document
99

1010

1111
@pytest.fixture
1212
def user():
13-
yield User()
13+
yield User()
1414

1515

16+
@pytest.fixture
17+
def project():
18+
yield Project()
19+
20+
@pytest.fixture
21+
def document_factory():
22+
def create_document(major, minor, date):
23+
mock_document = Mock(spec=Document)
24+
mock_document.get_document_major_version.return_value = major
25+
mock_document.get_document_minor_version.return_value = minor
26+
mock_document.get_document_creation_date.return_value = date
27+
return mock_document
28+
return create_document
29+
30+
@pytest.fixture
31+
def document_15(document_factory):
32+
return document_factory(1, 5, "2025-02-17T15:00:13Z")
33+
34+
@pytest.fixture
35+
def document_29(document_factory):
36+
return document_factory(2, 9, "2024-02-17T15:00:13Z")
37+
38+
@pytest.fixture
39+
def document_29_newer(document_factory):
40+
return document_factory(2, 9, "2024-02-18T15:00:13Z")
41+
42+
@pytest.fixture
43+
def document_210(document_factory):
44+
return document_factory(2, 10, "2023-02-17T15:00:13Z")
45+
46+
@pytest.fixture
47+
def document_210_newer(document_factory):
48+
return document_factory(2, 10, "2023-02-18T15:00:13Z")
49+
50+
@pytest.fixture
51+
def document_30(document_factory):
52+
return document_factory(3, 0, "2022-02-18T15:00:13Z")
53+
54+
@pytest.fixture
55+
def document_310(document_factory):
56+
return document_factory(3, 10, "2022-02-18T15:00:13Z")
57+
58+
@pytest.fixture
59+
def document_31(document_factory):
60+
return document_factory(3, 1, "2022-02-18T15:00:13Z")
61+
62+
def test_get_latest_version(project, document_15, document_29, document_29_newer, document_210, document_210_newer, document_30, document_31, document_310):
63+
assert project._get_latest_version([]) == (0, -1, None)
64+
assert project._get_latest_version([document_29]) == (2, 9, document_29)
65+
assert project._get_latest_version([document_29_newer, document_29]) == (2, 9, document_29_newer)
66+
assert project._get_latest_version([document_29, document_29_newer]) == (2, 9, document_29_newer)
67+
assert project._get_latest_version([document_29, document_210]) == (2, 10, document_210)
68+
assert project._get_latest_version([document_29_newer, document_210]) == (2, 10, document_210)
69+
assert project._get_latest_version([document_29, document_210_newer]) == (2, 10, document_210_newer)
70+
assert project._get_latest_version([document_29_newer, document_210_newer]) == (2, 10, document_210_newer)
71+
assert project._get_latest_version([document_29, document_29_newer, document_210]) == (2, 10, document_210)
72+
assert project._get_latest_version([document_29, document_210, document_210_newer]) == (2, 10, document_210_newer)
73+
assert project._get_latest_version([document_29, document_29_newer, document_210, document_210_newer]) == (2, 10, document_210_newer)
74+
assert project._get_latest_version([document_210, document_210_newer, document_29_newer, document_29]) == (2, 10, document_210_newer)
75+
assert project._get_latest_version([document_210, document_15, document_210_newer, document_29_newer, document_29]) == (2, 10, document_210_newer)
76+
assert project._get_latest_version([document_210, document_210_newer, document_29_newer, document_30, document_29]) == (3, 0, document_30)
77+
assert project._get_latest_version([document_210, document_15, document_210_newer, document_29_newer, document_30, document_29]) == (3, 0, document_30)
78+
assert project._get_latest_version([document_15, document_30, document_29]) == (3, 0, document_30)
79+
assert project._get_latest_version([document_31, document_310, document_30]) == (3, 10, document_310)
1680

1781
def test_get_user_email_with_private_email(user):
1882
""" Test user with a single private email instance """
@@ -28,5 +92,3 @@ def test_get_user_email(user):
2892
""" Test getting user email with valid email """
2993
user.model.user_emails = set(["[email protected]"])
3094
assert utils.get_public_email(user) == "[email protected]"
31-
32-

‎utils/get_all_current_documents_sf.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/bash
2+
# STAGE=prod
3+
if [ -z "${STAGE}" ]
4+
then
5+
export STAGE=dev
6+
fi
7+
if [ "${STAGE}" = "prod" ]
8+
then
9+
export TABLE="fivetran_ingest.dynamodb_product_us_east_1.cla_prod_projects"
10+
else
11+
export TABLE="fivetran_ingest.dynamodb_product_us_east1_dev.cla_dev_projects"
12+
fi
13+
14+
declare -A template_projects
15+
declare -A projects
16+
17+
JQ='[.[] | select((.document_major_version != null) and (.document_minor_version != null) and (.document_creation_date != null))] | sort_by((.document_major_version | tonumber), (.document_minor_version | tonumber), .document_creation_date) | .[-1].document_s3_url'
18+
19+
data=$(snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select distinct object_construct('project_id', project_id, 'project_name', data:project_name) from ${TABLE} order by 1 limit 10000" | jq -s -r '.')
20+
# echo $data
21+
while read -r project_id project_name
22+
do
23+
template=$(snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select data:project_individual_documents from ${TABLE} where project_id = '${project_id}'" | jq -r "${JQ}")
24+
if ( [ -z "${template}" ] || [ "${template}" = "null" ] || [ -z "${project_name}" ] )
25+
then
26+
continue
27+
fi
28+
echo "ICLA ${project_id} - ${project_name}"
29+
if [ -n "${template_projects[$template]}" ]
30+
then
31+
template_projects["$template"]+=";${project_id}"
32+
else
33+
template_projects["$template"]="${project_id}"
34+
fi
35+
projects["$project_id"]="${project_name}"
36+
done < <(echo "${data}" | jq -r -c '.[] | select(.project_id != null and .project_name != null) | "\(.project_id) \(.project_name)"')
37+
38+
while read -r project_id project_name
39+
do
40+
template=$(snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select data:project_corporate_documents from ${TABLE} where project_id = '${project_id}'" | jq -r "${JQ}")
41+
if ( [ -z "${template}" ] || [ "${template}" = "null" ] || [ -z "${project_name}" ] )
42+
then
43+
continue
44+
fi
45+
echo "CCLA ${project_id} - ${project_name}"
46+
if [ -n "${template_projects[$template]}" ]
47+
then
48+
template_projects["$template"]+=";${project_id}"
49+
else
50+
template_projects["$template"]="${project_id}"
51+
fi
52+
projects["$project_id"]="${project_name}"
53+
done < <(echo "${data}" | jq -r -c '.[] | select(.project_id != null and .project_name != null) | "\(.project_id) \(.project_name)"')
54+
55+
# for template in "${!template_projects[@]}"
56+
for template in $(printf "%s\n" "${!template_projects[@]}" | sort)
57+
do
58+
projs="${template_projects[$template]}"
59+
names=""
60+
IFS=';' read -ra idary <<< "${projs}"
61+
for id in "${idary[@]}"; do
62+
if [ -z "${names}" ]
63+
then
64+
names="${projects[$id]} (${id})"
65+
else
66+
names+=";${projects[$id]} (${id})"
67+
fi
68+
done
69+
IFS=';' read -ra ary <<< "${names}"
70+
sorted_projects=$(printf "%s\n" "${ary[@]}" | sort | uniq | paste -sd ',' | sed 's/),/), /g')
71+
echo "Template: ${template} used by project(s): ${sorted_projects}"
72+
done

‎utils/get_projects_corporate_documents_sf.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
# STAGE=prod
3+
# JQ='.[] | {document_major_version, document_minor_version, document_file_id, document_name, document_creation_date, document_s3_url}'
4+
# JQ='sort_by(.document_major_version, .document_minor_version, .document_creation_date) | .[-1]'
5+
# JQ='sort_by(.document_major_version, .document_minor_version, .document_creation_date) | .[-1] | {document_major_version, document_minor_version, document_file_id, document_name, document_creation_date, document_s3_url}'
6+
# d8cead54-92b7-48c5-a2c8-b1e295e8f7f1 - prod CNCF project ID
7+
if [ -z "$1" ]
8+
then
9+
echo "$0: you need to specify project_id as a 1st argument, example: 'd8cead54-92b7-48c5-a2c8-b1e295e8f7f1'"
10+
exit 1
11+
fi
12+
if [ -z "${STAGE}" ]
13+
then
14+
export STAGE=dev
15+
fi
16+
if [ "${STAGE}" = "prod" ]
17+
then
18+
export TABLE="fivetran_ingest.dynamodb_product_us_east_1.cla_prod_projects"
19+
else
20+
export TABLE="fivetran_ingest.dynamodb_product_us_east1_dev.cla_dev_projects"
21+
fi
22+
if [ -z "${JQ}" ]
23+
then
24+
snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select data:project_corporate_documents from ${TABLE} where project_id = '${1}'" | jq -r '.'
25+
else
26+
snowsql $(cat ./snowflake.secret) -o friendly=false -o header=false -o timing=false -o output_format=plain -q "select data:project_corporate_documents from ${TABLE} where project_id = '${1}'" | jq -r "${JQ}"
27+
fi

0 commit comments

Comments
 (0)