Skip to content

Commit 806c037

Browse files
zli82016zli82016
authored
tgc-revival: modify the testing logic by testing each test step (#14863)
1 parent ac9adbe commit 806c037

File tree

3 files changed

+143
-88
lines changed

3 files changed

+143
-88
lines changed

‎mmv1/products/certificatemanager/Certificate.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,28 +66,33 @@ examples:
6666
dns_auth_name2: 'dns-auth2'
6767
dns_auth_subdomain2: 'subdomain2'
6868
cert_name: 'dns-cert'
69+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
6970
- name: 'certificate_manager_google_managed_certificate_issuance_config'
7071
primary_resource_id: 'default'
7172
vars:
7273
cert_name: 'issuance-config-cert'
7374
ca_name: 'ca-authority'
7475
pool_name: 'ca-pool'
7576
issuance_config_name: 'issuance-config'
77+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
7678
- name: 'certificate_manager_self_managed_certificate'
7779
primary_resource_id: 'default'
7880
vars:
7981
cert_name: 'self-managed-cert'
82+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
8083
- name: 'certificate_manager_self_managed_certificate_regional'
8184
primary_resource_id: 'default'
8285
vars:
8386
cert_name: 'self-managed-cert'
87+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
8488
- name: 'certificate_manager_google_managed_certificate_issuance_config_all_regions'
8589
primary_resource_id: 'default'
8690
vars:
8791
cert_name: 'issuance-config-cert'
8892
ca_name: 'ca-authority'
8993
pool_name: 'ca-pool'
9094
issuance_config_name: 'issuance-config'
95+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
9196
- name: 'certificate_manager_google_managed_certificate_dns_all_regions'
9297
primary_resource_id: 'default'
9398
vars:
@@ -96,16 +101,19 @@ examples:
96101
dns_auth_name2: 'dns-auth2'
97102
dns_auth_subdomain2: 'subdomain2'
98103
cert_name: 'dns-cert'
104+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
99105
- name: 'certificate_manager_google_managed_regional_certificate_dns_auth'
100106
primary_resource_id: 'default'
101107
vars:
102108
dns_auth_name: 'dns-auth'
103109
dns_auth_subdomain: 'subdomain'
104110
cert_name: 'dns-cert'
111+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
105112
- name: 'certificate_manager_client_auth_certificate'
106113
primary_resource_id: 'default'
107114
vars:
108115
cert_name: 'client-auth-cert'
116+
tgc_skip_test: 'Need to figure out to get the desired cai asset state.'
109117
parameters:
110118
- name: 'name'
111119
type: String

‎mmv1/third_party/tgc_next/test/assert_test_files.go‎

Lines changed: 67 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -34,59 +34,77 @@ var (
3434
)
3535

3636
func BidirectionalConversion(t *testing.T, ignoredFields []string, ignoredAssetFields []string) {
37-
retries := 0
38-
flakyAction := func(ctx context.Context) error {
39-
log.Printf("Starting the retry %d", retries)
40-
resourceTestData, primaryResource, err := prepareTestData(t.Name(), retries)
41-
retries++
42-
if err != nil {
43-
return fmt.Errorf("error preparing the input data: %v", err)
44-
}
37+
testName := t.Name()
38+
stepNumbers, err := getStepNumbers(testName)
39+
if err != nil {
40+
t.Fatalf("error preparing the input data: %v", err)
41+
}
4542

46-
if resourceTestData == nil {
47-
return retry.RetryableError(fmt.Errorf("fail: test data is unavailable"))
48-
}
43+
if len(stepNumbers) == 0 {
44+
t.Skipf("test steps are unavailable")
45+
}
4946

50-
// Create a temporary directory for running terraform.
51-
tfDir, err := os.MkdirTemp(tmpDir, "terraform")
52-
if err != nil {
53-
return err
54-
}
55-
defer os.RemoveAll(tfDir)
47+
// Create a temporary directory for running terraform.
48+
tfDir, err := os.MkdirTemp(tmpDir, "terraform")
49+
if err != nil {
50+
t.Fatalf("error creating a temporary directory for running terraform: %v", err)
51+
}
52+
defer os.RemoveAll(tfDir)
5653

57-
logger := zaptest.NewLogger(t)
54+
logger := zaptest.NewLogger(t)
5855

59-
// If the primary resource is specified, only test the primary resource.
60-
// Otherwise, test all of the resources in the test.
61-
if primaryResource != "" {
62-
t.Logf("Test for the primary resource %s begins.", primaryResource)
63-
err = testSingleResource(t, t.Name(), resourceTestData[primaryResource], tfDir, ignoredFields, ignoredAssetFields, logger, true)
64-
if err != nil {
65-
return err
66-
}
67-
} else {
68-
for _, testData := range resourceTestData {
69-
err = testSingleResource(t, t.Name(), testData, tfDir, ignoredFields, ignoredAssetFields, logger, false)
56+
for _, stepN := range stepNumbers {
57+
subtestName := fmt.Sprintf("step%d", stepN)
58+
t.Run(subtestName, func(t *testing.T) {
59+
retries := 0
60+
flakyAction := func(ctx context.Context) error {
61+
testData, err := prepareTestData(testName, stepN, retries)
62+
retries++
63+
log.Printf("Starting the attempt %d", retries)
7064
if err != nil {
71-
return err
65+
return fmt.Errorf("error preparing the input data: %v", err)
7266
}
73-
}
74-
}
7567

76-
return nil
77-
}
68+
if testData == nil {
69+
return retry.RetryableError(fmt.Errorf("fail: test data is unavailable"))
70+
}
71+
72+
// If the primary resource is specified, only test the primary resource.
73+
// Otherwise, test all of the resources in the test.
74+
primaryResource := testData.PrimaryResource
75+
resourceTestData := testData.ResourceTestData
76+
tName := fmt.Sprintf("%s_%s", testName, subtestName)
77+
if primaryResource != "" {
78+
t.Logf("Test for the primary resource %s begins.", primaryResource)
79+
err = testSingleResource(t, tName, resourceTestData[primaryResource], tfDir, ignoredFields, ignoredAssetFields, logger, true)
80+
if err != nil {
81+
return err
82+
}
83+
} else {
84+
for _, testData := range resourceTestData {
85+
err = testSingleResource(t, tName, testData, tfDir, ignoredFields, ignoredAssetFields, logger, false)
86+
if err != nil {
87+
return err
88+
}
89+
}
90+
}
7891

79-
// Note maxAttempts-1 is retries, not attempts.
80-
backoffPolicy := retry.WithMaxRetries(maxAttempts-1, retry.NewConstant(50*time.Millisecond))
92+
return nil
93+
}
8194

82-
t.Log("Starting test with retry logic.")
95+
// Note maxAttempts-1 is retries, not attempts.
96+
backoffPolicy := retry.WithMaxRetries(maxAttempts-1, retry.NewConstant(50*time.Millisecond))
8397

84-
if err := retry.Do(context.Background(), backoffPolicy, flakyAction); err != nil {
85-
if strings.Contains(err.Error(), "test data is unavailable") {
86-
t.Skipf("Test skipped because data was unavailable after all retries: %v", err)
87-
} else {
88-
t.Fatalf("Failed after all retries %d: %v", retries, err)
89-
}
98+
t.Log("Starting test with retry logic.")
99+
100+
if err := retry.Do(context.Background(), backoffPolicy, flakyAction); err != nil {
101+
if strings.Contains(err.Error(), "test data is unavailable") {
102+
t.Skipf("Test skipped because data was unavailable after all retries: %v", err)
103+
} else {
104+
t.Fatalf("Failed after all attempts %d: %v", maxAttempts, err)
105+
}
106+
}
107+
})
90108
}
91109
}
92110

@@ -108,7 +126,8 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
108126
assets = append(assets, assetData.CaiAsset)
109127
assetType := assetData.CaiAsset.Type
110128
if assetType == "" {
111-
return fmt.Errorf("cai asset is unavailable for %s", assetName)
129+
log.Printf("cai asset is unavailable for %s", assetName)
130+
return retry.RetryableError(fmt.Errorf("fail: test data is unavailable"))
112131
}
113132
if _, cai2hclSupported = cai2hclconverters.ConverterMap[assetType]; !cai2hclSupported {
114133
log.Printf("%s is not supported in cai2hcl conversion.", assetType)
@@ -129,7 +148,7 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
129148
}
130149

131150
if os.Getenv("WRITE_FILES") != "" {
132-
assetFile := fmt.Sprintf("%s.json", t.Name())
151+
assetFile := fmt.Sprintf("%s.json", testName)
133152
writeJSONFile(assetFile, assets)
134153
}
135154

@@ -144,14 +163,14 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
144163
}
145164

146165
if os.Getenv("WRITE_FILES") != "" {
147-
exportTfFile := fmt.Sprintf("%s_export.tf", t.Name())
166+
exportTfFile := fmt.Sprintf("%s_export.tf", testName)
148167
err = os.WriteFile(exportTfFile, exportConfigData, 0644)
149168
if err != nil {
150169
return fmt.Errorf("error writing file %s", exportTfFile)
151170
}
152171
}
153172

154-
exportTfFilePath := fmt.Sprintf("%s/%s_export.tf", tfDir, t.Name())
173+
exportTfFilePath := fmt.Sprintf("%s/%s_export.tf", tfDir, testName)
155174
err = os.WriteFile(exportTfFilePath, exportConfigData, 0644)
156175
if err != nil {
157176
return fmt.Errorf("error when writing the file %s", exportTfFilePath)
@@ -374,7 +393,7 @@ func getRoundtripConfig(t *testing.T, testName string, tfDir string, ancestryCac
374393
deleteFieldsFromAssets(roundtripAssets, ignoredAssetFields)
375394

376395
if os.Getenv("WRITE_FILES") != "" {
377-
roundtripAssetFile := fmt.Sprintf("%s_roundtrip.json", t.Name())
396+
roundtripAssetFile := fmt.Sprintf("%s_roundtrip.json", testName)
378397
writeJSONFile(roundtripAssetFile, roundtripAssets)
379398
}
380399

‎mmv1/third_party/tgc_next/test/setup.go‎

Lines changed: 68 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,14 @@ type CaiData struct {
2828
}
2929

3030
type NightlyRun struct {
31-
MetadataByTest map[string]TgcMetadataPayload
32-
Date time.Time
31+
MetadataByTestAndStep map[string]map[int]TgcMetadataPayload
32+
Date time.Time
3333
}
3434

35+
// The metadata for each step in one test
3536
type TgcMetadataPayload struct {
3637
TestName string `json:"test_name"`
38+
StepNumber int `json:"step_number"`
3739
RawConfig string `json:"raw_config"`
3840
ResourceMetadata map[string]*ResourceMetadata `json:"resource_metadata"`
3941
PrimaryResource string `json:"primary_resource"`
@@ -44,6 +46,12 @@ type ResourceTestData struct {
4446
ResourceMetadata `json:"resource_metadata"`
4547
}
4648

49+
type StepTestData struct {
50+
StepNumber int
51+
PrimaryResource string
52+
ResourceTestData map[string]ResourceTestData // key is resource address
53+
}
54+
4755
type Resource struct {
4856
Type string `json:"type"`
4957
Name string `json:"name"`
@@ -62,7 +70,8 @@ var (
6270

6371
func ReadTestsDataFromGcs() ([]NightlyRun, error) {
6472
if !setupDone {
65-
bucketName := "cai_assets_metadata"
73+
// bucketName := "cai_assets_metadata"
74+
bucketName := "cai_assets" // Use the bucket in testing project for tansition
6675
currentDate := time.Now()
6776
ctx := context.Background()
6877
client, err := storage.NewClient(ctx)
@@ -94,8 +103,8 @@ func ReadTestsDataFromGcs() ([]NightlyRun, error) {
94103
}
95104
} else {
96105
TestsMetadata[i] = NightlyRun{
97-
MetadataByTest: metadata,
98-
Date: currentDate,
106+
MetadataByTestAndStep: metadata,
107+
Date: currentDate,
99108
}
100109
}
101110
currentDate = currentDate.AddDate(0, 0, -1)
@@ -113,8 +122,8 @@ func ReadTestsDataFromGcs() ([]NightlyRun, error) {
113122
return TestsMetadata, nil
114123
}
115124

116-
func readTestsDataFromGCSForRun(ctx context.Context, currentDate time.Time, bucketName string, bucket *storage.BucketHandle) (map[string]TgcMetadataPayload, error) {
117-
metadata := make(map[string]TgcMetadataPayload)
125+
func readTestsDataFromGCSForRun(ctx context.Context, currentDate time.Time, bucketName string, bucket *storage.BucketHandle) (map[string]map[int]TgcMetadataPayload, error) {
126+
metadata := make(map[string]map[int]TgcMetadataPayload)
118127
objectName := fmt.Sprintf("nightly_tests/%s/nightly_tests_meta.json", currentDate.Format(ymdFormat))
119128
log.Printf("Read object %s from the bucket %s", objectName, bucketName)
120129

@@ -142,61 +151,80 @@ func readTestsDataFromGCSForRun(ctx context.Context, currentDate time.Time, buck
142151
return metadata, nil
143152
}
144153

145-
func prepareTestData(testName string, retries int) (map[string]ResourceTestData, string, error) {
154+
func getStepNumbers(testName string) ([]int, error) {
146155
var err error
147156
cacheMutex.Lock()
148157
defer cacheMutex.Unlock()
149158
TestsMetadata, err = ReadTestsDataFromGcs()
150159
if err != nil {
151-
return nil, "", err
160+
return nil, err
161+
}
162+
163+
stepNumbers := make([]int, 0)
164+
for _, run := range TestsMetadata {
165+
testMetadata, ok := run.MetadataByTestAndStep[testName]
166+
if ok && len(testMetadata) > 0 {
167+
for stepNumber := range testMetadata {
168+
stepNumbers = append(stepNumbers, stepNumber)
169+
}
170+
break
171+
}
152172
}
173+
return stepNumbers, nil
174+
}
175+
176+
func prepareTestData(testName string, stepNumber int, retries int) (*StepTestData, error) {
177+
var err error
153178

154-
var testMetadata TgcMetadataPayload
155-
var resourceMetadata map[string]*ResourceMetadata
179+
var testMetadata map[int]TgcMetadataPayload
156180

157181
run := TestsMetadata[retries]
158-
testMetadata, ok := run.MetadataByTest[testName]
182+
testMetadata, ok := run.MetadataByTestAndStep[testName]
159183
if !ok {
160184
log.Printf("Data of test is unavailable: %s", testName)
161-
return nil, "", nil
162-
}
163-
resourceMetadata = testMetadata.ResourceMetadata
164-
if len(resourceMetadata) == 0 {
165-
log.Printf("Data of resource is unavailable: %s", testName)
166-
return nil, "", nil
185+
return nil, nil
167186
}
168187

169188
log.Printf("Found metadata for %s from run on %s", testName, run.Date.Format(ymdFormat))
170189

171-
rawTfFile := fmt.Sprintf("%s.tf", testName)
172-
err = os.WriteFile(rawTfFile, []byte(testMetadata.RawConfig), 0644)
173-
if err != nil {
174-
return nil, "", fmt.Errorf("error writing to file %s: %#v", rawTfFile, err)
175-
}
176-
if os.Getenv("WRITE_FILES") == "" {
177-
defer os.Remove(rawTfFile)
178-
}
190+
if stepMetadata, ok := testMetadata[stepNumber]; ok {
191+
resourceMetadata := stepMetadata.ResourceMetadata
179192

180-
rawResourceConfigs, err := parseResourceConfigs(rawTfFile)
181-
if err != nil {
182-
return nil, "", fmt.Errorf("error parsing resource configs: %#v", err)
183-
}
193+
rawTfFile := fmt.Sprintf("%s_step%d.tf", testName, stepNumber)
194+
err = os.WriteFile(rawTfFile, []byte(stepMetadata.RawConfig), 0644)
195+
if err != nil {
196+
return nil, fmt.Errorf("error writing to file %s: %#v", rawTfFile, err)
197+
}
198+
if os.Getenv("WRITE_FILES") == "" {
199+
defer os.Remove(rawTfFile)
200+
}
184201

185-
if len(rawResourceConfigs) == 0 {
186-
return nil, "", fmt.Errorf("test %s fails: raw config is unavailable", testName)
187-
}
202+
rawResourceConfigs, err := parseResourceConfigs(rawTfFile)
203+
if err != nil {
204+
return nil, fmt.Errorf("error parsing resource configs: %#v", err)
205+
}
188206

189-
rawConfigMap := convertToConfigMap(rawResourceConfigs)
207+
if len(rawResourceConfigs) == 0 {
208+
return nil, fmt.Errorf("Test %s fails: raw config is unavailable", testName)
209+
}
210+
211+
rawConfigMap := convertToConfigMap(rawResourceConfigs)
190212

191-
resourceTestData := make(map[string]ResourceTestData, 0)
192-
for address, metadata := range resourceMetadata {
193-
resourceTestData[address] = ResourceTestData{
194-
ParsedRawConfig: rawConfigMap[address],
195-
ResourceMetadata: *metadata,
213+
resourceTestData := make(map[string]ResourceTestData, 0)
214+
for address, metadata := range resourceMetadata {
215+
resourceTestData[address] = ResourceTestData{
216+
ParsedRawConfig: rawConfigMap[address],
217+
ResourceMetadata: *metadata,
218+
}
196219
}
220+
return &StepTestData{
221+
StepNumber: stepNumber,
222+
PrimaryResource: stepMetadata.PrimaryResource,
223+
ResourceTestData: resourceTestData,
224+
}, nil
197225
}
198226

199-
return resourceTestData, testMetadata.PrimaryResource, nil
227+
return nil, nil
200228
}
201229

202230
// Parses a Terraform configuation file written with HCL

0 commit comments

Comments
 (0)