Skip to content

Commit f210ea7

Browse files
surajkotasurajkota
authored
Job definition resources unit tests (#110)
Description of changes: Resources covered in this PR: - ModelQualityJobDefinition - Coverage: - total pkg: 60.6% - sdk.go: 73% - manager.go: 53.4% - ModelExplainabilityJobDefinition - Coverage: - total pkg: 61.6% - sdk.go: 76.1% - manager.go: 51.9% - DataQualityJobDefinition - Coverage: - total pkg: 59% - sdk.go: 72.2% - manager.go: 51.9% - ModelBiasJobDefinition - Coverage: - total pkg: 63.6% - sdk.go: 78.2% - manager.go: 51.9% Overall sagemaker controller pkg coverage: 38% Testing: ``` make test ``` By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent ed8bce0 commit f210ea7

File tree

61 files changed

+2571
-108
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+2571
-108
lines changed

‎pkg/resource/data_quality_job_definition/manager_test_suite_test.go

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,25 @@ func (d *testRunnerDelegate) Equal(a acktypes.AWSResource, b acktypes.AWSResourc
108108
// Ignore LastTransitionTime since it gets updated each run.
109109
opts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(ackv1alpha1.Condition{}, "LastTransitionTime")}
110110

111+
var specMatch = false
112+
if cmp.Equal(ac.ko.Spec, bc.ko.Spec, opts...) {
113+
specMatch = true
114+
} else {
115+
fmt.Printf("Difference ko.Spec (-expected +actual):\n\n")
116+
fmt.Println(cmp.Diff(ac.ko.Spec, bc.ko.Spec, opts...))
117+
specMatch = false
118+
}
119+
120+
var statusMatch = false
111121
if cmp.Equal(ac.ko.Status, bc.ko.Status, opts...) {
112-
return true
122+
statusMatch = true
113123
} else {
114-
fmt.Printf("Difference (-expected +actual):\n\n")
124+
fmt.Printf("Difference ko.Status (-expected +actual):\n\n")
115125
fmt.Println(cmp.Diff(ac.ko.Status, bc.ko.Status, opts...))
116-
return false
126+
statusMatch = false
117127
}
128+
129+
return statusMatch && specMatch
118130
}
119131

120132
// Checks to see if the given yaml file, with name stored as expectation,

‎pkg/resource/data_quality_job_definition/testdata/data_quality_job_definition/v1alpha1/dqjd_invalid_before_create.yaml

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎pkg/resource/data_quality_job_definition/testdata/sdkapi/create/success_create.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"JobDefinitionArn": "arn:aws:sagemaker:us-west-2:123456789012:data-quality-job-definition/unit-dq-job-definition"
3+
}

‎pkg/resource/data_quality_job_definition/testdata/sdkapi/describe/success_describe.json

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"CreationTime": "2021-09-14T19:19:45.805Z",
3+
"DataQualityAppSpecification": {
4+
"ContainerArguments": null,
5+
"ContainerEntrypoint": null,
6+
"Environment": null,
7+
"ImageUri": "159807026194.dkr.ecr.us-west-2.amazonaws.com/sagemaker-model-monitor-analyzer",
8+
"PostAnalyticsProcessorSourceUri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/code/postprocessor.py",
9+
"RecordPreprocessorSourceUri": null
10+
},
11+
"DataQualityBaselineConfig": {
12+
"BaseliningJobName": null,
13+
"ConstraintsResource": {
14+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/constraints.json"
15+
},
16+
"StatisticsResource": {
17+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/statistics.json"
18+
}
19+
},
20+
"DataQualityJobInput": {
21+
"EndpointInput": {
22+
"EndTimeOffset": null,
23+
"EndpointName": "xgboost-churn",
24+
"FeaturesAttribute": null,
25+
"InferenceAttribute": null,
26+
"LocalPath": "/opt/ml/processing/input/endpoint",
27+
"ProbabilityAttribute": null,
28+
"ProbabilityThresholdAttribute": null,
29+
"S3DataDistributionType": "FullyReplicated",
30+
"S3InputMode": "File",
31+
"StartTimeOffset": null
32+
}
33+
},
34+
"DataQualityJobOutputConfig": {
35+
"KmsKeyId": null,
36+
"MonitoringOutputs": [
37+
{
38+
"S3Output": {
39+
"LocalPath": "/opt/ml/processing/output",
40+
"S3UploadMode": "Continuous",
41+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/reports"
42+
}
43+
}
44+
]
45+
},
46+
"JobDefinitionArn": "arn:aws:sagemaker:us-west-2:123456789012:data-quality-job-definition/unit-dq-job-definition",
47+
"JobDefinitionName": "unit-dq-job-definition",
48+
"JobResources": {
49+
"ClusterConfig": {
50+
"InstanceCount": 1,
51+
"InstanceType": "ml.m5.large",
52+
"VolumeKmsKeyId": null,
53+
"VolumeSizeInGB": 20
54+
}
55+
},
56+
"NetworkConfig": null,
57+
"RoleArn": "arn:aws:iam::123456789012:role/ack-sagemaker-execution-role",
58+
"StoppingCondition": {
59+
"MaxRuntimeInSeconds": 1800
60+
}
61+
}

‎pkg/resource/data_quality_job_definition/testdata/test_suite.yaml

Lines changed: 116 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,119 @@ tests:
22
- name: "Data quality job definition create tests"
33
description: "Part of data quality job definition CRD tests."
44
scenarios:
5-
- name: "Create=InvalidInput"
6-
description: "Given one of the parameters is invalid, ko.Status shows a terminal condition"
7-
given:
8-
desired_state: "data_quality_job_definition/v1alpha1/dqjd_invalid_before_create.yaml"
9-
svc_api:
10-
- operation: CreateDataQualityJobDefinitionWithContext
11-
error:
12-
code: InvalidParameterValue
13-
message: "The job definition name must not include a special character."
14-
invoke: Create
15-
expect:
16-
latest_state: "data_quality_job_definition/v1alpha1/dqjd_invalid_create_attempted.yaml"
17-
error: resource is in terminal condition
5+
- name: "Create=InvalidInput"
6+
description: "Given one of the parameters is invalid, ko.Status shows a terminal condition"
7+
given:
8+
desired_state: "v1alpha1/create/desired/invalid_before_create.yaml"
9+
svc_api:
10+
- operation: CreateDataQualityJobDefinitionWithContext
11+
error:
12+
code: InvalidParameterValue
13+
message: "The job definition name must not include a special character."
14+
invoke: Create
15+
expect:
16+
latest_state: "v1alpha1/create/observed/invalid_create_attempted.yaml"
17+
error: resource is in terminal condition
18+
- name: "Create=Valid"
19+
description: "Create a new job definition successfully (ARN in status)."
20+
given:
21+
desired_state: "v1alpha1/create/desired/success_before_create.yaml"
22+
svc_api:
23+
- operation: CreateDataQualityJobDefinitionWithContext
24+
output_fixture: "sdkapi/create/success_create.json"
25+
invoke: Create
26+
expect:
27+
latest_state: "v1alpha1/create/observed/success_after_create.yaml"
28+
error: null
29+
- name: "Data Quality job definition readOne tests"
30+
description: "Testing the readOne operation"
31+
scenarios:
32+
- name: "ReadOne=MissingRequiredField"
33+
description: "Testing readOne when required field is missing. No API call is made and returns error."
34+
given:
35+
desired_state: "v1alpha1/readone/desired/missing_required_field.yaml"
36+
invoke: ReadOne
37+
expect:
38+
error: "resource not found"
39+
- name: "ReadOne=NotFound"
40+
description: "Testing readOne when Describe fails to find the resource on SageMaker"
41+
given:
42+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
43+
svc_api:
44+
- operation: DescribeDataQualityJobDefinitionWithContext
45+
error:
46+
code: ResourceNotFound
47+
message: "Could not find job definition"
48+
invoke: ReadOne
49+
expect:
50+
error: "resource not found"
51+
- name: "ReadOne=Fail"
52+
description: "This test checks if the condition is updated if describe fails and readOne returns error"
53+
given:
54+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
55+
svc_api:
56+
- operation: DescribeDataQualityJobDefinitionWithContext
57+
error:
58+
code: ServiceUnavailable
59+
message: "Server is down"
60+
invoke: ReadOne
61+
expect:
62+
latest_state: "v1alpha1/readone/observed/error_on_describe.yaml"
63+
error: "ServiceUnavailable: Server is down\n\tstatus code: 0, request id: "
64+
- name: "ReadOne=AfterCreate"
65+
description: "Testing readOne right after create, there should be no delta."
66+
given:
67+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
68+
svc_api:
69+
- operation: DescribeDataQualityJobDefinitionWithContext
70+
output_fixture: "sdkapi/describe/success_describe.json"
71+
invoke: ReadOne
72+
expect:
73+
latest_state: "v1alpha1/create/observed/success_after_create.yaml"
74+
- name: "ReadOne=SuccessClearsConditions"
75+
description: Testing a successful reconciliation clears conditions if terminal/recoverable condition were already set to true
76+
given:
77+
desired_state: "v1alpha1/readone/desired/error_conditions_true.yaml"
78+
svc_api:
79+
- operation: DescribeDataQualityJobDefinitionWithContext
80+
output_fixture: "sdkapi/describe/success_describe.json"
81+
invoke: ReadOne
82+
expect:
83+
latest_state: "v1alpha1/readone/observed/conditions_clear_on_success.yaml"
84+
- name: "Data Quality job definition update tests"
85+
description: "Testing the update operation"
86+
scenarios:
87+
- name: "Update=NotSupported"
88+
description: "This test checks if the controller throws error for update"
89+
given:
90+
desired_state: "v1alpha1/update/desired/updated_base.yaml"
91+
latest_state: "v1alpha1/create/observed/success_after_create.yaml"
92+
invoke: Update
93+
expect:
94+
latest_state: "v1alpha1/update/observed/error_on_update.yaml"
95+
error: "not implemented"
96+
- name: "Data Quality job definition delete tests"
97+
description: "Testing the delete operation"
98+
scenarios:
99+
- name: "Delete=Fail"
100+
description: "This test checks if the condition is updated if delete fails and returns error"
101+
given:
102+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
103+
svc_api:
104+
- operation: DeleteDataQualityJobDefinitionWithContext
105+
error:
106+
code: ServiceUnavailable
107+
message: "Server is down"
108+
invoke: Delete
109+
expect:
110+
latest_state: "v1alpha1/delete/observed/error_on_delete.yaml"
111+
error: "ServiceUnavailable: Server is down\n\tstatus code: 0, request id: "
112+
- name: "Delete=Successful"
113+
description: "This test checks if the Job definition is deleted successfully"
114+
given:
115+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
116+
svc_api:
117+
- operation: DeleteDataQualityJobDefinitionWithContext
118+
invoke: Delete
119+
expect:
120+
error: nil

‎pkg/resource/data_quality_job_definition/testdata/v1alpha1/create/desired/invalid_before_create.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
apiVersion: sagemaker.services.k8s.aws/v1alpha1
2+
kind: DataQualityJobDefinition
3+
metadata:
4+
creationTimestamp: null
5+
name: unit-dq-job-definition
6+
spec:
7+
dataQualityAppSpecification:
8+
imageURI: 159807026194.dkr.ecr.us-west-2.amazonaws.com/sagemaker-model-monitor-analyzer
9+
postAnalyticsProcessorSourceURI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/code/postprocessor.py
10+
dataQualityBaselineConfig:
11+
constraintsResource:
12+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/constraints.json
13+
statisticsResource:
14+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/statistics.json
15+
dataQualityJobInput:
16+
endpointInput:
17+
endpointName: xgboost-churn
18+
localPath: /opt/ml/processing/input/endpoint
19+
s3DataDistributionType: FullyReplicated
20+
s3InputMode: File
21+
dataQualityJobOutputConfig:
22+
monitoringOutputs:
23+
- s3Output:
24+
localPath: /opt/ml/processing/output
25+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/reports
26+
s3UploadMode: Continuous
27+
jobDefinitionName: intentionally@invalid-name
28+
jobResources:
29+
clusterConfig:
30+
instanceCount: 1
31+
instanceType: ml.m5.large
32+
volumeSizeInGB: 20
33+
roleARN: arn:aws:iam::123456789012:role/ack-sagemaker-execution-role
34+
stoppingCondition:
35+
maxRuntimeInSeconds: 1800
36+
tags:
37+
- key: confidentiality
38+
value: public
39+
- key: environment
40+
value: testing
41+
- key: customer
42+
value: test-user

‎pkg/resource/data_quality_job_definition/testdata/v1alpha1/create/desired/success_before_create.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
apiVersion: sagemaker.services.k8s.aws/v1alpha1
2+
kind: DataQualityJobDefinition
3+
metadata:
4+
creationTimestamp: null
5+
name: unit-dq-job-definition
6+
spec:
7+
dataQualityAppSpecification:
8+
imageURI: 159807026194.dkr.ecr.us-west-2.amazonaws.com/sagemaker-model-monitor-analyzer
9+
postAnalyticsProcessorSourceURI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/code/postprocessor.py
10+
dataQualityBaselineConfig:
11+
constraintsResource:
12+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/constraints.json
13+
statisticsResource:
14+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/statistics.json
15+
dataQualityJobInput:
16+
endpointInput:
17+
endpointName: xgboost-churn
18+
localPath: /opt/ml/processing/input/endpoint
19+
s3DataDistributionType: FullyReplicated
20+
s3InputMode: File
21+
dataQualityJobOutputConfig:
22+
monitoringOutputs:
23+
- s3Output:
24+
localPath: /opt/ml/processing/output
25+
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/reports
26+
s3UploadMode: Continuous
27+
jobDefinitionName: unit-dq-job-definition
28+
jobResources:
29+
clusterConfig:
30+
instanceCount: 1
31+
instanceType: ml.m5.large
32+
volumeSizeInGB: 20
33+
roleARN: arn:aws:iam::123456789012:role/ack-sagemaker-execution-role
34+
stoppingCondition:
35+
maxRuntimeInSeconds: 1800
36+
tags:
37+
- key: confidentiality
38+
value: public
39+
- key: environment
40+
value: testing
41+
- key: customer
42+
value: test-user

‎pkg/resource/data_quality_job_definition/testdata/data_quality_job_definition/v1alpha1/dqjd_invalid_create_attempted.yaml renamed to ‎pkg/resource/data_quality_job_definition/testdata/v1alpha1/create/observed/invalid_create_attempted.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ apiVersion: sagemaker.services.k8s.aws/v1alpha1
22
kind: DataQualityJobDefinition
33
metadata:
44
creationTimestamp: null
5-
name: unit_testing_job_definition
5+
name: unit-dq-job-definition
66
spec:
77
dataQualityAppSpecification:
8-
imageURI: 159807026194.dkr.ecr.us-west-2.amazonaws.com
8+
imageURI: 159807026194.dkr.ecr.us-west-2.amazonaws.com/sagemaker-model-monitor-analyzer
99
postAnalyticsProcessorSourceURI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/code/postprocessor.py
1010
dataQualityBaselineConfig:
1111
constraintsResource:
@@ -14,7 +14,7 @@ spec:
1414
s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/data_quality_job_definition/baselining/statistics.json
1515
dataQualityJobInput:
1616
endpointInput:
17-
endpointName: unit_testing_endpoint
17+
endpointName: xgboost-churn
1818
localPath: /opt/ml/processing/input/endpoint
1919
s3DataDistributionType: FullyReplicated
2020
s3InputMode: File

0 commit comments

Comments
 (0)