Skip to content

Commit 027d0aa

Browse files
ryansteakleyryansteakley
authored
Add unit tests for processing job (#115)
Description of changes: - `Sdk.go` coverage 72.0% - Custom code hook coverage 100% By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent ab3b492 commit 027d0aa

27 files changed

+1927
-21
lines changed

‎pkg/resource/processing_job/manager_test_suite_test.go

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ import (
1717
"errors"
1818
"fmt"
1919

20+
"path/filepath"
21+
"testing"
22+
2023
ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
2124
ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics"
2225
acktypes "github.com/aws-controllers-k8s/runtime/pkg/types"
@@ -27,9 +30,7 @@ import (
2730
"github.com/google/go-cmp/cmp"
2831
"github.com/google/go-cmp/cmp/cmpopts"
2932
"go.uber.org/zap/zapcore"
30-
"path/filepath"
3133
ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
32-
"testing"
3334
)
3435

3536
// provideResourceManagerWithMockSDKAPI accepts MockSageMakerAPI and returns pointer to resourceManager
@@ -102,13 +103,25 @@ func (d *testRunnerDelegate) Equal(a acktypes.AWSResource, b acktypes.AWSResourc
102103
// Ignore LastTransitionTime since it gets updated each run.
103104
opts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(ackv1alpha1.Condition{}, "LastTransitionTime")}
104105

106+
var specMatch = false
107+
if cmp.Equal(ac.ko.Spec, bc.ko.Spec, opts...) {
108+
specMatch = true
109+
} else {
110+
fmt.Printf("Difference ko.Spec (-expected +actual):\n\n")
111+
fmt.Println(cmp.Diff(ac.ko.Spec, bc.ko.Spec, opts...))
112+
specMatch = false
113+
}
114+
115+
var statusMatch = false
105116
if cmp.Equal(ac.ko.Status, bc.ko.Status, opts...) {
106-
return true
117+
statusMatch = true
107118
} else {
108-
fmt.Printf("Difference (-expected +actual):\n\n")
119+
fmt.Printf("Difference ko.Status (-expected +actual):\n\n")
109120
fmt.Println(cmp.Diff(ac.ko.Status, bc.ko.Status, opts...))
110-
return false
121+
statusMatch = false
111122
}
123+
124+
return statusMatch && specMatch
112125
}
113126

114127
// Checks to see if the given yaml file, with name stored as expectation,

‎pkg/resource/processing_job/testdata/sdkapi/create/success_create.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123456789012:processing-job/unit-testing-processing-job"
3+
}

‎pkg/resource/processing_job/testdata/sdkapi/describe/stopping_describe.json

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
{
2+
"AppSpecification": {
3+
"ContainerArguments": null,
4+
"ContainerEntrypoint": [
5+
"python",
6+
"/opt/ml/processing/code/kmeans_preprocessing.py"
7+
],
8+
"ImageUri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.5.0-cpu-py36-ubuntu16.04/pytorch-training:1.5.0-cpu-py36-ubuntu16.04"
9+
},
10+
"AutoMLJobArn": null,
11+
"CreationTime": "2021-09-29T05:51:48.716Z",
12+
"Environment": {
13+
"my_var": "my_value",
14+
"my_var2": "my_value2"
15+
},
16+
"ExitMessage": null,
17+
"ExperimentConfig": null,
18+
"FailureReason": null,
19+
"LastModifiedTime": "2021-09-29T05:52:26.041Z",
20+
"MonitoringScheduleArn": null,
21+
"NetworkConfig": null,
22+
"ProcessingEndTime": null,
23+
"ProcessingInputs": [
24+
{
25+
"AppManaged": false,
26+
"DatasetDefinition": null,
27+
"InputName": "mnist_tar",
28+
"S3Input": {
29+
"LocalPath": "/opt/ml/processing/input",
30+
"S3CompressionType": "None",
31+
"S3DataDistributionType": "FullyReplicated",
32+
"S3DataType": "S3Prefix",
33+
"S3InputMode": "File",
34+
"S3Uri": "s3://sagemaker-sample-data-us-west-2/algorithms/kmeans/mnist/mnist.pkl.gz"
35+
}
36+
},
37+
{
38+
"AppManaged": false,
39+
"DatasetDefinition": null,
40+
"InputName": "source_code",
41+
"S3Input": {
42+
"LocalPath": "/opt/ml/processing/code",
43+
"S3CompressionType": "None",
44+
"S3DataDistributionType": "FullyReplicated",
45+
"S3DataType": "S3Prefix",
46+
"S3InputMode": "File",
47+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/kmeans_preprocessing.py"
48+
}
49+
}
50+
],
51+
"ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123456789021:processing-job/unit-testing-processing-jobss",
52+
"ProcessingJobName": "unit-testing-processing-jobs",
53+
"ProcessingJobStatus": "Stopping",
54+
"ProcessingOutputConfig": {
55+
"KmsKeyId": null,
56+
"Outputs": [
57+
{
58+
"AppManaged": false,
59+
"FeatureStoreOutput": null,
60+
"OutputName": "train_data",
61+
"S3Output": {
62+
"LocalPath": "/opt/ml/processing/output_train/",
63+
"S3UploadMode": "EndOfJob",
64+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
65+
}
66+
},
67+
{
68+
"AppManaged": false,
69+
"FeatureStoreOutput": null,
70+
"OutputName": "test_data",
71+
"S3Output": {
72+
"LocalPath": "/opt/ml/processing/output_test/",
73+
"S3UploadMode": "EndOfJob",
74+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
75+
}
76+
},
77+
{
78+
"AppManaged": false,
79+
"FeatureStoreOutput": null,
80+
"OutputName": "valid_data",
81+
"S3Output": {
82+
"LocalPath": "/opt/ml/processing/output_valid/",
83+
"S3UploadMode": "EndOfJob",
84+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
85+
}
86+
}
87+
]
88+
},
89+
"ProcessingResources": {
90+
"ClusterConfig": {
91+
"InstanceCount": 1,
92+
"InstanceType": "ml.m5.large",
93+
"VolumeKmsKeyId": null,
94+
"VolumeSizeInGB": 20
95+
}
96+
},
97+
"ProcessingStartTime": null,
98+
"RoleArn": "arn:aws:iam::123456789012:role/service-role/AmazonSageMaker-ExecutionRole-20210920T111639",
99+
"StoppingCondition": {
100+
"MaxRuntimeInSeconds": 86400
101+
},
102+
"TrainingJobArn": null
103+
}

‎pkg/resource/processing_job/testdata/sdkapi/describe/success_completed_describe.json

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
{
2+
"AppSpecification": {
3+
"ContainerArguments": null,
4+
"ContainerEntrypoint": [
5+
"python",
6+
"/opt/ml/processing/code/kmeans_preprocessing.py"
7+
],
8+
"ImageUri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.5.0-cpu-py36-ubuntu16.04/pytorch-training:1.5.0-cpu-py36-ubuntu16.04"
9+
},
10+
"AutoMLJobArn": null,
11+
"CreationTime": "2021-09-29T04:51:45.539Z",
12+
"Environment": {
13+
"my_var": "my_value",
14+
"my_var2": "my_value2"
15+
},
16+
"ExitMessage": null,
17+
"ExperimentConfig": null,
18+
"FailureReason": null,
19+
"LastModifiedTime": "2021-09-29T04:51:45.784Z",
20+
"MonitoringScheduleArn": null,
21+
"NetworkConfig": null,
22+
"ProcessingEndTime": null,
23+
"ProcessingInputs": [
24+
{
25+
"AppManaged": false,
26+
"DatasetDefinition": null,
27+
"InputName": "mnist_tar",
28+
"S3Input": {
29+
"LocalPath": "/opt/ml/processing/input",
30+
"S3CompressionType": "None",
31+
"S3DataDistributionType": "FullyReplicated",
32+
"S3DataType": "S3Prefix",
33+
"S3InputMode": "File",
34+
"S3Uri": "s3://sagemaker-sample-data-us-west-2/algorithms/kmeans/mnist/mnist.pkl.gz"
35+
}
36+
},
37+
{
38+
"AppManaged": false,
39+
"DatasetDefinition": null,
40+
"InputName": "source_code",
41+
"S3Input": {
42+
"LocalPath": "/opt/ml/processing/code",
43+
"S3CompressionType": "None",
44+
"S3DataDistributionType": "FullyReplicated",
45+
"S3DataType": "S3Prefix",
46+
"S3InputMode": "File",
47+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/kmeans_preprocessing.py"
48+
}
49+
}
50+
],
51+
"ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123456789012:processing-job/unit-testing-processing-job",
52+
"ProcessingJobName": "unit-testing-processing-job",
53+
"ProcessingJobStatus": "Completed",
54+
"ProcessingOutputConfig": {
55+
"KmsKeyId": null,
56+
"Outputs": [
57+
{
58+
"AppManaged": false,
59+
"FeatureStoreOutput": null,
60+
"OutputName": "train_data",
61+
"S3Output": {
62+
"LocalPath": "/opt/ml/processing/output_train/",
63+
"S3UploadMode": "EndOfJob",
64+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
65+
}
66+
},
67+
{
68+
"AppManaged": false,
69+
"FeatureStoreOutput": null,
70+
"OutputName": "test_data",
71+
"S3Output": {
72+
"LocalPath": "/opt/ml/processing/output_test/",
73+
"S3UploadMode": "EndOfJob",
74+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
75+
}
76+
},
77+
{
78+
"AppManaged": false,
79+
"FeatureStoreOutput": null,
80+
"OutputName": "valid_data",
81+
"S3Output": {
82+
"LocalPath": "/opt/ml/processing/output_valid/",
83+
"S3UploadMode": "EndOfJob",
84+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
85+
}
86+
}
87+
]
88+
},
89+
"ProcessingResources": {
90+
"ClusterConfig": {
91+
"InstanceCount": 1,
92+
"InstanceType": "ml.m5.large",
93+
"VolumeKmsKeyId": null,
94+
"VolumeSizeInGB": 20
95+
}
96+
},
97+
"ProcessingStartTime": null,
98+
"RoleArn": "arn:aws:iam::123456789012:role/service-role/AmazonSageMaker-ExecutionRole-20210920T111639",
99+
"StoppingCondition": {
100+
"MaxRuntimeInSeconds": 86400
101+
},
102+
"TrainingJobArn": null
103+
}

‎pkg/resource/processing_job/testdata/sdkapi/describe/success_describe.json

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
{
2+
"AppSpecification": {
3+
"ContainerArguments": null,
4+
"ContainerEntrypoint": [
5+
"python",
6+
"/opt/ml/processing/code/kmeans_preprocessing.py"
7+
],
8+
"ImageUri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.5.0-cpu-py36-ubuntu16.04/pytorch-training:1.5.0-cpu-py36-ubuntu16.04"
9+
},
10+
"AutoMLJobArn": null,
11+
"CreationTime": "2021-09-29T04:51:45.539Z",
12+
"Environment": {
13+
"my_var": "my_value",
14+
"my_var2": "my_value2"
15+
},
16+
"ExitMessage": null,
17+
"ExperimentConfig": null,
18+
"FailureReason": null,
19+
"LastModifiedTime": "2021-09-29T04:51:45.784Z",
20+
"MonitoringScheduleArn": null,
21+
"NetworkConfig": null,
22+
"ProcessingEndTime": null,
23+
"ProcessingInputs": [
24+
{
25+
"AppManaged": false,
26+
"DatasetDefinition": null,
27+
"InputName": "mnist_tar",
28+
"S3Input": {
29+
"LocalPath": "/opt/ml/processing/input",
30+
"S3CompressionType": "None",
31+
"S3DataDistributionType": "FullyReplicated",
32+
"S3DataType": "S3Prefix",
33+
"S3InputMode": "File",
34+
"S3Uri": "s3://sagemaker-sample-data-us-west-2/algorithms/kmeans/mnist/mnist.pkl.gz"
35+
}
36+
},
37+
{
38+
"AppManaged": false,
39+
"DatasetDefinition": null,
40+
"InputName": "source_code",
41+
"S3Input": {
42+
"LocalPath": "/opt/ml/processing/code",
43+
"S3CompressionType": "None",
44+
"S3DataDistributionType": "FullyReplicated",
45+
"S3DataType": "S3Prefix",
46+
"S3InputMode": "File",
47+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/kmeans_preprocessing.py"
48+
}
49+
}
50+
],
51+
"ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123456789012:processing-job/unit-testing-processing-job",
52+
"ProcessingJobName": "unit-testing-processing-job",
53+
"ProcessingJobStatus": "InProgress",
54+
"ProcessingOutputConfig": {
55+
"KmsKeyId": null,
56+
"Outputs": [
57+
{
58+
"AppManaged": false,
59+
"FeatureStoreOutput": null,
60+
"OutputName": "train_data",
61+
"S3Output": {
62+
"LocalPath": "/opt/ml/processing/output_train/",
63+
"S3UploadMode": "EndOfJob",
64+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
65+
}
66+
},
67+
{
68+
"AppManaged": false,
69+
"FeatureStoreOutput": null,
70+
"OutputName": "test_data",
71+
"S3Output": {
72+
"LocalPath": "/opt/ml/processing/output_test/",
73+
"S3UploadMode": "EndOfJob",
74+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
75+
}
76+
},
77+
{
78+
"AppManaged": false,
79+
"FeatureStoreOutput": null,
80+
"OutputName": "valid_data",
81+
"S3Output": {
82+
"LocalPath": "/opt/ml/processing/output_valid/",
83+
"S3UploadMode": "EndOfJob",
84+
"S3Uri": "s3://source-data-bucket-592697580195-us-west-2/sagemaker/processing/output/"
85+
}
86+
}
87+
]
88+
},
89+
"ProcessingResources": {
90+
"ClusterConfig": {
91+
"InstanceCount": 1,
92+
"InstanceType": "ml.m5.large",
93+
"VolumeKmsKeyId": null,
94+
"VolumeSizeInGB": 20
95+
}
96+
},
97+
"ProcessingStartTime": null,
98+
"RoleArn": "arn:aws:iam::123456789012:role/service-role/AmazonSageMaker-ExecutionRole-20210920T111639",
99+
"StoppingCondition": {
100+
"MaxRuntimeInSeconds": 86400
101+
},
102+
"TrainingJobArn": null
103+
}

0 commit comments

Comments
 (0)