GODRIVER-3622 Automatically retry some test tasks. (#2147)

Author: matthewdale

.evergreen/config.yml

@@ -195,7 +195,6 @@ functions:
 
   run-tests:
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         env:
@@ -204,6 +203,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, "${DEFAULT_TASK}" ]
@@ -300,7 +300,6 @@ functions:
       params:
         role_arn: "${aws_test_secrets_role}"
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
@@ -309,6 +308,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, --silent, evg-test-enterprise-auth]
@@ -318,7 +318,6 @@ functions:
       params:
         role_arn: "${aws_test_secrets_role}"
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
@@ -327,6 +326,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, --silent, evg-test-enterprise-auth]
@@ -336,7 +336,6 @@ functions:
       params:
         role_arn: "${aws_test_secrets_role}"
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
@@ -345,13 +344,13 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, test-atlas-connect]
 
   run-ocsp-test:
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         env: 
@@ -362,14 +361,14 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         include_expansions_in_env: [OCSP_TLS_SHOULD_SUCCEED]
         args: [*task-runner, evg-test-ocsp]
 
   run-versioned-api-test:
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         env: 
@@ -379,13 +378,13 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, evg-test-versioned-api]
 
   run-load-balancer-tests:
     - command: subprocess.exec
-      type: test
       params:
         binary: bash
         include_expansions_in_env: [SINGLE_MONGOS_LB_URI, MULTI_MONGOS_LB_URI, AUTH, SSL, MONGO_GO_DRIVER_COMPRESSOR]
@@ -394,13 +393,13 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: bash
         args: [*task-runner, evg-test-load-balancers]
 
   run-atlas-data-lake-test:
     - command: subprocess.exec
-      type: test
       params:
         binary: "bash"
         env: 
@@ -411,6 +410,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: "bash"
         args: [*task-runner, evg-test-atlas-data-lake]
@@ -582,7 +582,6 @@ functions:
 
   run-kms-tls-test:
     - command: subprocess.exec
-      type: test
       params:
         binary: "bash"
         env: 
@@ -592,14 +591,14 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: "bash"
         include_expansions_in_env: [KMS_TLS_TESTCASE]
         args: [*task-runner, evg-test-kms]
 
   run-kmip-tests:
     - command: subprocess.exec
-      type: test
       params:
         binary: "bash"
         env: 
@@ -609,6 +608,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: "bash"
         env:
@@ -617,7 +617,6 @@ functions:
 
   run-retry-kms-requests:
     - command: subprocess.exec
-      type: test
       params:
         binary: "bash"
         env: 
@@ -627,6 +626,7 @@ functions:
         args: [*task-runner, setup-test]
     - command: subprocess.exec
       type: test
+      retry_on_failure: true
       params:
         binary: "bash"
         env: