Properly daemonize the csfle servers (#570)

Author: blink1073

.evergreen/config.yml

@@ -515,6 +515,20 @@ functions:
     - command: ec2.assume_role
       params:
         role_arn: ${aws_test_secrets_role}
+    # Ensure that we can run setup and teardown commands in the foreground without hanging.
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
+        args: [src/.evergreen/csfle/setup.sh]
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
+        args: [src/.evergreen/csfle/teardown.sh]
+    # Test setup and teardown for each supported version of Python.
     - command: subprocess.exec
       type: test
       params:

.evergreen/csfle/README.md

@@ -29,48 +29,25 @@ The following servers will be started:
 - KMS HTTP server with an expired cert on port 9000
 - KMS HTTP server with an "wrong host" cert on port 9001
 - KMS HTTP server with a correct cert on port 9002
+- KMS Failpoint Server on port 9003
 - Mock Azure IMDS server on port 8080
 
 When finished, stop the servers by running:
 
 ```bash
-$DRIVERS_TOOLS/.evergreen/csfle/stop-servers.sh
+${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
 ```
 
-If you are starting your CSFLE servers in a separate Evergreen function, it is recommended that you setup secrets
-and start the servers in the background, and then have a separate function that uses `await-servers.sh`
-in the foreground to wait for the servers to be ready.  This will ensure the servers are not torn down
-between functions (or the function may stall and not finish because there are processes still running).
-If you are starting the servers in a step within the same function as your tests, you
-can just start the servers directly in a foreground step.
-
-
-
 ```yaml
 start-csfle-servers:
   - command: ec2.assume_role
       params:
       role_arn: ${aws_test_secrets_role}
   - command: subprocess.exec
       params:
-      working_dir: src
-      binary: bash
-      include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
-      args: |
-          ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh
-  - command: subprocess.exec
-      params:
-      working_dir: src
-      binary: bash
-      background: true
-      args:
-          - ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh
-  - command: subprocess.exec
-      params:
-      working_dir: src
       binary: bash
-      args:
-          - ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
+      include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
+      args: [${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh]
 ```
 
 ## Legacy Usage

.evergreen/csfle/start-servers.sh

@@ -30,50 +30,60 @@ done
 . ./activate-kmstlsvenv.sh
 
 # The -u options forces the stdout and stderr streams to be unbuffered.
-# TMPDIR is required to avoid "AF_UNIX path too long" errors.
+COMMAND="python -u"
+if [ "$(uname -s)" != "Darwin" ]; then
+  # On linux and windows host, we need to use nohup to daemonize the process
+  # and prevent the task from hanging.
+  # The macos hosts do not support nohup.
+  COMMAND="nohup $COMMAND"
+fi
+
+
 echo "Starting KMIP Server..."
-TMPDIR="$(dirname "$DRIVERS_TOOLS")" python -u kms_kmip_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file $CSFLE_TLS_CERT_FILE --port 5698 > kms_kmip_server.log 2>&1 &
+# TMPDIR is required to avoid "AF_UNIX path too long" errors.
+TMPDIR="$(dirname "$DRIVERS_TOOLS")" $COMMAND kms_kmip_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file $CSFLE_TLS_CERT_FILE --port 5698 > kms_kmip_server.log 2>&1 &
 echo "$!" > kmip_pids.pid
 sleep 1
 cat kms_kmip_server.log
 echo "Starting KMIP Server...done."
 
 
 echo "Starting HTTP Server 1..."
-python -u kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file ../x509gen/expired.pem --port 9000 > http1.log 2>&1 &
+$COMMAND kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file ../x509gen/expired.pem --port 9000 > http1.log 2>&1 &
 echo "$!" >> kmip_pids.pid
 sleep 1
 cat http1.log
 echo "Starting HTTP Server 1...done."
 
 
 echo "Starting HTTP Server 2..."
-python -u kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file ../x509gen/wrong-host.pem --port 9001 > http2.log 2>&1 &
+$COMMAND kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file ../x509gen/wrong-host.pem --port 9001 > http2.log 2>&1 &
 echo "$!" >> kmip_pids.pid
 sleep 1
 cat http2.log
 echo "Starting HTTP Server 2...done."
 
 
 echo "Starting HTTP Server 3..."
-python -u kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file $CSFLE_TLS_CERT_FILE --port 9002 --require_client_cert > http3.log 2>&1 &
+$COMMAND kms_http_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file $CSFLE_TLS_CERT_FILE --port 9002 --require_client_cert > http3.log 2>&1 &
 echo "$!" >> kmip_pids.pid
 sleep 1
 cat http3.log
 echo "Starting HTTP Server 3...done."
 
 
 echo "Starting Failpoint Server..."
-python -u kms_failpoint_server.py --port 9003 > failpoint.log 2>&1 &
+$COMMAND kms_failpoint_server.py --port 9003 > failpoint.log 2>&1 &
 echo "$!" >> kmip_pids.pid
 echo "Starting Failpoint Server...done."
 sleep 1
 
 echo "Starting Fake Azure IMDS..."
-python bottle.py fake_azure:imds > fake_azure.log 2>&1 &
+$COMMAND bottle.py fake_azure:imds > fake_azure.log 2>&1 &
 echo "$!" >> kmip_pids.pid
 sleep 1
 cat fake_azure.log
 echo "Starting Fake Azure IMDS...done."
 
+# Wait for all of the servers to start.
 bash ./await-servers.sh