diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml index d4820c76..e0ca720a 100644 --- a/.tekton/on-pull-request.yaml +++ b/.tekton/on-pull-request.yaml @@ -10,7 +10,7 @@ metadata: # The branch or tag we are targeting (ie: main, refs/tags/*) pipelinesascode.tekton.dev/on-target-branch: "[main, rh-main]" - # The path we are targeting to build the image + # Paths that trigger the pipeline on change pipelinesascode.tekton.dev/on-path-change: "[src/**, metrics_lib/**, requirements.yaml, pyproject.toml, Dockerfile, docker/**, .dockerignore]" # Fetch the git-clone task from hub, we are able to reference later on it # with taskRef and it will automatically be embedded into our pipeline. @@ -26,6 +26,8 @@ spec: value: "{{ repo_url }}" - name: revision value: "{{ revision }}" + - name: skip-integration-test + value: "false" - name: image-expires-after value: 5d - name: output-image @@ -38,6 +40,9 @@ spec: params: - name: repo_url - name: revision + - name: skip-integration-test + default: "false" + type: string - name: output-image description: Fully Qualified Output Image type: string @@ -146,7 +151,9 @@ spec: value: "ignore" - name: AGENT_MORPHEUS_TRACING_ENABLED value: "false" - image: docker.io/condaforge/miniforge3:latest + - name: LANGCHAIN_TRACING_V2 + value: "false" + image: docker.io/condaforge/miniforge3@sha256:f1f537ff80646e3aaf84e64abf6249cdc3f20889aa0a8d41535e62925c959f64 workingDir: $(workspaces.source.path) volumeMounts: - name: env-file-volume @@ -202,6 +209,60 @@ spec: make test-unit print_banner "LINT AND TEST COMPLETE" + - name: set-image-in-kustomize + runAfter: + - buildah-pvc + when: + - input: "$(params.skip-integration-test)" + operator: notin + values: ["true"] + workspaces: + - name: source + workspace: source + params: + - name: IMAGE_URL + value: $(params.output-image) + - name: KUSTOMIZE_PATH + value: kustomize/overlays/developer + taskSpec: + workspaces: + - name: source + params: + - name: IMAGE_URL + - name: KUSTOMIZE_PATH + steps: + - name: set-image + image: registry.k8s.io/kustomize/kustomize:v5.0.1 + workingDir: $(workspaces.source.path) + script: | + #!/bin/sh + set -ex + # Edit the kustomization file to use the newly built image for the integration test. + # The image name 'agent-morpheus-rh' is derived from the deployment name in the test script. + cd $(params.KUSTOMIZE_PATH) + kustomize edit set image agent-morpheus-rh=$(params.IMAGE_URL) + - name: integration-test + taskRef: + resolver: cluster + params: + - name: kind + value: task + - name: name + value: agent-morpheus-integration-test + - name: namespace + value: ruben-morpheus + runAfter: + - set-image-in-kustomize + workspaces: + - name: source + workspace: source + params: + - name: pr-number + value: "{{ pull_request_number }}" + when: + - input: "$(params.skip-integration-test)" + operator: notin + values: ["true"] workspaces: - name: source volumeClaimTemplate: diff --git a/.tekton/tasks/integration-test.yaml b/.tekton/tasks/integration-test.yaml new file mode 100644 index 00000000..ee1736f9 --- /dev/null +++ b/.tekton/tasks/integration-test.yaml @@ -0,0 +1,67 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: agent-morpheus-integration-test +spec: + description: >- + Runs the end-to-end integration test for the Agent Morpheus application. + This task executes the main test script which handles setup, deployment, + testing, and cleanup. + + params: + - name: pr-number + description: "Pull Request number, used to create a unique temporary namespace." + type: string + - name: verbose + description: "If set to true, enables verbose debug logging in the task script." + type: string + default: "false" + + workspaces: + - name: source + description: "The workspace containing the application's source code." + + stepTemplate: + env: + - name: PR_NUMBER + value: "$(params.pr-number)" + - name: SOURCE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AGENT_MORPHEUS_TRACING_ENABLED + value: "false" + - name: LANGCHAIN_TRACING_V2 + value: "false" + - name: VERBOSE + value: "$(params.verbose)" + + steps: + - name: run-integration-test + image: quay.io/openshift/origin-cli:latest + workingDir: "$(workspaces.source.path)" + script: | + #!/bin/bash + set -euo pipefail + + main() { + readonly TEST_SCRIPT_PATH="e2e/run-integration-test.sh" + + echo "--- [ Running Integration Test Script: ${TEST_SCRIPT_PATH} ] ---" + + if [[ ! -f "${TEST_SCRIPT_PATH}" ]]; then + echo "Error: Test script not found at ${TEST_SCRIPT_PATH}" >&2 + + # Only show debug info if the debug mode is enabled. + if [[ "${VERBOSE:-false}" == "true" ]]; then + echo "Current directory contents (debug mode):" >&2 + ls -lah + fi + exit 1 + fi + + chmod +x "${TEST_SCRIPT_PATH}" + exec "${TEST_SCRIPT_PATH}" "$@" + } + + main "$@" \ No newline at end of file diff --git a/.tekton/tasks/rbac.yaml b/.tekton/tasks/rbac.yaml new file mode 100644 index 00000000..5aecb107 --- /dev/null +++ b/.tekton/tasks/rbac.yaml @@ -0,0 +1,42 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: namespace-manager +rules: +- apiGroups: ["project.openshift.io"] + resources: ["projects", "projectrequests"] + verbs: ["create", "delete", "get", "list"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["create", "delete", "get", "list"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "create", "update", "patch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +- apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pipeline-namespace-manager +subjects: +- kind: ServiceAccount + name: pipeline + namespace: ruben-morpheus +roleRef: + kind: ClusterRole + name: namespace-manager + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/e2e/run-integration-test.sh b/e2e/run-integration-test.sh new file mode 100644 index 00000000..12705be8 --- /dev/null +++ b/e2e/run-integration-test.sh @@ -0,0 +1,326 @@ +#!/bin/bash +# +# ============================================================================= +# E2E Integration Test for Agent Morpheus +# +# Description: +# This script automates the deployment and testing of the application in +# a temporary OpenShift namespace. It is designed to be run from a CI/CD +# pipeline and handles its own setup and cleanup. +# +# Steps: +# 1. Creates a temporary, isolated namespace and waits for it to be ready. +# 2. Copies the image pull secret from the pipeline's namespace. +# 3. Creates dummy .env files required by Kustomize. +# 4. Deploys the application using a Kustomize overlay. +# 5. Waits for the application deployment to become available. +# 6. Runs a readiness check against the server. +# 7. Guarantees the deletion of the namespace upon completion or failure. +# +# Usage: +# PR_NUMBER=123 SOURCE_NAMESPACE=ruben-morpheus ./e2e/run-integration-test.sh +# ============================================================================= + +# Exit immediately if a command exits with a non-zero status. +# Treat unset variables as an error. +set -euo pipefail + +# ============================================================================= +# --- SCRIPT CONSTANTS --- +# All configuration is centralized here. +# ============================================================================= + +# The name of the deployment resource in OpenShift. +readonly APP_DEPLOYMENT_NAME="agent-morpheus-rh" +# The value of the 'component' label used to uniquely identify the server pod. +readonly APP_COMPONENT_LABEL="agent-morpheus-rh" +# The internal port the application server listens on. +readonly APP_PORT="8080" +# The readiness probe endpoint. +readonly HEALTH_ENDPOINT="/ready" +# The name of the image pull secret to be copied. +readonly PULL_SECRET_NAME="morpheus-pull-secret" +# The name of the default service account, used as a signal for namespace readiness. +readonly DEFAULT_SERVICE_ACCOUNT="default" + +# --- Deployment Paths --- +# The path to the kustomize directory for deployment. +readonly KUSTOMIZE_DIR="kustomize/overlays/developer" +# Paths to placeholder files required by Kustomize. +readonly KUSTOMIZE_BASE_SECRETS_ENV_PATH="kustomize/base/secrets.env" +readonly KUSTOMIZE_BASE_OAUTH_SECRETS_ENV_PATH="kustomize/base/oauth-secrets.env" +readonly KUSTOMIZE_DEV_LANGSMITH_SECRET_ENV_PATH="kustomize/overlays/developer/langsmith-secret.env" + +# The name of the conda environment inside the container. +readonly CONDA_ENV_NAME="morpheus-vuln-analysis" + +# --- Test Parameters --- +# How long to wait for the main deployment to become available. +readonly DEPLOYMENT_TIMEOUT="5m" +# How long (in seconds) to wait for a new namespace to be fully initialized. +readonly NAMESPACE_INIT_TIMEOUT_SECONDS=60 +# Number of times to poll the readiness endpoint. +readonly READINESS_PROBE_RETRIES=10 +# Seconds to wait between readiness polls. +readonly READINESS_PROBE_INTERVAL_SECONDS=10 + +# --- Internal Constants --- +# The path inside the pod where server logs will be redirected for debugging. +readonly SERVER_LOG_PATH="/tmp/server.log" + +# ============================================================================= +# --- HELPER FUNCTIONS --- +# ============================================================================= + +# Prints a formatted step header. +# Arguments: +# $1: The title of the step. +function print_step_header() { + echo "" >&2 + echo "--- [ ${1} ] ---" >&2 +} + +# Ensures required environment variables are set. +function check_env_vars() { + if [[ -z "${PR_NUMBER:-}" ]]; then + echo "Error: PR_NUMBER environment variable is not set." >&2 + exit 1 + fi + if [[ -z "${SOURCE_NAMESPACE:-}" ]]; then + echo "Error: SOURCE_NAMESPACE environment variable is not set." >&2 + exit 1 + fi +} + +# Cleans up the temporary namespace on exit. +# Globals: +# NAMESPACE +function cleanup() { + if [[ -n "${NAMESPACE:-}" ]]; then + print_step_header "CLEANUP" + echo "Deleting namespace: ${NAMESPACE}..." >&2 + if ! oc delete project "${NAMESPACE}" --ignore-not-found=true --timeout=5m; then + echo "Warning: Failed to delete namespace ${NAMESPACE}" >&2 + fi + echo "Cleanup complete." >&2 + fi +} + +# ============================================================================= +# --- MAIN LOGIC FUNCTIONS --- +# ============================================================================= + +# Finds and returns the name of the application pod. +# Arguments: +# $1: The namespace to search in. +# Returns: +# The name of the pod. +function get_pod_name() { + local namespace="${1}" + local pod_name + + echo "Finding application pod with label 'component=${APP_COMPONENT_LABEL}'..." >&2 + pod_name=$(oc get pods -n "${namespace}" \ + -l "component=${APP_COMPONENT_LABEL}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [[ -z "${pod_name}" ]]; then + echo "Error: Could not find application pod." >&2 + exit 1 + fi + echo "${pod_name}" +} + +# Creates a new temporary namespace for the test run. +# Arguments: +# $1: The namespace to create. +function setup_namespace() { + local namespace="${1}" + local delete_project_timeout="5m" + print_step_header "SETUP NAMESPACE" + echo "Ensuring a clean project '${namespace}' exists and is ready..." >&2 + + # Delete the project if it exists to avoid side effects from previous runs. + if oc get project "${namespace}" &>/dev/null; then + echo "Project '${namespace}' already exists. Deleting it..." >&2 + if ! oc delete project "${namespace}" --ignore-not-found=true --timeout="${delete_project_timeout}"; then + echo "Warning: Failed to delete namespace ${namespace}" >&2 + fi + fi + + echo "Creating new project: ${namespace}..." >&2 + if ! oc new-project "${namespace}" >/dev/null; then + echo "FATAL: 'oc new-project' failed." >&2 + exit 1 + fi + + # After creation, we are waiting for the namespace to be fully + # initialized before we can use it. + echo "Waiting for namespace to be fully initialized..." >&2 + local counter=0 + while [[ $counter -lt ${NAMESPACE_INIT_TIMEOUT_SECONDS} ]]; do + if oc get sa "${DEFAULT_SERVICE_ACCOUNT}" -n "${namespace}" &>/dev/null; then + echo "Namespace is fully initialized and ready. ✓" >&2 + return 0 + fi + sleep 1; ((counter++)) + done + + echo "Error: Namespace did not become ready within ${NAMESPACE_INIT_TIMEOUT_SECONDS} seconds." >&2 + exit 1 +} + +# TODO, temp, review +# Copies the image pull secret from the source namespace. +# Arguments: +# $1: The target namespace. +function copy_image_pull_secret() { + local target_namespace="${1}" + + echo "Copying image pull secret '${PULL_SECRET_NAME}' from '${SOURCE_NAMESPACE}' to '${target_namespace}'..." >&2 + + # Ensure the source secret exists before attempting to copy it. + if ! oc get secret "${PULL_SECRET_NAME}" -n "${SOURCE_NAMESPACE}" > /dev/null 2>&1; then + echo "Error: Source secret '${PULL_SECRET_NAME}' not found in namespace '${SOURCE_NAMESPACE}'." >&2 + exit 1 + fi + + # Get the secret in YAML format, strip server-managed fields using sed, + # and then apply it to the new namespace. + oc get secret "${PULL_SECRET_NAME}" -n "${SOURCE_NAMESPACE}" -o yaml | \ + sed -e "/namespace:/c\\ namespace: ${target_namespace}" \ + -e '/resourceVersion/d' \ + -e '/uid/d' \ + -e '/creationTimestamp/d' | \ + oc apply -f - + + echo "Image pull secret copied successfully ✓" >&2 +} + +# Creates placeholder .env files required by Kustomize. +function create_kustomize_env_files() { + print_step_header "CREATE KUSTOMIZE PLACEHOLDERS" + + # Create empty secrets.env if it doesn't exist + if [[ ! -f "${KUSTOMIZE_BASE_SECRETS_ENV_PATH}" ]]; then + echo "Creating empty ${KUSTOMIZE_BASE_SECRETS_ENV_PATH} file..." >&2 + touch "${KUSTOMIZE_BASE_SECRETS_ENV_PATH}" + fi + # Create dummy oauth-secrets.env if it doesn't exist + if [[ ! -f "${KUSTOMIZE_BASE_OAUTH_SECRETS_ENV_PATH}" ]]; then + echo "Creating dummy ${KUSTOMIZE_BASE_OAUTH_SECRETS_ENV_PATH} file..." >&2 + { + echo "client-secret=dummy-oauth-client-secret" + echo "openshift-domain=dummy.openshift.com" + } > "${KUSTOMIZE_BASE_OAUTH_SECRETS_ENV_PATH}" + fi + # Create dummy langsmith-secret.env if it doesn't exist + if [[ ! -f "${KUSTOMIZE_DEV_LANGSMITH_SECRET_ENV_PATH}" ]]; then + echo "Creating dummy ${KUSTOMIZE_DEV_LANGSMITH_SECRET_ENV_PATH} file..." >&2 + echo "langchainApiKey=dummy-langsmith-api-key" > "${KUSTOMIZE_DEV_LANGSMITH_SECRET_ENV_PATH}" + fi + echo "Kustomize placeholder files are ready ✓" +} + +# Deploys the application using kustomize and waits for it to be available. +# Arguments: +# $1: The namespace to deploy into. +function deploy_and_wait() { + local namespace="${1}" + print_step_header "DEPLOY APPLICATION" + echo "Deploying application from '${KUSTOMIZE_DIR}'..." >&2 + + if ! oc apply -k "${KUSTOMIZE_DIR}" -n "${namespace}"; then + echo "Error: 'oc apply' failed." >&2 + exit 1 + fi + + print_step_header "CHECK READINESS" + echo "Waiting for deployment '${APP_DEPLOYMENT_NAME}' to become available..." >&2 + oc wait --for=condition=Available "deployment/${APP_DEPLOYMENT_NAME}" \ + -n "${namespace}" --timeout="${DEPLOYMENT_TIMEOUT}" + echo "Deployment is ready!" >&2 +} + +# Starts the application server inside the running pod. +# Arguments: +# $1: The namespace where the pod is running. +# $2: The name of the pod. +function start_server_in_pod() { + local namespace="${1}" + local pod_name="${2}" + + print_step_header "START SERVER" + echo "Starting application server in the background..." >&2 + + # The AGENT_MORPHEUS_PYTHON_COMMAND env var is expected to be set in the pod's environment. + local start_command="source /opt/conda/etc/profile.d/conda.sh && conda activate ${CONDA_ENV_NAME} && eval \"\$AGENT_MORPHEUS_PYTHON_COMMAND\" > ${SERVER_LOG_PATH} 2>&1" + + oc exec "${pod_name}" -n "${namespace}" -- /bin/bash -c "${start_command}" & +} + +# Polls the server's readiness endpoint from within the pod to confirm it's up. +# Arguments: +# $1: The namespace where the pod is running. +# $2: The name of the pod to test. +function check_server_readiness() { + local namespace="${1}" + local pod_name="${2}" + + print_step_header "TEST" + echo "Waiting for server to become ready by polling the readiness endpoint..." >&2 + + local i + for i in $(seq 1 "${READINESS_PROBE_RETRIES}"); do + echo "Attempt ${i}/${READINESS_PROBE_RETRIES} to check internal readiness..." >&2 + # Add '|| true' to prevent 'set -e' from exiting the script if curl fails. + local http_code + http_code=$(oc exec "${pod_name}" -n "${namespace}" -- \ + curl -s -o /dev/null -w "%{http_code}" \ + "http://localhost:${APP_PORT}${HEALTH_ENDPOINT}" || true) + + if [[ "${http_code}" -eq 200 ]]; then + echo "SUCCESS! Server is ready and responding with 200 OK." >&2 + return 0 + fi + + echo "Server not yet ready (HTTP code: ${http_code}). Retrying in ${READINESS_PROBE_INTERVAL_SECONDS} seconds..." >&2 + sleep "${READINESS_PROBE_INTERVAL_SECONDS}" + done + + echo "Error: Server failed to become ready after ${READINESS_PROBE_RETRIES} attempts." >&2 + print_step_header "CAPTURED SERVER LOGS" + oc exec "${pod_name}" -n "${namespace}" -- cat "${SERVER_LOG_PATH}" + return 1 +} + +# ============================================================================= +# --- MAIN EXECUTION --- +# ============================================================================= + +function main() { + # Set the trap at the very beginning of execution. + # It catches EXIT, TERM (termination), and INT (interrupt from Ctrl+C). + trap 'cleanup' EXIT TERM INT + + check_env_vars + + # NAMESPACE is used by the cleanup trap, so it must have global scope. + readonly NAMESPACE="exploitiq-e2e-${PR_NUMBER}" + + setup_namespace "${NAMESPACE}" + copy_image_pull_secret "${NAMESPACE}" + create_kustomize_env_files + deploy_and_wait "${NAMESPACE}" + + local pod_name + pod_name=$(get_pod_name "${NAMESPACE}") + + start_server_in_pod "${NAMESPACE}" "${pod_name}" + check_server_readiness "${NAMESPACE}" "${pod_name}" + + print_step_header "RESULT" + echo "✅ INTEGRATION TEST PASSED!" +} + +main "$@" \ No newline at end of file