diff --git a/build.sh b/build.sh index 9930c7bc..7f51bd2f 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ SCENARIOS=( application-outages container-scenarios network-chaos node-cpu-hog node-io-hog \ node-memory-hog node-scenarios node-scenarios-bm pod-network-chaos pod-scenarios power-outages pvc-scenario \ -service-disruption-scenarios service-hijacking syn-flood time-scenarios zone-outages node-network-filter pod-network-filter kubevirt-outage) +service-disruption-scenarios service-hijacking syn-flood time-scenarios zone-outages node-network-filter pod-network-filter kubevirt-outage http-load) for i in "${SCENARIOS[@]}"; do export KRKNCTL_INPUT=$(cat $i/krknctl-input.json|tr -d "\n") envsubst < $i/Dockerfile.template > $i/Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index c54ea10c..674f0ac5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -113,4 +113,9 @@ services: build: context: ./ dockerfile: ./kubevirt-outage/Dockerfile - image: quay.io/krkn-chaos/krkn-hub:kubevirt-outage \ No newline at end of file + image: quay.io/krkn-chaos/krkn-hub:kubevirt-outage + http-load: + build: + context: ./ + dockerfile: ./http-load/Dockerfile + image: quay.io/krkn-chaos/krkn-hub:http-load \ No newline at end of file diff --git a/docs/http-load.md b/docs/http-load.md new file mode 100644 index 00000000..f66322c8 --- /dev/null +++ b/docs/http-load.md @@ -0,0 +1,66 @@ +### HTTP Load scenario +This scenario generates distributed HTTP load against one or more target endpoints using Vegeta load testing pods deployed inside the cluster. +For more details, please refer to the [Kraken documentation](https://github.com/krkn-chaos/krkn/blob/main/docs/http_load_scenarios.md). + +#### Run +If enabling [Cerberus](https://github.com/krkn-chaos/krkn#kraken-scenario-passfail-criteria-and-report) to monitor the cluster and pass/fail the scenario post chaos, refer [docs](https://github.com/redhat-chaos/krkn-hub/tree/main/docs/cerberus.md). Make sure to start it before injecting the chaos and set `CERBERUS_ENABLED` environment variable for the chaos injection container to autoconnect. + +``` +$ podman run --name= --net=host --env-host=true -v :/home/krkn/.kube/config:Z +-e TARGET_ENDPOINTS="GET https://myapp.example.com/health" \ +-e NAMESPACE= \ +-e TOTAL_CHAOS_DURATION=30s \ +-e NUMBER_OF_PODS=2 \ +-e NODE_SELECTORS==;= \ +-d +quay.io/krkn-chaos/krkn-hub:http-load + +$ podman logs -f # Streams Kraken logs +$ podman inspect --format "{{.State.ExitCode}}" # Outputs exit code which can considered as pass/fail for the scenario +``` + +``` +$ docker run $(./get_docker_params.sh) --name= --net=host -v :/home/krkn/.kube/config:Z +-e TARGET_ENDPOINTS="GET https://myapp.example.com/health" \ +-e NAMESPACE= \ +-e TOTAL_CHAOS_DURATION=30s \ +-e NUMBER_OF_PODS=2 \ +-e NODE_SELECTORS==;= \ +-d +quay.io/krkn-chaos/krkn-hub:http-load + +$ docker logs -f # Streams Kraken logs +$ docker inspect --format "{{.State.ExitCode}}" # Outputs exit code which can considered as pass/fail for the scenario +``` + +**TIP**: Because the container runs with a non-root user, ensure the kube config is globally readable before mounting it in the container. You can achieve this with the following commands: +```kubectl config view --flatten > ~/kubeconfig && chmod 444 ~/kubeconfig && docker run $(./get_docker_params.sh) --name= --net=host -v ~kubeconfig:/home/krkn/.kube/config:Z -d quay.io/krkn-chaos/krkn-hub:http-load``` +#### Supported parameters + +The following environment variables can be set on the host running the container to tweak the scenario/faults being injected: + +ex.) +`export =` + +See list of variables that apply to all scenarios [here](all_scenarios_env.md) that can be used/set in addition to these scenario specific variables + + +|Parameter | Description | Default | +|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +|TARGET_ENDPOINTS| Semicolon-separated list of target endpoints. Format: METHOD URL;METHOD URL HEADER1:VAL1,HEADER2:VAL2 BODY. Example: GET https://myapp.example.com/health;POST https://myapp.example.com/api Content-Type:application/json {\"key\":\"value\"} | **Required** | +|RATE| Request rate per pod (e.g. 50/1s, 1000/1m, 0 for max throughput) |50/1s| +|TOTAL_CHAOS_DURATION| Duration of the load test (e.g. 30s, 5m, 1h) |30s| +|NAMESPACE| The namespace where the attacker pods will be deployed |default| +|NUMBER_OF_PODS| The number of attacker pods that will be deployed |2| +|WORKERS| Initial number of concurrent workers per pod |10| +|MAX_WORKERS| Maximum number of concurrent workers per pod (auto-scales) |100| +|CONNECTIONS| Maximum number of idle open connections per host |100| +|TIMEOUT| Per-request timeout (e.g. 10s, 30s) |10s| +|IMAGE| The container image that will be used to perform the scenario |quay.io/krkn-chaos/krkn-http-load:latest| +|INSECURE| Skip TLS certificate verification (for self-signed certs) |false| +|NODE_SELECTORS| The node selectors are used to guide the cluster on where to deploy attacker pods. You can specify one or more labels in the format key=value;key=value2 (even using the same key) to choose one or more node categories. If left empty, the pods will be scheduled on any available node, depending on the cluster's capacity. || + +**NOTE** In case of using custom metrics profile or alerts profile when `CAPTURE_METRICS` or `ENABLE_ALERTS` is enabled, mount the metrics profile from the host on which the container is run using podman/docker under `/home/krkn/kraken/config/metrics-aggregated.yaml` and `/home/krkn/kraken/config/alerts`. For example: +``` +$ podman run --name= --net=host --env-host=true -v :/home/krkn/kraken/config/metrics-aggregated.yaml -v :/home/krkn/kraken/config/alerts -v :/home/krkn/.kube/config:Z -d quay.io/krkn-chaos/krkn-hub:http-load +``` diff --git a/http-load/Dockerfile.template b/http-load/Dockerfile.template new file mode 100644 index 00000000..c1479882 --- /dev/null +++ b/http-load/Dockerfile.template @@ -0,0 +1,25 @@ +# Dockerfile for kraken + +FROM quay.io/krkn-chaos/krkn:latest + +ENV KUBECONFIG /home/krkn/.kube/config + +# Copy configurations + +COPY config.yaml.template /home/krkn/kraken/config/config.yaml.template +COPY http-load/env.sh /home/krkn/env.sh +COPY http-load/build_config_file.py /home/krkn/build_config_file.py +COPY env.sh /home/krkn/main_env.sh +COPY http-load/run.sh /home/krkn/run.sh +COPY common_run.sh /home/krkn/common_run.sh + +LABEL krknctl.kubeconfig_path="/home/krkn/.kube/config" +LABEL krknctl.title="HTTP Load" +LABEL krknctl.description="This scenario generates distributed HTTP load against one or more target endpoints \ +using Vegeta load testing pods deployed inside the cluster. For more details, please refer to \ +the following documentation (https://github.com/krkn-chaos/krkn-hub/blob/main/docs/http-load.md)." + + +LABEL krknctl.input_fields='$KRKNCTL_INPUT' + +ENTRYPOINT /home/krkn/kraken/containers/setup-ssh.sh && /home/krkn/run.sh diff --git a/http-load/README.md b/http-load/README.md new file mode 100644 index 00000000..ae66501b --- /dev/null +++ b/http-load/README.md @@ -0,0 +1,3 @@ +# HTTP Load Scenario Docs + +See [doc](../docs/http-load.md) for how to run and all the variables listed diff --git a/http-load/build_config_file.py b/http-load/build_config_file.py new file mode 100644 index 00000000..8f0ba05b --- /dev/null +++ b/http-load/build_config_file.py @@ -0,0 +1,100 @@ +import logging +import re +import yaml +import os +import argparse + + +def main(): + + parser = argparse.ArgumentParser(description='') + parser.add_argument('--outconfig', type=str, help='Output config path') + args = parser.parse_args() + + runs = os.getenv("RUNS", "1") + number_of_pods = os.getenv("NUMBER_OF_PODS", "2") + namespace = os.getenv("NAMESPACE", "default") + image = os.getenv("IMAGE", "quay.io/krkn-chaos/krkn-http-load:latest") + node_selectors = os.getenv("NODE_SELECTORS", "") + target_endpoints = os.getenv("TARGET_ENDPOINTS", "") + rate = os.getenv("RATE", "50/1s") + duration = os.getenv("TOTAL_CHAOS_DURATION", "30s") + workers = os.getenv("WORKERS", "10") + max_workers = os.getenv("MAX_WORKERS", "100") + connections = os.getenv("CONNECTIONS", "100") + timeout = os.getenv("TIMEOUT", "10s") + keepalive = os.getenv("KEEPALIVE", "true") + http2 = os.getenv("HTTP2", "true") + insecure = os.getenv("INSECURE", "false") + + if not target_endpoints: + logging.error("TARGET_ENDPOINTS must be set. Format: " + "METHOD URL;METHOD URL or " + "METHOD URL HEADER1:VAL1,HEADER2:VAL2 BODY;...") + exit(1) + + node_selectors_re = re.compile(r"^$|^(.+=.*)(;.+=.*)*$") + if not node_selectors_re.match(node_selectors): + logging.error(f"{node_selectors} is not a valid list of node selectors, " + f"node selectors must be one or more selectors separated by ;" + f"e.g. key1=value or key1=value1;key1=value2;key2=value3") + exit(1) + + # Parse endpoints: "GET https://url1;POST https://url2 Content-Type:application/json {\"key\":\"val\"}" + endpoints = [] + for entry in target_endpoints.split(";"): + parts = entry.strip().split(" ", 3) + if len(parts) < 2: + logging.error(f"Invalid endpoint format: {entry}. " + f"Expected: METHOD URL [HEADERS] [BODY]") + exit(1) + endpoint = {"method": parts[0], "url": parts[1]} + if len(parts) >= 3 and parts[2]: + headers = {} + for header in parts[2].split(","): + if ":" in header: + k, v = header.split(":", 1) + headers[k.strip()] = v.strip() + if headers: + endpoint["headers"] = headers + if len(parts) >= 4 and parts[3]: + endpoint["body"] = parts[3] + endpoints.append(endpoint) + + # Parse node selectors + parsed_node_selectors = {} + if node_selectors and node_selectors != '': + for selector in node_selectors.split(";"): + key_value = selector.split("=") + if key_value[0] not in parsed_node_selectors.keys(): + parsed_node_selectors[key_value[0]] = [] + parsed_node_selectors[key_value[0]].append(key_value[1]) + + config = [{ + "http_load_scenario": { + "runs": int(runs), + "number-of-pods": int(number_of_pods), + "namespace": namespace, + "image": image, + "attacker-nodes": parsed_node_selectors if parsed_node_selectors else {}, + "targets": { + "endpoints": endpoints + }, + "rate": rate, + "duration": duration, + "workers": int(workers), + "max_workers": int(max_workers), + "connections": int(connections), + "timeout": timeout, + "keepalive": keepalive.lower() == "true", + "http2": http2.lower() == "true", + "insecure": insecure.lower() == "true", + } + }] + + with open(args.outconfig, "w") as out: + yaml.dump(config, out, default_flow_style=False, allow_unicode=True) + + +if __name__ == '__main__': + main() diff --git a/http-load/env.sh b/http-load/env.sh new file mode 100644 index 00000000..318c1cf2 --- /dev/null +++ b/http-load/env.sh @@ -0,0 +1,19 @@ +#!/bin/bash +export RUNS=${RUNS:="1"} +export NUMBER_OF_PODS=${NUMBER_OF_PODS:="2"} +export NAMESPACE=${NAMESPACE:="default"} +export IMAGE=${IMAGE:="quay.io/krkn-chaos/krkn-http-load:latest"} +export NODE_SELECTORS=${NODE_SELECTORS:=""} +export TARGET_ENDPOINTS=${TARGET_ENDPOINTS:=""} +export RATE=${RATE:="50/1s"} +export TOTAL_CHAOS_DURATION=${TOTAL_CHAOS_DURATION:="30s"} +export WORKERS=${WORKERS:="10"} +export MAX_WORKERS=${MAX_WORKERS:="100"} +export CONNECTIONS=${CONNECTIONS:="100"} +export TIMEOUT=${TIMEOUT:="10s"} +export KEEPALIVE=${KEEPALIVE:="true"} +export HTTP2=${HTTP2:="true"} +export INSECURE=${INSECURE:="false"} + +export SCENARIO_TYPE=${SCENARIO_TYPE:=http_load_scenarios} +export SCENARIO_FILE=${SCENARIO_FILE:="$KRAKEN_FOLDER/scenarios/http-load.yaml"} diff --git a/http-load/krknctl-input.json b/http-load/krknctl-input.json new file mode 100644 index 00000000..a3acabaf --- /dev/null +++ b/http-load/krknctl-input.json @@ -0,0 +1 @@ +[{"name":"target-endpoints","short_description":"Target endpoints","description":"Semicolon-separated list of target endpoints. Format: METHOD URL;METHOD URL HEADER1:VAL1,HEADER2:VAL2 BODY. Example: GET https://myapp.example.com/health;POST https://myapp.example.com/api Content-Type:application/json {\"key\":\"value\"}","variable":"TARGET_ENDPOINTS","type":"string","default":"","required":"true"},{"name":"rate","short_description":"Request rate","description":"Request rate per pod (e.g. 50/1s, 1000/1m, 0 for max throughput)","variable":"RATE","type":"string","default":"50/1s","required":"false"},{"name":"chaos-duration","short_description":"Chaos duration","description":"Duration of the load test (e.g. 30s, 5m, 1h)","variable":"TOTAL_CHAOS_DURATION","type":"string","default":"30s","required":"false"},{"name":"namespace","short_description":"Namespace","description":"The namespace where the attacker pods will be deployed","variable":"NAMESPACE","type":"string","default":"default","required":"false"},{"name":"number-of-pods","short_description":"Number of pods","description":"The number of attacker pods that will be deployed","variable":"NUMBER_OF_PODS","type":"number","default":"2","required":"false"},{"name":"workers","short_description":"Workers","description":"Initial number of concurrent workers per pod","variable":"WORKERS","type":"number","default":"10","required":"false"},{"name":"max-workers","short_description":"Max workers","description":"Maximum number of concurrent workers per pod (auto-scales)","variable":"MAX_WORKERS","type":"number","default":"100","required":"false"},{"name":"connections","short_description":"Connections","description":"Maximum number of idle open connections per host","variable":"CONNECTIONS","type":"number","default":"100","required":"false"},{"name":"timeout","short_description":"Timeout","description":"Per-request timeout (e.g. 10s, 30s)","variable":"TIMEOUT","type":"string","default":"10s","required":"false"},{"name":"image","short_description":"Workload image","description":"The container image that will be used to perform the scenario","variable":"IMAGE","type":"string","default":"quay.io/krkn-chaos/krkn-http-load:latest","required":"false"},{"name":"insecure","short_description":"Insecure TLS","description":"Skip TLS certificate verification (for self-signed certs)","variable":"INSECURE","type":"string","default":"false","required":"false"},{"name":"node-selectors","short_description":"Node selectors","description":"The node selectors are used to guide the cluster on where to deploy attacker pods. You can specify one or more labels in the format key=value;key=value2 (even using the same key) to choose one or more node categories. If left empty, the pods will be scheduled on any available node, depending on the cluster s capacity.","variable":"NODE_SELECTORS","type":"string","validator":"^$|^(([a-zA-Z0-9._-]+\\=[a-zA-Z0-9._-]+)(;)?)+[^;]$","validation_message":"node selector must be in the format key=value or a list of semicolon-separated selectors key=value;key2=value2;key3=value3","default":"","required":"false"}] diff --git a/http-load/run.sh b/http-load/run.sh new file mode 100644 index 00000000..ba3d2aa9 --- /dev/null +++ b/http-load/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash +ROOT_FOLDER="/home/krkn" +KRAKEN_FOLDER="$ROOT_FOLDER/kraken" +SCENARIO_FOLDER="$KRAKEN_FOLDER/scenarios/http-load" + +# Source env.sh to read all the vars +source $ROOT_FOLDER/main_env.sh +source $ROOT_FOLDER/env.sh +source $ROOT_FOLDER/common_run.sh +extra_var="" +if [[ $KRKN_DEBUG == "True" ]];then + set -ex + extra_var="--debug True" +fi + +# Build scenario config from environment variables +python3.11 $ROOT_FOLDER/build_config_file.py --outconfig $KRAKEN_FOLDER/scenarios/http-load.yaml +envsubst < $KRAKEN_FOLDER/config/config.yaml.template > $KRAKEN_FOLDER/config/http_load_config.yaml + +cat $KRAKEN_FOLDER/config/http_load_config.yaml +cat $KRAKEN_FOLDER/scenarios/http-load.yaml + +checks + +# Run Kraken +cd $KRAKEN_FOLDER +python3.11 run_kraken.py --config=config/http_load_config.yaml $extra_var