Skip to content

K8SPS-393 PMM3 support #849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hors merged 4 commits into from
Mar 4, 2025
Merged

K8SPS-393 PMM3 support #849

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9f034c2
K8SPS-393
gkech Feb 27, 2025
2cbab5e
Merge branch 'main' into K8SPS-393
gkech Mar 3, 2025
fa5aa8a
random fixes
gkech Mar 3, 2025
6978c3b
fix monitoring test
hors Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deploy/cr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ spec:
pmm:
enabled: false

image: perconalab/pmm-client:dev-latest
image: perconalab/pmm-client:3-dev-latest
imagePullPolicy: Always

resources:
Expand Down
2 changes: 1 addition & 1 deletion deploy/secrets.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ stringData:
root: root_password
xtrabackup: backup_password
monitor: monitor_password
# pmmserverkey: my_pmm_server_key
#pmmserverkey: my_pmm_server_key
operator: operator_password
replication: replication_password
orchestrator: orchestrator_password
Expand Down
2 changes: 1 addition & 1 deletion e2e-tests/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ full list of variables is the following one:

* `IMAGE` - the Operator, `perconalab/percona-server-mysql-operator:main` by default,
* `IMAGE_MYSQL` - Percona Distribution for MySQL, `perconalab/percona-server:main` by default,
* `IMAGE_PMM_CLIENT` - Percona Monitoring and Management (PMM) client, `perconalab/pmm-client:dev-latest` by default,
* `IMAGE_PMM_CLIENT` - Percona Monitoring and Management (PMM) client, `perconalab/pmm-client:3-dev-latest` by default,
* `IMAGE_PROXY` - ProxySQL, `perconalab/percona-xtradb-cluster-operator:main-proxysql` by default,
* `IMAGE_HAPROXY` - HA Proxy, `perconalab/haproxy:main` by default,
* `IMAGE_BACKUP` - backups, `perconalab/percona-xtrabackup:main` by default,
Expand Down
48 changes: 41 additions & 7 deletions e2e-tests/functions
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,16 @@ deploy_pmm_server() {
local additional_params="--set platform=openshift --set sa=pmm-server --set supresshttp2=false"
fi

helm install monitoring -n "${NAMESPACE}" --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} $additional_params https://percona-charts.storage.googleapis.com/pmm-server-$PMM_SERVER_VERSION.tgz
kubectl delete clusterrole monitoring --ignore-not-found
kubectl delete clusterrolebinding monitoring --ignore-not-found

helm install monitoring percona/pmm -n "${NAMESPACE}" \
--set fullnameOverride=monitoring \
--version ${PMM_SERVER_VERSION} \
--set imageTag=${IMAGE_PMM_SERVER#*:} \
--set imageRepo=${IMAGE_PMM_SERVER%:*} \
--set service.type=LoadBalancer \
--force

until kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null"; do
echo "Retry $retry"
Expand All @@ -117,8 +126,30 @@ get_pmm_api_key() {
fi

local ADMIN_PASSWORD
ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2")
curl --insecure -X POST -H "Content-Type: application/json" -d "{\"name\":\"$key_name\", \"role\": \"Admin\"}" "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq .key
ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" get secret pmm-secret -o jsonpath="{.data.PMM_ADMIN_PASSWORD}" | base64 --decode)

if [[ -z $ADMIN_PASSWORD ]]; then
echo "Error: ADMIN_PASSWORD is empty or not found!" >&2
return 1
fi

local response status_code json_response
response=$(curl --insecure -s -X POST -H 'Content-Type: application/json' \
-d "{\"name\":\"${key_name}\", \"role\": \"Admin\"}" \
--user "admin:${ADMIN_PASSWORD}" \
"https://$(get_service_ip monitoring-service)/graph/api/auth/keys" \
-w "\n%{http_code}")

status_code=$(echo "$response" | tail -n1)
json_response=$(echo "$response" | sed '$ d')

if [[ $status_code -eq 200 ]]; then
echo "$json_response" | jq -r '.key'
else
echo "Error: Failed to get API key with HTTP status code: $status_code)" >&2
echo "Response: $json_response" >&2
return 1
fi
}

delete_pmm_api_key() {
Expand All @@ -129,11 +160,14 @@ delete_pmm_api_key() {
fi

local ADMIN_PASSWORD
ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2")
ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" get secret pmm-secret -o jsonpath="{.data.PMM_ADMIN_PASSWORD}" | base64 --decode)

local user_credentials="admin:${ADMIN_PASSWORD}"

local key_id
key_id=$(curl --insecure -X GET "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq '.[] | select( .name == "operator").id')
curl --insecure -X DELETE "https://admin:$ADMIN_PASSWORD@$(get_service_ip monitoring-service)/graph/api/auth/keys/$key_id"
key_id=$(curl --insecure -X GET --user "${user_credentials}" "https://$(get_service_ip monitoring-service)/graph/api/auth/keys" | jq '.[] | select( .name == "operator").id')

curl --insecure -X DELETE --user "${user_credentials}" "https://$(get_service_ip monitoring-service)/graph/api/auth/keys/$key_id"
}

deploy_minio() {
Expand Down Expand Up @@ -370,7 +404,7 @@ get_qan20_values() {
EOF
)

run_curl -XPOST -d "'$(echo ${payload} | sed 's/\n//g')'" "https://${user_pass}@${endpoint}/v0/qan/GetReport" \
run_curl -XPOST -d "'$(echo ${payload} | sed 's/\n//g')'" "https://${user_pass}@${endpoint}/v1/qan/metrics:getReport" \
| jq '.rows[].fingerprint'
}

Expand Down
54 changes: 27 additions & 27 deletions e2e-tests/tests/init-deploy/01-create-cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,30 @@ commands:
source ../../functions

get_cr \
| yq eval '.spec.mysql.clusterType="async"' - \
| yq eval '.spec.mysql.size=3' - \
| yq eval '.spec.proxy.haproxy.enabled=true' - \
| yq eval '.spec.proxy.haproxy.size=3' - \
| yq eval '.spec.orchestrator.enabled=true' - \
| yq eval '.spec.mysql.resources.limits.cpu="500m"' - \
| yq eval '.spec.mysql.resources.limits.memory="1G"' - \
| yq eval '.spec.mysql.resources.requests.cpu="400m"' - \
| yq eval '.spec.mysql.resources.requests.memory="1G"' - \
| yq eval '.spec.backup.resources.limits.cpu="200m"' - \
| yq eval '.spec.backup.resources.limits.memory="256Mi"' - \
| yq eval '.spec.backup.resources.requests.cpu="100m"' - \
| yq eval '.spec.backup.resources.requests.memory="128Mi"' - \
| yq eval '.spec.toolkit.resources.limits.cpu="150m"' - \
| yq eval '.spec.toolkit.resources.limits.memory="256Mi"' - \
| yq eval '.spec.toolkit.resources.requests.cpu="100m"' - \
| yq eval '.spec.toolkit.resources.requests.memory="128Mi"' - \
| yq eval '.spec.proxy.haproxy.resources.limits.cpu="600m"' - \
| yq eval '.spec.proxy.haproxy.resources.limits.memory="512Mi"' - \
| yq eval '.spec.proxy.haproxy.resources.requests.cpu="500m"' - \
| yq eval '.spec.proxy.haproxy.resources.requests.memory="256Mi"' - \
| yq eval '.spec.orchestrator.resources.limits.cpu="200m"' - \
| yq eval '.spec.orchestrator.resources.limits.memory="256Mi"' - \
| yq eval '.spec.orchestrator.resources.requests.cpu="100m"' - \
| yq eval '.spec.orchestrator.resources.requests.memory="128Mi"' - \
| yq eval '.spec.orchestrator.size=3' - \
| kubectl -n "${NAMESPACE}" apply -f -
| yq eval '.spec.mysql.clusterType="async"' - \
| yq eval '.spec.mysql.size=3' - \
| yq eval '.spec.proxy.haproxy.enabled=true' - \
| yq eval '.spec.proxy.haproxy.size=3' - \
| yq eval '.spec.orchestrator.enabled=true' - \
| yq eval '.spec.mysql.resources.limits.cpu="500m"' - \
| yq eval '.spec.mysql.resources.limits.memory="1G"' - \
| yq eval '.spec.mysql.resources.requests.cpu="400m"' - \
| yq eval '.spec.mysql.resources.requests.memory="1G"' - \
| yq eval '.spec.backup.resources.limits.cpu="200m"' - \
| yq eval '.spec.backup.resources.limits.memory="256Mi"' - \
| yq eval '.spec.backup.resources.requests.cpu="100m"' - \
| yq eval '.spec.backup.resources.requests.memory="128Mi"' - \
| yq eval '.spec.toolkit.resources.limits.cpu="150m"' - \
| yq eval '.spec.toolkit.resources.limits.memory="256Mi"' - \
| yq eval '.spec.toolkit.resources.requests.cpu="100m"' - \
| yq eval '.spec.toolkit.resources.requests.memory="128Mi"' - \
| yq eval '.spec.proxy.haproxy.resources.limits.cpu="600m"' - \
| yq eval '.spec.proxy.haproxy.resources.limits.memory="512Mi"' - \
| yq eval '.spec.proxy.haproxy.resources.requests.cpu="500m"' - \
| yq eval '.spec.proxy.haproxy.resources.requests.memory="256Mi"' - \
| yq eval '.spec.orchestrator.resources.limits.cpu="200m"' - \
| yq eval '.spec.orchestrator.resources.limits.memory="256Mi"' - \
| yq eval '.spec.orchestrator.resources.requests.cpu="100m"' - \
| yq eval '.spec.orchestrator.resources.requests.memory="128Mi"' - \
| yq eval '.spec.orchestrator.size=3' - \
| kubectl -n "${NAMESPACE}" apply -f -
2 changes: 1 addition & 1 deletion e2e-tests/tests/monitoring/01-deploy-pmm-server.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ commands:
sleep 30 # wait for PMM Server to start

API_KEY=$(get_pmm_api_key)
kubectl patch -n "${NAMESPACE}" secret test-secrets --type merge --patch '{"stringData": {"pmmserverkey": '$API_KEY'}}'
kubectl patch -n "${NAMESPACE}" secret test-secrets --type merge --patch "$(jq -n --arg key "$API_KEY" '{"stringData": {"pmmserverkey": $key}}')"
timeout: 120
42 changes: 6 additions & 36 deletions e2e-tests/tests/monitoring/02-assert.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ spec:
fieldPath: metadata.namespace
- name: CLUSTER_NAME
value: monitoring
- name: CLIENT_PORT_LISTEN
value: "7777"
- name: CLIENT_PORT_MIN
value: "30100"
- name: CLIENT_PORT_MAX
value: "30105"
- name: PMM_AGENT_SERVER_ADDRESS
value: monitoring-service
- name: PMM_AGENT_SERVER_USERNAME
Expand All @@ -42,23 +36,14 @@ spec:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_SERVER
value: monitoring-service
- name: PMM_USER
value: api_key
- name: PMM_PASSWORD
valueFrom:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_AGENT_LISTEN_PORT
value: "7777"
- name: PMM_AGENT_PORTS_MIN
value: "30100"
- name: PMM_AGENT_PORTS_MAX
value: "30105"
- name: PMM_AGENT_CONFIG_FILE
value: /usr/local/percona/pmm2/config/pmm-agent.yaml
value: /usr/local/percona/pmm/config/pmm-agent.yaml
- name: PMM_AGENT_SERVER_INSECURE_TLS
value: "1"
- name: PMM_AGENT_LISTEN_ADDRESS
Expand All @@ -73,14 +58,14 @@ spec:
value: "1"
- name: PMM_AGENT_SETUP_NODE_TYPE
value: container
- name: PMM_AGENT_PRERUN_SCRIPT
value: /opt/percona/pmm-prerun.sh
- name: PMM_AGENT_SIDECAR
value: "true"
- name: PMM_AGENT_SIDECAR_SLEEP
value: "5"
- name: PMM_AGENT_PATHS_TEMPDIR
value: "/tmp"
- name: PMM_AGENT_PRERUN_SCRIPT
value: "/opt/percona/pmm-prerun.sh"
- name: DB_CLUSTER
value: monitoring
- name: DB_TYPE
Expand Down Expand Up @@ -159,12 +144,6 @@ spec:
fieldPath: metadata.namespace
- name: CLUSTER_NAME
value: monitoring
- name: CLIENT_PORT_LISTEN
value: "7777"
- name: CLIENT_PORT_MIN
value: "30100"
- name: CLIENT_PORT_MAX
value: "30105"
- name: PMM_AGENT_SERVER_ADDRESS
value: monitoring-service
- name: PMM_AGENT_SERVER_USERNAME
Expand All @@ -174,23 +153,14 @@ spec:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_SERVER
value: monitoring-service
- name: PMM_USER
value: api_key
- name: PMM_PASSWORD
valueFrom:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_AGENT_LISTEN_PORT
value: "7777"
- name: PMM_AGENT_PORTS_MIN
value: "30100"
- name: PMM_AGENT_PORTS_MAX
value: "30105"
- name: PMM_AGENT_CONFIG_FILE
value: /usr/local/percona/pmm2/config/pmm-agent.yaml
value: /usr/local/percona/pmm/config/pmm-agent.yaml
- name: PMM_AGENT_SERVER_INSECURE_TLS
value: "1"
- name: PMM_AGENT_LISTEN_ADDRESS
Expand All @@ -205,14 +175,14 @@ spec:
value: "1"
- name: PMM_AGENT_SETUP_NODE_TYPE
value: container
- name: PMM_AGENT_PRERUN_SCRIPT
value: /opt/percona/pmm-prerun.sh
- name: PMM_AGENT_SIDECAR
value: "true"
- name: PMM_AGENT_SIDECAR_SLEEP
value: "5"
- name: PMM_AGENT_PATHS_TEMPDIR
value: "/tmp"
- name: PMM_AGENT_PRERUN_SCRIPT
value: "/opt/percona/pmm-prerun.sh"
Comment on lines +184 to +185
Copy link
Copy Markdown
Contributor Author

@gkech gkech Mar 3, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gkech check why this env var is not references in the PMM documentation

Copy link
Copy Markdown
Contributor Author

@gkech gkech Mar 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ticket created

- name: DB_CLUSTER
value: monitoring
- name: DB_TYPE
Expand Down
42 changes: 6 additions & 36 deletions e2e-tests/tests/monitoring/03-assert.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ spec:
fieldPath: metadata.namespace
- name: CLUSTER_NAME
value: monitoring
- name: CLIENT_PORT_LISTEN
value: "7777"
- name: CLIENT_PORT_MIN
value: "30100"
- name: CLIENT_PORT_MAX
value: "30105"
- name: PMM_AGENT_SERVER_ADDRESS
value: monitoring-service
- name: PMM_AGENT_SERVER_USERNAME
Expand All @@ -42,23 +36,14 @@ spec:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_SERVER
value: monitoring-service
- name: PMM_USER
value: api_key
- name: PMM_PASSWORD
valueFrom:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_AGENT_LISTEN_PORT
value: "7777"
- name: PMM_AGENT_PORTS_MIN
value: "30100"
- name: PMM_AGENT_PORTS_MAX
value: "30105"
- name: PMM_AGENT_CONFIG_FILE
value: /usr/local/percona/pmm2/config/pmm-agent.yaml
value: /usr/local/percona/pmm/config/pmm-agent.yaml
- name: PMM_AGENT_SERVER_INSECURE_TLS
value: "1"
- name: PMM_AGENT_LISTEN_ADDRESS
Expand All @@ -73,14 +58,14 @@ spec:
value: "1"
- name: PMM_AGENT_SETUP_NODE_TYPE
value: container
- name: PMM_AGENT_PRERUN_SCRIPT
value: /opt/percona/pmm-prerun.sh
- name: PMM_AGENT_SIDECAR
value: "true"
- name: PMM_AGENT_SIDECAR_SLEEP
value: "5"
- name: PMM_AGENT_PATHS_TEMPDIR
value: "/tmp"
- name: PMM_AGENT_PRERUN_SCRIPT
value: "/opt/percona/pmm-prerun.sh"
- name: DB_CLUSTER
value: monitoring
- name: DB_TYPE
Expand Down Expand Up @@ -158,12 +143,6 @@ spec:
fieldPath: metadata.namespace
- name: CLUSTER_NAME
value: monitoring
- name: CLIENT_PORT_LISTEN
value: "7777"
- name: CLIENT_PORT_MIN
value: "30100"
- name: CLIENT_PORT_MAX
value: "30105"
- name: PMM_AGENT_SERVER_ADDRESS
value: monitoring-service
- name: PMM_AGENT_SERVER_USERNAME
Expand All @@ -173,23 +152,14 @@ spec:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_SERVER
value: monitoring-service
- name: PMM_USER
value: api_key
- name: PMM_PASSWORD
valueFrom:
secretKeyRef:
key: pmmserverkey
name: internal-monitoring
- name: PMM_AGENT_LISTEN_PORT
value: "7777"
- name: PMM_AGENT_PORTS_MIN
value: "30100"
- name: PMM_AGENT_PORTS_MAX
value: "30105"
- name: PMM_AGENT_CONFIG_FILE
value: /usr/local/percona/pmm2/config/pmm-agent.yaml
value: /usr/local/percona/pmm/config/pmm-agent.yaml
- name: PMM_AGENT_SERVER_INSECURE_TLS
value: "1"
- name: PMM_AGENT_LISTEN_ADDRESS
Expand All @@ -204,14 +174,14 @@ spec:
value: "1"
- name: PMM_AGENT_SETUP_NODE_TYPE
value: container
- name: PMM_AGENT_PRERUN_SCRIPT
value: /opt/percona/pmm-prerun.sh
- name: PMM_AGENT_SIDECAR
value: "true"
- name: PMM_AGENT_SIDECAR_SLEEP
value: "5"
- name: PMM_AGENT_PATHS_TEMPDIR
value: "/tmp"
- name: PMM_AGENT_PRERUN_SCRIPT
value: "/opt/percona/pmm-prerun.sh"
- name: DB_CLUSTER
value: monitoring
- name: DB_TYPE
Expand Down
2 changes: 1 addition & 1 deletion e2e-tests/tests/monitoring/03-rotate-pmm-key.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ commands:

# add new PMM API key to secret
API_KEY_NEW=$(get_pmm_api_key "operator-new")
kubectl patch -n "${NAMESPACE}" secret test-secrets --type merge --patch '{"stringData": {"pmmserverkey": '$API_KEY_NEW'}}'
kubectl patch -n "${NAMESPACE}" secret test-secrets --type merge --patch "$(jq -n --arg key "$API_KEY_NEW" '{"stringData": {"pmmserverkey": $key}}')"

# delete old PMM key
delete_pmm_api_key "operator"
Expand Down
Loading
Loading