Skip to content

Commit 5c4e28a

Browse files
saikumar-vssaikumar-vs
authored
Merge branch 'v3' into pmm3-mongo-mlaunch
2 parents 0d580ec + 6e876e6 commit 5c4e28a

File tree

5 files changed

+71
-43
lines changed

5 files changed

+71
-43
lines changed

‎.github/workflows/PMM_PS.yaml‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ on:
33
workflow_dispatch:
44
inputs:
55
ps_version:
6-
description: "ps version example: 8.0.34-26 , Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS"
7-
default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS'
6+
description: "PS Version ex: 5.7/8.0/8.4, Minor versions from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS"
7+
default: '8.4'
88
required: true
99
ps_glibc:
10-
description: "ps glibc version, example: 2.35"
10+
description: "PS Glibc version, example: 2.35"
1111
default: '2.35'
1212
required: false
1313
pmm_qa_branch:
@@ -19,11 +19,11 @@ on:
1919
required: false
2020
type: string
2121
pmm_server_image:
22-
description: "pmm_image, example: perconalab/pmm-server:3-dev-latest"
22+
description: "PMM Server image, example: perconalab/pmm-server:3-dev-latest"
2323
default: 'perconalab/pmm-server:3-dev-latest'
2424
required: false
2525
pmm_client_version:
26-
description: 'PMM Client version (3-dev-latest|pmm3-rc|x.xx.x|https...)'
26+
description: 'PMM Client version (3-dev-latest|pmm3-rc|3.xx.x|https...)'
2727
default: '3-dev-latest'
2828
required: false
2929
type: string
@@ -41,7 +41,7 @@ jobs:
4141
timeout-minutes: 40
4242
env:
4343
ADMIN_PASSWORD: 'admin'
44-
PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }}
44+
PS_VERSION: ${{ github.event.inputs.ps_version || '8.4' }}
4545
PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.35' }}
4646
PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }}
4747
PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }}
@@ -92,7 +92,7 @@ jobs:
9292
working-directory: qa-integration
9393
run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV
9494

95-
- name: Run Setup for E2E Tests
95+
- name: Run Setup for PS E2E Tests
9696
working-directory: qa-integration/pmm_qa
9797
run: |
9898
mkdir -m 777 -p /tmp/backup_data
@@ -107,6 +107,16 @@ jobs:
107107
python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS,TARBALL=${{ env.PS_TARBALL_PATH }}
108108
fi
109109
110+
- name: Run Setup for PS Replica E2E Tests
111+
working-directory: qa-integration/pmm_qa
112+
run: |
113+
. virtenv/bin/activate
114+
if [[ "${{ github.event.inputs.ps_version }}" != http* ]]; then
115+
python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS=${{ env.PS_VERSION }},SETUP_TYPE=replica
116+
else
117+
python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS,TARBALL=${{ env.PS_TARBALL_PATH }},SETUP_TYPE=replica
118+
fi
119+
110120
- name: Checkout PMM UI tests
111121
uses: actions/checkout@v4
112122
with:
@@ -118,8 +128,8 @@ jobs:
118128
run: npm ci && npx playwright install --with-deps && npx codeceptjs def pr.codecept.js
119129
working-directory: pmm-ui-tests
120130

121-
- name: Run the Integration tests of PS
131+
- name: Run the Integration tests of PS and PS Replica
122132
run: |
123133
export PMM_UI_URL="http://127.0.0.1:8081/"
124-
npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js
134+
npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js tests/qa-integration/pmm_ps_replica_integration_test.js
125135
working-directory: pmm-ui-tests

‎pmm_qa/client_container_ps_setup.sh‎

Lines changed: 46 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -56,24 +56,27 @@ export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999))
5656
# Initialize my_cnf_options
5757
my_cnf_options=""
5858

59-
# Check if ps_version is 8.4 or greater to enable the plugin to change the password
60-
if [[ "$ps_version" =~ ^8\.[4-9]([0-9])? || "$ps_version" =~ ^[9-9][0-9]\. ]]; then
61-
my_cnf_options="mysql-native-password=ON"
59+
if [[ "$number_of_nodes" -gt 1 ]]; then
60+
# Check if ps_version is greater than 8.0
61+
if [[ "$ps_version" =~ ^8\.[1-9]([0-9])? || "$ps_version" =~ ^9\.[0-9]+ ]]; then
62+
my_cnf_options="caching_sha2_password_auto_generate_rsa_keys=ON"
63+
else
64+
# MySQL 5.7, create user fails which already exists, to ignore this we do:
65+
my_cnf_options="replicate-ignore-table=mysql.user"
66+
fi
6267
fi
6368

6469
if [[ "$number_of_nodes" == 1 ]];then
6570
if [[ ! -z $group_replication ]]; then
6671
dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"}
6772
export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
6873
node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
69-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"
70-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';"
74+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';"
7175
else
7276
dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"}
7377
export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
7478
node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
75-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"
76-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';"
79+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';"
7780
if [[ "${query_source}" == "slowlog" ]]; then
7881
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';"
7982
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;"
@@ -123,37 +126,51 @@ if [[ "$number_of_nodes" == 1 ]];then
123126
pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ps-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port
124127
fi
125128
else
126-
dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 ${my_cnf_options:+--my-cnf-options="$my_cnf_options"}
129+
dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 --my-cnf-options=gtid_mode=ON --my-cnf-options=enforce-gtid-consistency=ON --my-cnf-options=binlog-format=ROW --my-cnf-options=log-slave-updates=ON --my-cnf-options=binlog-checksum=NONE ${my_cnf_options:+--my-cnf-options="$my_cnf_options"}
127130
export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
128-
node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
129-
for j in `seq 1 $number_of_nodes`; do
130-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"
131-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';"
131+
master_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
132+
if echo "$ps_version" | grep '5.7'; then
133+
slave_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $2}'`
134+
else
135+
slave_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $3}'`
136+
fi
137+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $master_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';"
138+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $master_port -e "FLUSH PRIVILEGES;"
139+
mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "CREATE USER 'repl'@'%' IDENTIFIED BY 'repl_pass';"
140+
mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'repl'@'%';"
141+
mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "FLUSH PRIVILEGES;"
142+
if echo "$ps_version" | grep '5.7'; then
143+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='repl', MASTER_PASSWORD='repl_pass', MASTER_PORT=${master_port}, MASTER_AUTO_POSITION=1;"
144+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "START SLAVE;"
145+
else
146+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "CHANGE REPLICATION SOURCE TO SOURCE_HOST='127.0.0.1', SOURCE_USER='repl', SOURCE_PASSWORD='repl_pass', SOURCE_PORT=${master_port}, SOURCE_AUTO_POSITION=1, SOURCE_SSL=1;"
147+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "START REPLICA;"
148+
fi
149+
for port in $master_port $slave_port; do
132150
if [[ "${query_source}" == "slowlog" ]]; then
133-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';"
134-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;"
135-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_rate_limit=1;"
136-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;"
137-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;"
151+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL slow_query_log='ON';"
152+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL long_query_time=0;"
153+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_rate_limit=1;"
154+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_admin_statements=ON;"
155+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_slave_statements=ON;"
138156
else
139-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;"
140-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;"
141-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';"
157+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL userstat=1;"
158+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL innodb_monitor_enable=all;"
159+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';"
142160
if echo "$ps_version" | grep '5.7'; then
143-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';"
144-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';"
145-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';"
146-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';"
147-
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL query_response_time_stats=ON;"
161+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';"
162+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';"
163+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';"
164+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';"
165+
mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL query_response_time_stats=ON;"
148166
fi
149167
fi
150-
if [ $(( ${j} % 2 )) -eq 0 ]; then
151-
pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-prod --cluster=ps-prod-cluster --replication-set=ps-repl2 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port
168+
if [[ "$port" == "$master_port" ]]; then
169+
pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-master-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$master_port
152170
else
153-
pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port
171+
pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-slave-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$slave_port
154172
fi
155173
#run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-multiple-node
156-
node_port=$(($node_port + 1))
157174
sleep 20
158175
done
159176
fi

‎pmm_qa/pmm-framework.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None):
190190
'PS_NODES': no_of_nodes,
191191
'PS_VERSION': ps_version,
192192
'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1',
193-
'PS_CONTAINER': 'ps_pmm_' + str(ps_version),
193+
'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''),
194+
'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317,
194195
'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config),
195196
'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config),
196197
'PS_TARBALL': get_value('TARBALL', db_type, args, db_config),

‎pmm_qa/ps_pmm_setup.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}"
1616
pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}"
1717
number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_NODES') | default('3', true) ) }}"
18+
ps_port: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_PORT') | default('3317', true) ) }}"
1819
group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION') | default('', true) ) }}"
1920

2021
tasks:
@@ -32,7 +33,7 @@
3233
- name: Prepare Container for ps and Proxysql
3334
shell: >
3435
docker run -d --name={{ ps_container }}
35-
-p 3317:3307
36+
-p {{ ps_port }}:3307
3637
phusion/baseimage:jammy-1.0.1
3738
3839
- name: Copy all required Artifacts to the docker ps_container
@@ -68,5 +69,5 @@
6869
- name: Execute Setup script inside the ps ps_container
6970
shell: "{{ item }}"
7071
with_items:
71-
- docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ps_{{ ps_version }}.log
72+
- docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ps_{{ ps_container }}.log
7273

‎pmm_qa/setup_docker_client_images.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,9 @@ echo "Please wait...."
77
docker network create docker-client-check || true
88
docker compose -f docker-compose-clients.yaml down -v --remove-orphans
99
docker compose -f docker-compose-clients.yaml build --no-cache
10-
docker compose -f docker-compose-clients.yaml up -d
10+
CLIENT_DOCKER_VERSION=$CLIENT_DOCKER_VERSION docker compose -f docker-compose-clients.yaml up -d
1111
sleep 20
1212
echo "Adding DB Clients to PMM Server"
1313
docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
1414
docker exec pmm-client-1 pmm-admin add postgresql --query-source=pgstatements --username=pmm --password=pmm-pass --service-name=pdpgsql-1 --host=pdpgsql-1 --port=5432 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
1515
docker exec pmm-client-1 pmm-admin add mongodb --username=pmm --password=pmm-pass --service-name=mongodb-7.0 --host=psmdb-1 --port=27017 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
16-

0 commit comments

Comments
 (0)