Skip to content

Commit 90d97c0

Browse files
Zuulopenstack-gerrit
authored andcommitted
Merge "Add proxysql image"
2 parents 1dc1041 + febbb5b commit 90d97c0

File tree

9 files changed

+273
-0
lines changed

9 files changed

+273
-0
lines changed

docker/base/Dockerfile.j2

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ COPY dnf.conf /etc/dnf/dnf.conf
7676
'elasticsearch.repo',
7777
'grafana.repo',
7878
'influxdb.repo',
79+
'proxysql.repo',
7980
'rabbitmq_rabbitmq-server.repo',
8081
'rabbitmq_rabbitmq-erlang.repo',
8182
'td.repo',
@@ -89,18 +90,21 @@ COPY dnf.conf /etc/dnf/dnf.conf
8990
'https://repos.influxdata.com/influxdb.key',
9091
'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc',
9192
'https://packages.treasuredata.com/GPG-KEY-td-agent',
93+
'https://repo.proxysql.com/ProxySQL/repo_pub_key',
9294
] %}
9395
{% elif base_arch == 'aarch64' %}
9496
{% set base_yum_repo_files = [
9597
'elasticsearch.repo',
9698
'grafana.repo',
99+
'proxysql.repo',
97100
'rabbitmq_rabbitmq-server.repo',
98101
'td.repo',
99102
] %}
100103

101104
{% set base_yum_repo_keys = [
102105
'https://packages.grafana.com/gpg.key',
103106
'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc',
107+
'https://repo.proxysql.com/ProxySQL/repo_pub_key',
104108
] %}
105109
# FIXME(mgoddard): Not available for CentOS 8 yet.
106110
#
@@ -328,6 +332,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
328332
{'name': 'grafana', 'url': 'https://packages.grafana.com/gpg.key'},
329333
{'name': 'influxdb', 'url': 'https://repos.influxdata.com/influxdb.key'},
330334
{'name': 'mariadb', 'url': 'https://downloads.mariadb.com/MariaDB/mariadb-keyring-2019.gpg', 'type': 'gpg'},
335+
{'name': 'proxysql', 'url': 'https://repo.proxysql.com/ProxySQL/repo_pub_key'},
331336
{'name': 'treasuredata', 'url': 'https://packages.treasuredata.com/GPG-KEY-td-agent'},
332337
] %}
333338

docker/base/proxysql.repo

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[proxysql]
2+
name = ProxySQL
3+
baseurl = https://repo.proxysql.com/ProxySQL/proxysql-2.3.x/centos/$releasever
4+
gpgkey = https://repo.proxysql.com/ProxySQL/repo_pub_key
5+
gpgcheck = 1
6+
enabled = 0

docker/proxysql/Dockerfile.j2

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
FROM {{ namespace }}/{{ infra_image_prefix }}base:{{ tag }}
2+
{% block labels %}
3+
LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
4+
{% endblock %}
5+
6+
{% block proxysql_header %}{% endblock %}
7+
8+
{% import "macros.j2" as macros with context %}
9+
10+
{{ macros.enable_extra_repos(['proxysql']) }}
11+
12+
{% set proxysql_packages = [
13+
'proxysql',
14+
] %}
15+
16+
{% if base_package_type == 'rpm' %}
17+
{% set proxysql_packages = proxysql_packages + [
18+
'python3-PyYAML',
19+
] %}
20+
{% elif base_package_type == 'deb' %}
21+
{% set proxysql_packages = proxysql_packages + [
22+
'python3-yaml',
23+
] %}
24+
{% endif %}
25+
26+
{{ macros.configure_user(name='proxysql') }}
27+
{{ macros.install_packages(proxysql_packages | customizable("packages")) }}
28+
29+
COPY kolla_proxysql_start kolla_proxysql_config_sync kolla_extend_start /usr/local/bin/
30+
RUN chmod 755 /usr/local/bin/kolla_proxysql_start \
31+
&& chmod 755 /usr/local/bin/kolla_proxysql_config_sync \
32+
&& chmod 755 /usr/local/bin/kolla_extend_start
33+
34+
{% block proxysql_footer %}{% endblock %}
35+
{% block footer %}{% endblock %}

docker/proxysql/kolla_extend_start

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
PROXYSQL_LIB_DIR="/var/lib/proxysql"
4+
PROXYSQL_LOG_DIR="/var/log/kolla/proxysql"
5+
6+
if [[ ! -d "${PROXYSQL_LOG_DIR}" ]]; then
7+
mkdir -p "${PROXYSQL_LOG_DIR}"
8+
fi
9+
chown -R proxysql:kolla "${PROXYSQL_LOG_DIR}"
10+
11+
# Proxysql has configuration stored in
12+
# its own internal DB and config file
13+
# is used only for bootstrap DB by default.
14+
# Once DB is bootstraped, config
15+
# file is ignored.
16+
#
17+
# This behaviour is not what we want
18+
# while configuring via kolla-ansible.
19+
# Therefore let's remove internal DB
20+
# and bootstrap it every time as config
21+
# file is generated on kolla-ansible side.
22+
rm -f ${PROXYSQL_LIB_DIR}/proxysql.db
23+
24+
# Remove old pid
25+
rm -f ${PROXYSQL_LIB_DIR}/proxysql.pid
26+
27+
# As proxysql supports only one
28+
# configuration file, there is a
29+
# script which reads global and services
30+
# configs (similar to haproxy) and
31+
# generate single configuration file
32+
# from which is internal DB created
33+
# every time.
34+
kolla_proxysql_config_sync
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/usr/bin/python3
2+
3+
# Copyright 2021 Michal Arbet
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import logging
18+
import os
19+
import yaml
20+
21+
# Default paths
22+
PROXYSQL_CONFIG_DIR = "/etc/proxysql"
23+
PROXYSQL_CONFIG = "/etc/proxysql.cnf"
24+
25+
# Logging
26+
log_format = '%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s'
27+
logging.basicConfig(format=log_format,
28+
datefmt='%H:%M:%S',
29+
level=logging.DEBUG)
30+
LOG = logging.getLogger("proxysql_config_sync")
31+
32+
33+
class ProxySQLConfig:
34+
35+
def __init__(self, conf_dir, conf_file):
36+
self.configs = dict()
37+
self.config = dict()
38+
self.configs['global'] = "{}/proxysql.yaml".format(conf_dir)
39+
self.configs['users'] = "{}/users".format(conf_dir)
40+
self.configs['rules'] = "{}/rules".format(conf_dir)
41+
self.conf_file = conf_file
42+
self._load_config()
43+
44+
def _load_config(self):
45+
users = dict()
46+
rules = dict()
47+
48+
for cfg, path in self.configs.items():
49+
if not os.path.isdir(path):
50+
with open(path) as config_file:
51+
self.config.update(
52+
yaml.safe_load(config_file))
53+
else:
54+
users['mysql_users'] = list()
55+
rules['mysql_query_rules'] = list()
56+
user_paths = [os.path.join(self.configs['users'], f)
57+
for f in os.listdir(self.configs['users'])]
58+
rule_paths = [os.path.join(self.configs['rules'], f)
59+
for f in os.listdir(self.configs['rules'])]
60+
61+
for user_conf in user_paths:
62+
with open(user_conf) as config_file:
63+
tmp_users = yaml.safe_load(config_file)
64+
for i in tmp_users['mysql_users']:
65+
users['mysql_users'].append(i)
66+
self.config.update(users)
67+
for rule_conf in rule_paths:
68+
with open(rule_conf) as config_file:
69+
tmp_rules = yaml.safe_load(config_file)
70+
for i in tmp_rules['mysql_query_rules']:
71+
rules['mysql_query_rules'].append(i)
72+
self.config.update(rules)
73+
self._sanity()
74+
75+
def _sanity(self):
76+
self._users_sanity()
77+
self._rules_sanity()
78+
79+
def _users_sanity(self):
80+
users_added = list()
81+
users = list()
82+
for user in self.config['mysql_users']:
83+
if user['username'] not in users_added:
84+
users_added.append(user['username'])
85+
users.append(user)
86+
else:
87+
LOG.warning("User {} already exist, ignoring."
88+
.format(user['username']))
89+
self.config['mysql_users'] = users
90+
91+
def _rules_sanity(self):
92+
rules_added = list()
93+
rules = list()
94+
rule_id = 1
95+
for rule in self.config['mysql_query_rules']:
96+
if rule['schemaname'] not in rules_added:
97+
rules_added.append(rule['schemaname'])
98+
rule['rule_id'] = rule_id
99+
rules.append(rule)
100+
rule_id += 1
101+
else:
102+
LOG.warning("Rule witch schemaname {} already exist, ignoring."
103+
.format(rule['schemaname']))
104+
self.config['mysql_query_rules'] = rules
105+
106+
def _write_dict(self, key, value):
107+
if not isinstance(value, list):
108+
value = [value]
109+
with open(self.conf_file, "a+") as f:
110+
if key:
111+
f.write("{} =\n".format(key))
112+
for i in range(len(value)):
113+
f.write(" {\n")
114+
for k, v in value[i].items():
115+
if isinstance(v, str):
116+
v = '"{}"'.format(v)
117+
f.write(" {} = {}\n".format(k, v))
118+
if i == len(value)-1:
119+
f.write(" }\n")
120+
else:
121+
f.write(" },\n")
122+
123+
def _write_list(self, key, values):
124+
with open(self.conf_file, "a+") as f:
125+
f.write("{} =\n".format(key))
126+
f.write("(\n")
127+
self._write_dict(key=None, value=values)
128+
with open(self.conf_file, "a+") as f:
129+
f.write(")\n")
130+
131+
def _write(self, key, value):
132+
with open(self.conf_file, "a+") as f:
133+
if isinstance(value, str):
134+
value = '"{}"'.format(value)
135+
f.write("{} = {}\n".format(key, value))
136+
137+
def write_config(self):
138+
LOG.info("Writing config to {}".format(self.conf_file))
139+
if os.path.exists(self.conf_file):
140+
os.remove(self.conf_file)
141+
for k, v in self.config.items():
142+
if isinstance(v, dict):
143+
self._write_dict(k, v)
144+
elif isinstance(v, list):
145+
self._write_list(k, v)
146+
else:
147+
self._write(k, v)
148+
149+
150+
if __name__ == "__main__":
151+
config = ProxySQLConfig(PROXYSQL_CONFIG_DIR, PROXYSQL_CONFIG)
152+
config.write_config()
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
PROXYSQL_LOG_FILE="/var/log/kolla/proxysql/proxysql.log"
4+
5+
proxysql \
6+
--exit-on-error \
7+
--idle-threads \
8+
--no-version-check \
9+
-f -D /var/lib/proxysql -c /etc/proxysql.cnf >> ${PROXYSQL_LOG_FILE} 2>&1

kolla/common/config.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
'mariadb',
6363
'memcached',
6464
'openvswitch',
65+
'proxysql',
6566
'qdrouterd',
6667
'rabbitmq',
6768
'redis',
@@ -133,6 +134,7 @@
133134
'neutron',
134135
'nova-',
135136
'placement',
137+
'proxysql',
136138
'openvswitch',
137139
'rabbitmq',
138140
],
@@ -944,6 +946,10 @@
944946
'uid': 42486,
945947
'gid': 42486,
946948
'group': 'haclient',
949+
},
950+
'proxysql-user': {
951+
'uid': 42487,
952+
'gid': 42487,
947953
}
948954
}
949955

kolla/template/repos.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ centos:
1616
openvswitch: "centos-nfv-openvswitch"
1717
opstools: "centos-opstools"
1818
powertools: "powertools"
19+
proxysql: "proxysql"
1920
rabbitmq: "rabbitmq_rabbitmq-server"
2021
td-agent: "treasuredata"
2122

@@ -35,6 +36,7 @@ centos-aarch64:
3536
openvswitch: "centos-nfv-openvswitch"
3637
opstools: "centos-opstools"
3738
powertools: "powertools"
39+
proxysql: "proxysql"
3840
rabbitmq: "rabbitmq_rabbitmq-server"
3941
td-agent: "treasuredata"
4042

@@ -85,6 +87,11 @@ debian:
8587
suite: "bullseye"
8688
component: "contrib"
8789
gpg_key: "treasuredata.asc"
90+
proxysql:
91+
url: "https://repo.proxysql.com/ProxySQL/proxysql-2.3.x/bullseye/"
92+
suite: "./"
93+
component: ""
94+
gpg_key: "proxysql.asc"
8895

8996
debian-aarch64:
9097
elasticsearch:
@@ -133,6 +140,11 @@ debian-aarch64:
133140
suite: "bullseye"
134141
component: "contrib"
135142
gpg_key: "treasuredata.asc"
143+
proxysql:
144+
url: "https://repo.proxysql.com/ProxySQL/proxysql-2.3.x/bullseye/"
145+
suite: "./"
146+
component: ""
147+
gpg_key: "proxysql.asc"
136148

137149
ubuntu:
138150
elasticsearch:
@@ -185,6 +197,11 @@ ubuntu:
185197
suite: "focal"
186198
component: "contrib"
187199
gpg_key: "treasuredata.asc"
200+
proxysql:
201+
url: "https://repo.proxysql.com/ProxySQL/proxysql-2.3.x/focal/"
202+
suite: "./"
203+
component: ""
204+
gpg_key: "proxysql.asc"
188205

189206
ubuntu-aarch64:
190207
elasticsearch:
@@ -232,3 +249,8 @@ ubuntu-aarch64:
232249
suite: "focal"
233250
component: "contrib"
234251
gpg_key: "treasuredata.asc"
252+
proxysql:
253+
url: "https://repo.proxysql.com/ProxySQL/proxysql-2.3.x/focal/"
254+
suite: "./"
255+
component: ""
256+
gpg_key: "proxysql.asc"
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
features:
3+
- Adds proxysql image. Proxysql provides
4+
intelligent load balancing for databases.

0 commit comments

Comments
 (0)