Skip to content

Commit 7c67feb

Browse files
adk3798ShwetaBhosale1
adk3798
authored and
ShwetaBhosale1
committed
mgr/cephadm: add tcp mode for ingress over RGW
In order to allow TLS for the full path client -> haproxy -> RGW, give the option for the ingress service to just serve in tcp mode and pass received data directly to the backend servers. Makes no difference for the ingress over nfs case that already used tcp mode. Signed-off-by: Adam King <[email protected]>
1 parent 6d0b137 commit 7c67feb

File tree

4 files changed

+95
-4
lines changed

4 files changed

+95
-4
lines changed

‎src/pybind/mgr/cephadm/services/ingress.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ def haproxy_generate_config(
203203
'port': 0,
204204
})
205205
else:
206-
mode = 'http'
206+
mode = 'tcp' if spec.use_tcp_mode_over_rgw else 'http'
207207
servers = [
208208
{
209209
'name': d.name(),

‎src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ frontend frontend
6868
bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem {{ v4v6_flag }}
6969
{% else %}
7070
bind {{ ip }}:{{ frontend_port }} {{ v4v6_flag }}
71+
{% endif %}
72+
{% if mode == 'tcp' %}
73+
option tcplog
7174
{% endif %}
7275
default_backend backend
7376

@@ -80,16 +83,24 @@ backend backend
8083
{% endif %}
8184
balance static-rr
8285
option httpchk HEAD / HTTP/1.0
83-
{% for server in servers %}
84-
server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }}
85-
{% endfor %}
8686
{% endif %}
8787
{% if mode == 'tcp' %}
8888
mode tcp
8989
balance roundrobin
90+
{% if spec.use_tcp_mode_over_rgw %}
91+
{% if backend_spec.ssl %}
92+
option ssl-hello-chk
93+
{% endif %}
94+
{% endif %}
9095
{% if default_server_opts %}
9196
default-server {{ default_server_opts|join(" ") }}
9297
{% endif %}
98+
{% endif %}
99+
{% if backend_spec.service_type == 'rgw' %}
100+
{% for server in servers %}
101+
server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }}
102+
{% endfor %}
103+
{% else %}
93104
{% for server in servers %}
94105
server {{ server.name }} {{ server.ip }}:{{ server.port }} check
95106
{% endfor %}

‎src/pybind/mgr/cephadm/tests/test_services.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2783,6 +2783,7 @@ def fake_get_addr(hostname: str) -> str:
27832783
' monitor-uri /health\n\n'
27842784
'frontend frontend\n'
27852785
' bind 192.168.122.100:2049\n'
2786+
' option tcplog\n'
27862787
' default_backend backend\n\n'
27872788
'backend backend\n'
27882789
' mode tcp\n'
@@ -3095,6 +3096,82 @@ def test_ingress_config_ssl_rgw(self, _run_cephadm, cephadm_module: CephadmOrche
30953096
exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()]
30963097
assert gen_config_lines == exp_config_lines
30973098

3099+
@patch("cephadm.serve.CephadmServe._run_cephadm")
3100+
def test_haproxy_config_rgw_tcp_mode(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
3101+
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
3102+
with with_host(cephadm_module, 'test'):
3103+
cephadm_module.cache.update_host_networks('test', {
3104+
'1.2.3.0/24': {
3105+
'if0': ['1.2.3.1']
3106+
}
3107+
})
3108+
3109+
# the ingress backend
3110+
s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1),
3111+
rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True)
3112+
3113+
ispec = IngressSpec(service_type='ingress',
3114+
service_id='test',
3115+
backend_service='rgw.foo',
3116+
frontend_port=8089,
3117+
monitor_port=8999,
3118+
monitor_user='admin',
3119+
monitor_password='12345',
3120+
virtual_interface_networks=['1.2.3.0/24'],
3121+
virtual_ip="1.2.3.4/32",
3122+
use_tcp_mode_over_rgw=True)
3123+
with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _:
3124+
# generate the haproxy conf based on the specified spec
3125+
haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config(
3126+
CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name()))
3127+
3128+
haproxy_expected_conf = {
3129+
'files':
3130+
{
3131+
'haproxy.cfg':
3132+
'# This file is generated by cephadm.'
3133+
'\nglobal\n log '
3134+
'127.0.0.1 local2\n '
3135+
'chroot /var/lib/haproxy\n '
3136+
'pidfile /var/lib/haproxy/haproxy.pid\n '
3137+
'maxconn 8000\n '
3138+
'daemon\n '
3139+
'stats socket /var/lib/haproxy/stats\n'
3140+
'\ndefaults\n '
3141+
'mode tcp\n '
3142+
'log global\n '
3143+
'timeout queue 1m\n '
3144+
'timeout connect 10s\n '
3145+
'timeout client 1m\n '
3146+
'timeout server 1m\n '
3147+
'timeout check 10s\n '
3148+
'maxconn 8000\n'
3149+
'\nfrontend stats\n '
3150+
'mode http\n '
3151+
'bind 1.2.3.4:8999\n '
3152+
'bind 1::4:8999\n '
3153+
'stats enable\n '
3154+
'stats uri /stats\n '
3155+
'stats refresh 10s\n '
3156+
'stats auth admin:12345\n '
3157+
'http-request use-service prometheus-exporter if { path /metrics }\n '
3158+
'monitor-uri /health\n'
3159+
'\nfrontend frontend\n '
3160+
'bind 1.2.3.4:8089 \n '
3161+
'option tcplog\n '
3162+
'default_backend backend\n\n'
3163+
'backend backend\n '
3164+
'mode tcp\n '
3165+
'balance roundrobin\n '
3166+
'hash-type consistent\n '
3167+
'option ssl-hello-chk\n '
3168+
'server '
3169+
+ haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n'
3170+
}
3171+
}
3172+
3173+
assert haproxy_generated_conf[0] == haproxy_expected_conf
3174+
30983175
@patch("cephadm.serve.CephadmServe._run_cephadm")
30993176
def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
31003177
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
@@ -3602,6 +3679,7 @@ def fake_keys():
36023679
' monitor-uri /health\n\n'
36033680
'frontend frontend\n'
36043681
' bind 192.168.122.100:2049\n'
3682+
' option tcplog\n'
36053683
' default_backend backend\n\n'
36063684
'backend backend\n'
36073685
' mode tcp\n'

‎src/python-common/ceph/deployment/service_spec.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2213,6 +2213,7 @@ def __init__(self,
22132213
monitor_cert_source: Optional[str] = MonitorCertSource.REUSE_SERVICE_CERT.value,
22142214
monitor_networks: Optional[List[str]] = None,
22152215
monitor_ip_addrs: Optional[Dict[str, str]] = None,
2216+
use_tcp_mode_over_rgw: bool = False,
22162217
):
22172218
assert service_type == 'ingress'
22182219

@@ -2257,6 +2258,7 @@ def __init__(self,
22572258
self.monitor_cert_source = monitor_cert_source
22582259
self.monitor_networks = monitor_networks
22592260
self.monitor_ip_addrs = monitor_ip_addrs
2261+
self.use_tcp_mode_over_rgw = use_tcp_mode_over_rgw
22602262

22612263
def get_port_start(self) -> List[int]:
22622264
ports = []

0 commit comments

Comments
 (0)