Skip to content

Commit 300fa0c

Browse files
Zuulopenstack-gerrit
Zuul
authored and
openstack-gerrit
committed
Merge "Add Health monitor sync logic"
2 parents b814516 + 615bb01 commit 300fa0c

File tree

4 files changed

+866
-44
lines changed

4 files changed

+866
-44
lines changed

‎ovn_octavia_provider/driver.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,21 @@ def _get_member_request_info(self, member, create=True):
175175

176176
return request_info
177177

178+
def _get_healthmonitor_request_info(self, healthmonitor):
179+
self._validate_hm_support(healthmonitor)
180+
admin_state_up = healthmonitor.admin_state_up
181+
if isinstance(admin_state_up, o_datamodels.UnsetType):
182+
admin_state_up = True
183+
request_info = {'id': healthmonitor.healthmonitor_id,
184+
'pool_id': healthmonitor.pool_id,
185+
'type': healthmonitor.type,
186+
'interval': healthmonitor.delay,
187+
'timeout': healthmonitor.timeout,
188+
'failure_count': healthmonitor.max_retries_down,
189+
'success_count': healthmonitor.max_retries,
190+
'admin_state_up': admin_state_up}
191+
return request_info
192+
178193
def loadbalancer_create(self, loadbalancer):
179194
request = {'type': ovn_const.REQ_TYPE_LB_CREATE,
180195
'info': self._get_loadbalancer_request_info(
@@ -691,9 +706,19 @@ def _ensure_loadbalancer(self, loadbalancer):
691706
status_member = self._ovn_helper.member_create(
692707
self._get_member_request_info(member))
693708
status[constants.MEMBERS].append(status_member)
694-
709+
if pool.healthmonitor is not None and not isinstance(
710+
pool.healthmonitor, o_datamodels.UnsetType):
711+
status[constants.HEALTHMONITORS] = []
712+
lbhcs, ovn_hm_lb = (
713+
self._ovn_helper._find_ovn_lb_from_hm_id(
714+
pool.healthmonitor.healthmonitor_id)
715+
)
716+
if not lbhcs and ovn_hm_lb is None:
717+
status_hm = self._ovn_helper.hm_create(
718+
self._get_healthmonitor_request_info(
719+
pool.healthmonitor))
720+
status[constants.HEALTHMONITORS].append(status_hm)
695721
self._ovn_helper._update_status_to_octavia(status)
696-
697722
else:
698723
# Load Balancer found, check LB and listener/pool/member/hms
699724
# related
@@ -759,6 +784,16 @@ def _ensure_loadbalancer(self, loadbalancer):
759784
}
760785
self._ovn_helper.handle_member_dvr(
761786
mb_delete_dvr_info)
787+
# Check health monitor
788+
if pool.healthmonitor is not None and not isinstance(
789+
pool.healthmonitor, o_datamodels.UnsetType):
790+
self._ovn_helper.hm_sync(
791+
self._get_healthmonitor_request_info(
792+
pool.healthmonitor),
793+
ovn_lb,
794+
ovn_pool_key)
795+
# Purge HM
796+
self._ovn_helper.hm_purge(loadbalancer.loadbalancer_id)
762797
status = self._ovn_helper._get_current_operating_statuses(
763798
ovn_lb)
764799
self._ovn_helper._update_status_to_octavia(status)
@@ -792,6 +827,14 @@ def do_sync(self, **lb_filters):
792827
for m in members]
793828
else:
794829
provider_pool.members = o_datamodels.Unset
830+
831+
# format healthmonitor provider
832+
if not isinstance(
833+
provider_pool.healthmonitor, o_datamodels.UnsetType
834+
) and provider_pool.healthmonitor is not None:
835+
provider_pool.healthmonitor = \
836+
o_datamodels.HealthMonitor.from_dict(
837+
provider_pool.healthmonitor)
795838
provider_pools.append(provider_pool)
796839

797840
provider_lb.pools = (

‎ovn_octavia_provider/helper.py‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3268,6 +3268,184 @@ def _add_lbhc(self, ovn_lb, pool_key, info):
32683268
LOG.exception(ovn_const.EXCEPTION_MSG, "set of health check")
32693269
return status
32703270

3271+
def _sync_lbhc(self, ovn_lb, pool_key, hm):
3272+
hm_id = hm[constants.ID]
3273+
# Example
3274+
# MONITOR_PRT = 80
3275+
# ID=$(ovn-nbctl --bare --column _uuid find
3276+
# Load_Balancer_Health_Check vip="${LB_VIP_ADDR}\:${MONITOR_PRT}")
3277+
# In our case the monitor port will be the members protocol port
3278+
vips = []
3279+
if ovn_const.LB_EXT_IDS_VIP_KEY in ovn_lb.external_ids:
3280+
vips.append(ovn_lb.external_ids.get(ovn_const.LB_EXT_IDS_VIP_KEY))
3281+
if ovn_const.LB_EXT_IDS_ADDIT_VIP_KEY in ovn_lb.external_ids:
3282+
vips.extend(ovn_lb.external_ids.get(
3283+
ovn_const.LB_EXT_IDS_ADDIT_VIP_KEY).split(','))
3284+
fips = []
3285+
if ovn_const.LB_EXT_IDS_VIP_FIP_KEY in ovn_lb.external_ids:
3286+
fips.append(ovn_lb.external_ids.get(
3287+
ovn_const.LB_EXT_IDS_VIP_FIP_KEY))
3288+
if ovn_const.LB_EXT_IDS_ADDIT_VIP_FIP_KEY in ovn_lb.external_ids:
3289+
fips.extend(ovn_lb.external_ids.get(
3290+
ovn_const.LB_EXT_IDS_ADDIT_VIP_FIP_KEY).split(','))
3291+
if not vips:
3292+
msg = (f"Could not find VIP for HM {hm_id}, LB external_ids: "
3293+
f"{ovn_lb.external_ids}")
3294+
raise driver_exceptions.DriverError(msg)
3295+
3296+
vip_port = self._get_pool_listener_port(ovn_lb, pool_key)
3297+
3298+
# This is to enable lookups by Octavia DB ID value
3299+
external_ids = {
3300+
ovn_const.LB_EXT_IDS_HM_KEY: hm_id,
3301+
ovn_const.LB_EXT_IDS_HM_POOL_KEY: pool_key[
3302+
len(ovn_const.LB_EXT_IDS_POOL_PREFIX):],
3303+
}
3304+
3305+
options = {
3306+
'interval': str(hm['interval']),
3307+
'timeout': str(hm['timeout']),
3308+
'success_count': str(hm['success_count']),
3309+
'failure_count': str(hm['failure_count'])}
3310+
3311+
try:
3312+
with self.ovn_nbdb_api.transaction(check_error=True) as txn:
3313+
for vip in vips:
3314+
recreate = False
3315+
# Just seems like this needs ovsdbapp support, see:
3316+
# ovsdbapp/schema/ovn_northbound/impl_idl.py
3317+
# - lb_add()
3318+
# ovsdbapp/schema/ovn_northbound/commands.py
3319+
# - LbAddCommand()
3320+
# then this could just be self.ovn_nbdb_api.lb_hm_add()
3321+
external_ids_vip = copy.deepcopy(external_ids)
3322+
external_ids_vip[ovn_const.LB_EXT_IDS_HM_VIP] = vip
3323+
if netaddr.IPNetwork(vip).version == n_const.IP_VERSION_6:
3324+
vip = f'[{vip}]'
3325+
kwargs = {
3326+
'vip': vip + ':' + str(vip_port) if vip_port else '',
3327+
'options': options,
3328+
'external_ids': external_ids_vip}
3329+
3330+
hms_key = ovn_lb.external_ids.get(
3331+
ovn_const.LB_EXT_IDS_HMS_KEY, [])
3332+
if hms_key:
3333+
hms_key = jsonutils.loads(hms_key)
3334+
lbhcs, _ = self._find_ovn_lb_from_hm_id(hm_id)
3335+
if not lbhcs:
3336+
recreate = True
3337+
for lbhc in lbhcs:
3338+
commands = []
3339+
if lbhc.vip != kwargs.get('vip'):
3340+
commands.append(
3341+
self.ovn_nbdb_api.db_set(
3342+
'Load_Balancer_Health_Check',
3343+
lbhc.uuid,
3344+
('vip', kwargs.get('vip'))))
3345+
if lbhc.options != options:
3346+
commands.append(
3347+
self.ovn_nbdb_api.db_set(
3348+
'Load_Balancer_Health_Check',
3349+
lbhc.uuid,
3350+
('options', options)))
3351+
if lbhc.external_ids != external_ids_vip:
3352+
commands.append(
3353+
self.ovn_nbdb_api.db_set(
3354+
'Load_Balancer_Health_Check',
3355+
lbhc.uuid,
3356+
('external_ids', external_ids_vip)))
3357+
3358+
found_in_exist = False
3359+
for hc in ovn_lb.health_check:
3360+
if str(hc.uuid) == str(lbhc.uuid):
3361+
found_in_exist = True
3362+
break
3363+
if not found_in_exist:
3364+
commands.append(
3365+
self.ovn_nbdb_api.db_add(
3366+
'Load_Balancer', ovn_lb.uuid,
3367+
('health_check', lbhc.uuid)))
3368+
if hm_id not in hms_key:
3369+
hms_key.append(hm_id)
3370+
commands.append(self.ovn_nbdb_api.db_set(
3371+
'Load_Balancer', ovn_lb.uuid,
3372+
('external_ids',
3373+
{ovn_const.LB_EXT_IDS_HMS_KEY:
3374+
jsonutils.dumps(hms_key)})))
3375+
self._execute_commands(commands)
3376+
if recreate:
3377+
health_check = txn.add(
3378+
self.ovn_nbdb_api.db_create(
3379+
'Load_Balancer_Health_Check',
3380+
**kwargs))
3381+
txn.add(self.ovn_nbdb_api.db_set(
3382+
'Load_Balancer_Health_Check',
3383+
health_check,
3384+
('vip', kwargs.get('vip'))))
3385+
txn.add(self.ovn_nbdb_api.db_add(
3386+
'Load_Balancer', ovn_lb.uuid,
3387+
'health_check', health_check))
3388+
if hm_id not in hms_key:
3389+
hms_key.append(hm_id)
3390+
txn.add(self.ovn_nbdb_api.db_set(
3391+
'Load_Balancer', ovn_lb.uuid,
3392+
('external_ids', {ovn_const.LB_EXT_IDS_HMS_KEY:
3393+
jsonutils.dumps(hms_key)})))
3394+
if fips:
3395+
external_ids_fip = copy.deepcopy(external_ids)
3396+
for fip in fips:
3397+
recreate = False
3398+
external_ids_fip[ovn_const.LB_EXT_IDS_HM_VIP] = fip
3399+
if netaddr.IPNetwork(
3400+
fip).version == n_const.IP_VERSION_6:
3401+
fip = f'[{fip}]'
3402+
fip_kwargs = {
3403+
'vip': fip + ':' + str(vip_port)
3404+
if vip_port
3405+
else '',
3406+
'options': options,
3407+
'external_ids': external_ids_fip}
3408+
3409+
lbhcs, _ = self._find_ovn_lb_from_hm_id(hm_id)
3410+
if not lbhcs:
3411+
recreate = True
3412+
for lbhc in lbhcs:
3413+
commands = []
3414+
if lbhc.vip != fip_kwargs['vip']:
3415+
commands.append(
3416+
self.ovn_nbdb_api.db_set(
3417+
'Load_Balancer_Health_Check',
3418+
lbhc.uuid,
3419+
('vip', fip_kwargs['vip'])))
3420+
if lbhc.options != options:
3421+
commands.append(
3422+
self.ovn_nbdb_api.db_set(
3423+
'Load_Balancer_Health_Check',
3424+
lbhc.uuid,
3425+
('options', options)))
3426+
if lbhc.external_ids != external_ids_vip:
3427+
commands.append(
3428+
self.ovn_nbdb_api.db_set(
3429+
'Load_Balancer_Health_Check',
3430+
lbhc.uuid,
3431+
('external_ids', external_ids_fip)))
3432+
self._execute_commands(commands)
3433+
if recreate:
3434+
fip_health_check = txn.add(
3435+
self.ovn_nbdb_api.db_create(
3436+
'Load_Balancer_Health_Check',
3437+
**fip_kwargs))
3438+
txn.add(self.ovn_nbdb_api.db_set(
3439+
'Load_Balancer_Health_Check',
3440+
fip_health_check,
3441+
('vip', fip_kwargs['vip'])))
3442+
txn.add(self.ovn_nbdb_api.db_add(
3443+
'Load_Balancer', ovn_lb.uuid,
3444+
'health_check', fip_health_check))
3445+
except Exception as e:
3446+
msg = (f"Error syncing Load Balancer Health Check: {e}")
3447+
raise driver_exceptions.DriverError(msg)
3448+
32713449
def _update_lbhc_vip_port(self, lbhc, vip_port):
32723450
if lbhc.vip:
32733451
vip = lbhc.vip.rsplit(":")[0] + ':' + str(vip_port)
@@ -3494,6 +3672,46 @@ def hm_create(self, info):
34943672
status[constants.HEALTHMONITORS] = [hm_status]
34953673
return status
34963674

3675+
def hm_sync(self, hm, ovn_lb, pool_key):
3676+
"""Sync Health Monitor object with an OVN LoadBalancer
3677+
3678+
The method performs the following steps:
3679+
1. Create Health Monitor in OVN NB in case we don't find
3680+
load_balancer_health_checks entries associated
3681+
2. If we found load_balancer_health_checks entries associated
3682+
2.1. Update member status affected on OVN loadbalancer external_ids
3683+
2.2. Sync OVN load_balancer_health_checks entries
3684+
2.3. Update OVN Loadbalancer ip_port_mappings
3685+
2.4. Update OVN Loadbalancer member_status info on external_ids
3686+
3687+
:param hm: The source health monitor object from Octavia DB
3688+
:param ovn_lb: The OVN LoadBalancer object that needs to be sync
3689+
:param pool_key: The pool_key where health monitor is associated
3690+
"""
3691+
lbhcs, ovn_lb = self._find_ovn_lb_from_hm_id(hm[constants.ID])
3692+
if not lbhcs:
3693+
LOG.debug("Loadbalancer health check %s not found!",
3694+
hm[constants.ID])
3695+
# Create in case we don't found it
3696+
self.hm_create(hm)
3697+
return
3698+
3699+
pool_id = hm[constants.POOL_ID]
3700+
self._update_member_statuses(ovn_lb, pool_id, constants.ACTIVE,
3701+
constants.ONLINE)
3702+
try:
3703+
self._sync_lbhc(ovn_lb, pool_key, hm)
3704+
except Exception as e:
3705+
LOG.exception(f"Failed syncing Load Balancer Health Monitor: {e}")
3706+
3707+
for mb_ip, mb_port, mb_subnet, mb_id in self._extract_member_info(
3708+
ovn_lb.external_ids[pool_key]):
3709+
mb_status = self._update_hm_member(ovn_lb, pool_key, mb_ip)
3710+
if not mb_status:
3711+
self._clean_ip_port_mappings(ovn_lb, pool_key)
3712+
break
3713+
self._update_external_ids_member_status(ovn_lb, mb_id, mb_status)
3714+
34973715
def hm_update(self, info):
34983716
status = {
34993717
constants.HEALTHMONITORS: [
@@ -3870,3 +4088,42 @@ def hm_update_event(self, info):
38704088
status[k].extend(status_lb[k])
38714089

38724090
return status
4091+
4092+
def hm_purge(self, lb_id):
4093+
# remove redundant hm if any, if no
4094+
# ovn_const.LB_EXT_IDS_HMS_KEY from lb matches,
4095+
# this hm has no used and should already got replaced.
4096+
ovn_lbs = []
4097+
try:
4098+
ovn_lbs = self._find_ovn_lbs_with_retry(lb_id)
4099+
except idlutils.RowNotFound:
4100+
LOG.debug(f"OVN loadbalancer {lb_id} not found.")
4101+
fetch_hc_ids = []
4102+
for ovn_lb in ovn_lbs:
4103+
hm_ids = ovn_lb.external_ids.get(
4104+
ovn_const.LB_EXT_IDS_HMS_KEY, [])
4105+
if hm_ids:
4106+
hm_ids = jsonutils.loads(hm_ids)
4107+
for hm_id in hm_ids:
4108+
lbhcs = []
4109+
try:
4110+
lbhcs = self._lookup_lbhcs_by_hm_id(hm_id)
4111+
except idlutils.RowNotFound:
4112+
continue
4113+
fetch_hc_ids.extend([str(lbhc.uuid) for lbhc in lbhcs])
4114+
4115+
for hc_id in ovn_lb.health_check:
4116+
if str(hc_id.uuid) not in fetch_hc_ids:
4117+
commands = []
4118+
commands.append(
4119+
self.ovn_nbdb_api.db_remove(
4120+
'Load_Balancer', ovn_lb.uuid,
4121+
'health_check', hc_id.uuid))
4122+
commands.append(
4123+
self.ovn_nbdb_api.db_destroy(
4124+
'Load_Balancer_Health_Check',
4125+
hc_id.uuid))
4126+
try:
4127+
self._execute_commands(commands)
4128+
except idlutils.RowNotFound:
4129+
LOG.debug("health check not found for purge.")

0 commit comments

Comments
 (0)