Skip to content

Commit 615bb01

Browse files
fernandoroyosanchezricolin
fernandoroyosanchez
and
ricolin
committed
Add Health monitor sync logic
This patch adds the logic to sync a LB health monitor entity from the Octavia database, correcting any discrepancies in fields or creating it if it does not exist in the OVN Northbound (NB) database. Related-Bug: #2045415 Co-authored-by: Fernando Royo <[email protected]> Co-authored-by: Rico Lin <[email protected]> Change-Id: I8da56617a68279adea73eb1393653bfcc4530ed0
1 parent f419488 commit 615bb01

File tree

4 files changed

+866
-44
lines changed

4 files changed

+866
-44
lines changed

‎ovn_octavia_provider/driver.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,21 @@ def _get_member_request_info(self, member, create=True):
175175

176176
return request_info
177177

178+
def _get_healthmonitor_request_info(self, healthmonitor):
179+
self._validate_hm_support(healthmonitor)
180+
admin_state_up = healthmonitor.admin_state_up
181+
if isinstance(admin_state_up, o_datamodels.UnsetType):
182+
admin_state_up = True
183+
request_info = {'id': healthmonitor.healthmonitor_id,
184+
'pool_id': healthmonitor.pool_id,
185+
'type': healthmonitor.type,
186+
'interval': healthmonitor.delay,
187+
'timeout': healthmonitor.timeout,
188+
'failure_count': healthmonitor.max_retries_down,
189+
'success_count': healthmonitor.max_retries,
190+
'admin_state_up': admin_state_up}
191+
return request_info
192+
178193
def loadbalancer_create(self, loadbalancer):
179194
request = {'type': ovn_const.REQ_TYPE_LB_CREATE,
180195
'info': self._get_loadbalancer_request_info(
@@ -691,9 +706,19 @@ def _ensure_loadbalancer(self, loadbalancer):
691706
status_member = self._ovn_helper.member_create(
692707
self._get_member_request_info(member))
693708
status[constants.MEMBERS].append(status_member)
694-
709+
if pool.healthmonitor is not None and not isinstance(
710+
pool.healthmonitor, o_datamodels.UnsetType):
711+
status[constants.HEALTHMONITORS] = []
712+
lbhcs, ovn_hm_lb = (
713+
self._ovn_helper._find_ovn_lb_from_hm_id(
714+
pool.healthmonitor.healthmonitor_id)
715+
)
716+
if not lbhcs and ovn_hm_lb is None:
717+
status_hm = self._ovn_helper.hm_create(
718+
self._get_healthmonitor_request_info(
719+
pool.healthmonitor))
720+
status[constants.HEALTHMONITORS].append(status_hm)
695721
self._ovn_helper._update_status_to_octavia(status)
696-
697722
else:
698723
# Load Balancer found, check LB and listener/pool/member/hms
699724
# related
@@ -759,6 +784,16 @@ def _ensure_loadbalancer(self, loadbalancer):
759784
}
760785
self._ovn_helper.handle_member_dvr(
761786
mb_delete_dvr_info)
787+
# Check health monitor
788+
if pool.healthmonitor is not None and not isinstance(
789+
pool.healthmonitor, o_datamodels.UnsetType):
790+
self._ovn_helper.hm_sync(
791+
self._get_healthmonitor_request_info(
792+
pool.healthmonitor),
793+
ovn_lb,
794+
ovn_pool_key)
795+
# Purge HM
796+
self._ovn_helper.hm_purge(loadbalancer.loadbalancer_id)
762797
status = self._ovn_helper._get_current_operating_statuses(
763798
ovn_lb)
764799
self._ovn_helper._update_status_to_octavia(status)
@@ -792,6 +827,14 @@ def do_sync(self, **lb_filters):
792827
for m in members]
793828
else:
794829
provider_pool.members = o_datamodels.Unset
830+
831+
# format healthmonitor provider
832+
if not isinstance(
833+
provider_pool.healthmonitor, o_datamodels.UnsetType
834+
) and provider_pool.healthmonitor is not None:
835+
provider_pool.healthmonitor = \
836+
o_datamodels.HealthMonitor.from_dict(
837+
provider_pool.healthmonitor)
795838
provider_pools.append(provider_pool)
796839

797840
provider_lb.pools = (

‎ovn_octavia_provider/helper.py‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3269,6 +3269,184 @@ def _add_lbhc(self, ovn_lb, pool_key, info):
32693269
LOG.exception(ovn_const.EXCEPTION_MSG, "set of health check")
32703270
return status
32713271

3272+
def _sync_lbhc(self, ovn_lb, pool_key, hm):
3273+
hm_id = hm[constants.ID]
3274+
# Example
3275+
# MONITOR_PRT = 80
3276+
# ID=$(ovn-nbctl --bare --column _uuid find
3277+
# Load_Balancer_Health_Check vip="${LB_VIP_ADDR}\:${MONITOR_PRT}")
3278+
# In our case the monitor port will be the members protocol port
3279+
vips = []
3280+
if ovn_const.LB_EXT_IDS_VIP_KEY in ovn_lb.external_ids:
3281+
vips.append(ovn_lb.external_ids.get(ovn_const.LB_EXT_IDS_VIP_KEY))
3282+
if ovn_const.LB_EXT_IDS_ADDIT_VIP_KEY in ovn_lb.external_ids:
3283+
vips.extend(ovn_lb.external_ids.get(
3284+
ovn_const.LB_EXT_IDS_ADDIT_VIP_KEY).split(','))
3285+
fips = []
3286+
if ovn_const.LB_EXT_IDS_VIP_FIP_KEY in ovn_lb.external_ids:
3287+
fips.append(ovn_lb.external_ids.get(
3288+
ovn_const.LB_EXT_IDS_VIP_FIP_KEY))
3289+
if ovn_const.LB_EXT_IDS_ADDIT_VIP_FIP_KEY in ovn_lb.external_ids:
3290+
fips.extend(ovn_lb.external_ids.get(
3291+
ovn_const.LB_EXT_IDS_ADDIT_VIP_FIP_KEY).split(','))
3292+
if not vips:
3293+
msg = (f"Could not find VIP for HM {hm_id}, LB external_ids: "
3294+
f"{ovn_lb.external_ids}")
3295+
raise driver_exceptions.DriverError(msg)
3296+
3297+
vip_port = self._get_pool_listener_port(ovn_lb, pool_key)
3298+
3299+
# This is to enable lookups by Octavia DB ID value
3300+
external_ids = {
3301+
ovn_const.LB_EXT_IDS_HM_KEY: hm_id,
3302+
ovn_const.LB_EXT_IDS_HM_POOL_KEY: pool_key[
3303+
len(ovn_const.LB_EXT_IDS_POOL_PREFIX):],
3304+
}
3305+
3306+
options = {
3307+
'interval': str(hm['interval']),
3308+
'timeout': str(hm['timeout']),
3309+
'success_count': str(hm['success_count']),
3310+
'failure_count': str(hm['failure_count'])}
3311+
3312+
try:
3313+
with self.ovn_nbdb_api.transaction(check_error=True) as txn:
3314+
for vip in vips:
3315+
recreate = False
3316+
# Just seems like this needs ovsdbapp support, see:
3317+
# ovsdbapp/schema/ovn_northbound/impl_idl.py
3318+
# - lb_add()
3319+
# ovsdbapp/schema/ovn_northbound/commands.py
3320+
# - LbAddCommand()
3321+
# then this could just be self.ovn_nbdb_api.lb_hm_add()
3322+
external_ids_vip = copy.deepcopy(external_ids)
3323+
external_ids_vip[ovn_const.LB_EXT_IDS_HM_VIP] = vip
3324+
if netaddr.IPNetwork(vip).version == n_const.IP_VERSION_6:
3325+
vip = f'[{vip}]'
3326+
kwargs = {
3327+
'vip': vip + ':' + str(vip_port) if vip_port else '',
3328+
'options': options,
3329+
'external_ids': external_ids_vip}
3330+
3331+
hms_key = ovn_lb.external_ids.get(
3332+
ovn_const.LB_EXT_IDS_HMS_KEY, [])
3333+
if hms_key:
3334+
hms_key = jsonutils.loads(hms_key)
3335+
lbhcs, _ = self._find_ovn_lb_from_hm_id(hm_id)
3336+
if not lbhcs:
3337+
recreate = True
3338+
for lbhc in lbhcs:
3339+
commands = []
3340+
if lbhc.vip != kwargs.get('vip'):
3341+
commands.append(
3342+
self.ovn_nbdb_api.db_set(
3343+
'Load_Balancer_Health_Check',
3344+
lbhc.uuid,
3345+
('vip', kwargs.get('vip'))))
3346+
if lbhc.options != options:
3347+
commands.append(
3348+
self.ovn_nbdb_api.db_set(
3349+
'Load_Balancer_Health_Check',
3350+
lbhc.uuid,
3351+
('options', options)))
3352+
if lbhc.external_ids != external_ids_vip:
3353+
commands.append(
3354+
self.ovn_nbdb_api.db_set(
3355+
'Load_Balancer_Health_Check',
3356+
lbhc.uuid,
3357+
('external_ids', external_ids_vip)))
3358+
3359+
found_in_exist = False
3360+
for hc in ovn_lb.health_check:
3361+
if str(hc.uuid) == str(lbhc.uuid):
3362+
found_in_exist = True
3363+
break
3364+
if not found_in_exist:
3365+
commands.append(
3366+
self.ovn_nbdb_api.db_add(
3367+
'Load_Balancer', ovn_lb.uuid,
3368+
('health_check', lbhc.uuid)))
3369+
if hm_id not in hms_key:
3370+
hms_key.append(hm_id)
3371+
commands.append(self.ovn_nbdb_api.db_set(
3372+
'Load_Balancer', ovn_lb.uuid,
3373+
('external_ids',
3374+
{ovn_const.LB_EXT_IDS_HMS_KEY:
3375+
jsonutils.dumps(hms_key)})))
3376+
self._execute_commands(commands)
3377+
if recreate:
3378+
health_check = txn.add(
3379+
self.ovn_nbdb_api.db_create(
3380+
'Load_Balancer_Health_Check',
3381+
**kwargs))
3382+
txn.add(self.ovn_nbdb_api.db_set(
3383+
'Load_Balancer_Health_Check',
3384+
health_check,
3385+
('vip', kwargs.get('vip'))))
3386+
txn.add(self.ovn_nbdb_api.db_add(
3387+
'Load_Balancer', ovn_lb.uuid,
3388+
'health_check', health_check))
3389+
if hm_id not in hms_key:
3390+
hms_key.append(hm_id)
3391+
txn.add(self.ovn_nbdb_api.db_set(
3392+
'Load_Balancer', ovn_lb.uuid,
3393+
('external_ids', {ovn_const.LB_EXT_IDS_HMS_KEY:
3394+
jsonutils.dumps(hms_key)})))
3395+
if fips:
3396+
external_ids_fip = copy.deepcopy(external_ids)
3397+
for fip in fips:
3398+
recreate = False
3399+
external_ids_fip[ovn_const.LB_EXT_IDS_HM_VIP] = fip
3400+
if netaddr.IPNetwork(
3401+
fip).version == n_const.IP_VERSION_6:
3402+
fip = f'[{fip}]'
3403+
fip_kwargs = {
3404+
'vip': fip + ':' + str(vip_port)
3405+
if vip_port
3406+
else '',
3407+
'options': options,
3408+
'external_ids': external_ids_fip}
3409+
3410+
lbhcs, _ = self._find_ovn_lb_from_hm_id(hm_id)
3411+
if not lbhcs:
3412+
recreate = True
3413+
for lbhc in lbhcs:
3414+
commands = []
3415+
if lbhc.vip != fip_kwargs['vip']:
3416+
commands.append(
3417+
self.ovn_nbdb_api.db_set(
3418+
'Load_Balancer_Health_Check',
3419+
lbhc.uuid,
3420+
('vip', fip_kwargs['vip'])))
3421+
if lbhc.options != options:
3422+
commands.append(
3423+
self.ovn_nbdb_api.db_set(
3424+
'Load_Balancer_Health_Check',
3425+
lbhc.uuid,
3426+
('options', options)))
3427+
if lbhc.external_ids != external_ids_vip:
3428+
commands.append(
3429+
self.ovn_nbdb_api.db_set(
3430+
'Load_Balancer_Health_Check',
3431+
lbhc.uuid,
3432+
('external_ids', external_ids_fip)))
3433+
self._execute_commands(commands)
3434+
if recreate:
3435+
fip_health_check = txn.add(
3436+
self.ovn_nbdb_api.db_create(
3437+
'Load_Balancer_Health_Check',
3438+
**fip_kwargs))
3439+
txn.add(self.ovn_nbdb_api.db_set(
3440+
'Load_Balancer_Health_Check',
3441+
fip_health_check,
3442+
('vip', fip_kwargs['vip'])))
3443+
txn.add(self.ovn_nbdb_api.db_add(
3444+
'Load_Balancer', ovn_lb.uuid,
3445+
'health_check', fip_health_check))
3446+
except Exception as e:
3447+
msg = (f"Error syncing Load Balancer Health Check: {e}")
3448+
raise driver_exceptions.DriverError(msg)
3449+
32723450
def _update_lbhc_vip_port(self, lbhc, vip_port):
32733451
if lbhc.vip:
32743452
vip = lbhc.vip.rsplit(":")[0] + ':' + str(vip_port)
@@ -3495,6 +3673,46 @@ def hm_create(self, info):
34953673
status[constants.HEALTHMONITORS] = [hm_status]
34963674
return status
34973675

3676+
def hm_sync(self, hm, ovn_lb, pool_key):
3677+
"""Sync Health Monitor object with an OVN LoadBalancer
3678+
3679+
The method performs the following steps:
3680+
1. Create Health Monitor in OVN NB in case we don't find
3681+
load_balancer_health_checks entries associated
3682+
2. If we found load_balancer_health_checks entries associated
3683+
2.1. Update member status affected on OVN loadbalancer external_ids
3684+
2.2. Sync OVN load_balancer_health_checks entries
3685+
2.3. Update OVN Loadbalancer ip_port_mappings
3686+
2.4. Update OVN Loadbalancer member_status info on external_ids
3687+
3688+
:param hm: The source health monitor object from Octavia DB
3689+
:param ovn_lb: The OVN LoadBalancer object that needs to be sync
3690+
:param pool_key: The pool_key where health monitor is associated
3691+
"""
3692+
lbhcs, ovn_lb = self._find_ovn_lb_from_hm_id(hm[constants.ID])
3693+
if not lbhcs:
3694+
LOG.debug("Loadbalancer health check %s not found!",
3695+
hm[constants.ID])
3696+
# Create in case we don't found it
3697+
self.hm_create(hm)
3698+
return
3699+
3700+
pool_id = hm[constants.POOL_ID]
3701+
self._update_member_statuses(ovn_lb, pool_id, constants.ACTIVE,
3702+
constants.ONLINE)
3703+
try:
3704+
self._sync_lbhc(ovn_lb, pool_key, hm)
3705+
except Exception as e:
3706+
LOG.exception(f"Failed syncing Load Balancer Health Monitor: {e}")
3707+
3708+
for mb_ip, mb_port, mb_subnet, mb_id in self._extract_member_info(
3709+
ovn_lb.external_ids[pool_key]):
3710+
mb_status = self._update_hm_member(ovn_lb, pool_key, mb_ip)
3711+
if not mb_status:
3712+
self._clean_ip_port_mappings(ovn_lb, pool_key)
3713+
break
3714+
self._update_external_ids_member_status(ovn_lb, mb_id, mb_status)
3715+
34983716
def hm_update(self, info):
34993717
status = {
35003718
constants.HEALTHMONITORS: [
@@ -3871,3 +4089,42 @@ def hm_update_event(self, info):
38714089
status[k].extend(status_lb[k])
38724090

38734091
return status
4092+
4093+
def hm_purge(self, lb_id):
4094+
# remove redundant hm if any, if no
4095+
# ovn_const.LB_EXT_IDS_HMS_KEY from lb matches,
4096+
# this hm has no used and should already got replaced.
4097+
ovn_lbs = []
4098+
try:
4099+
ovn_lbs = self._find_ovn_lbs_with_retry(lb_id)
4100+
except idlutils.RowNotFound:
4101+
LOG.debug(f"OVN loadbalancer {lb_id} not found.")
4102+
fetch_hc_ids = []
4103+
for ovn_lb in ovn_lbs:
4104+
hm_ids = ovn_lb.external_ids.get(
4105+
ovn_const.LB_EXT_IDS_HMS_KEY, [])
4106+
if hm_ids:
4107+
hm_ids = jsonutils.loads(hm_ids)
4108+
for hm_id in hm_ids:
4109+
lbhcs = []
4110+
try:
4111+
lbhcs = self._lookup_lbhcs_by_hm_id(hm_id)
4112+
except idlutils.RowNotFound:
4113+
continue
4114+
fetch_hc_ids.extend([str(lbhc.uuid) for lbhc in lbhcs])
4115+
4116+
for hc_id in ovn_lb.health_check:
4117+
if str(hc_id.uuid) not in fetch_hc_ids:
4118+
commands = []
4119+
commands.append(
4120+
self.ovn_nbdb_api.db_remove(
4121+
'Load_Balancer', ovn_lb.uuid,
4122+
'health_check', hc_id.uuid))
4123+
commands.append(
4124+
self.ovn_nbdb_api.db_destroy(
4125+
'Load_Balancer_Health_Check',
4126+
hc_id.uuid))
4127+
try:
4128+
self._execute_commands(commands)
4129+
except idlutils.RowNotFound:
4130+
LOG.debug("health check not found for purge.")

0 commit comments

Comments
 (0)