lltable: use own lock

Add struct mtx to struct lltable and stop using IF_AFDATA_LOCK, that
was created for a completely different purpose.  No functional change
intended.

Reviewed by:		zlei, melifaro
Differential Revision:	https://reviews.freebsd.org/D54086

Author: glebius

sys/net/if_llatbl.c

@@ -206,7 +206,7 @@ htable_link_entry(struct lltable *llt, struct llentry *lle)
 	if ((lle->la_flags & LLE_LINKED) != 0)
 		return (0);
 
-	IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp);
+	LLTABLE_LOCK_ASSERT(llt);
 
 	if (llt->llt_maxentries > 0 &&
 	    llt->llt_entries >= llt->llt_maxentries)
@@ -233,7 +233,7 @@ htable_unlink_entry(struct llentry *lle)
 		return (0);
 
 	llt = lle->lle_tbl;
-	IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp);
+	LLTABLE_LOCK_ASSERT(llt);
 	KASSERT(llt->llt_entries > 0, ("%s: lltable %p (%s) entries %d <= 0",
 	    __func__, llt, if_name(llt->llt_ifp), llt->llt_entries));
 
@@ -283,12 +283,12 @@ htable_prefix_free(struct lltable *llt, const struct sockaddr *addr,
 	pmd.flags = flags;
 	CK_LIST_INIT(&pmd.dchain);
 
-	IF_AFDATA_WLOCK(llt->llt_ifp);
+	LLTABLE_LOCK(llt);
 	/* Push matching lles to chain */
 	lltable_foreach_lle(llt, htable_prefix_free_cb, &pmd);
 
 	llentries_unlink(llt, &pmd.dchain);
-	IF_AFDATA_WUNLOCK(llt->llt_ifp);
+	LLTABLE_UNLOCK(llt);
 
 	CK_LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next)
 		lltable_free_entry(llt, lle);
@@ -298,6 +298,7 @@ static void
 htable_free_tbl(struct lltable *llt)
 {
 
+	mtx_destroy(&llt->llt_lock);
 	free(llt->lle_head, M_LLTABLE);
 	free(llt, M_LLTABLE);
 }
@@ -357,22 +358,20 @@ lltable_set_entry_addr(struct ifnet *ifp, struct llentry *lle,
  * On failure, false is returned and lle wlock is still held.
  */
 bool
-lltable_acquire_wlock(struct ifnet *ifp, struct llentry *lle)
+lltable_trylock(struct llentry *lle)
 {
 	NET_EPOCH_ASSERT();
 
-	/* Perform real LLE update */
-	/* use afdata WLOCK to update fields */
 	LLE_WUNLOCK(lle);
-	IF_AFDATA_WLOCK(ifp);
+	LLTABLE_LOCK(lle->lle_tbl);
 	LLE_WLOCK(lle);
 
 	/*
 	 * Since we droppped LLE lock, other thread might have deleted
 	 * this lle. Check and return
 	 */
 	if ((lle->la_flags & LLE_DELETED) != 0) {
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(lle->lle_tbl);
 		return (false);
 	}
 
@@ -392,13 +391,13 @@ lltable_try_set_entry_addr(struct ifnet *ifp, struct llentry *lle,
     const char *linkhdr, size_t linkhdrsize, int lladdr_off)
 {
 
-	if (!lltable_acquire_wlock(ifp, lle))
+	if (!lltable_trylock(lle))
 		return (0);
 
 	/* Update data */
 	lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, lladdr_off);
 
-	IF_AFDATA_WUNLOCK(ifp);
+	LLTABLE_UNLOCK(lle->lle_tbl);
 
 	return (1);
 }
@@ -630,9 +629,9 @@ lltable_update_ifaddr(struct lltable *llt)
 	if (llt->llt_ifp->if_flags & IFF_LOOPBACK)
 		return;
 
-	IF_AFDATA_WLOCK(llt->llt_ifp);
+	LLTABLE_LOCK(llt);
 	lltable_foreach_lle(llt, llentry_update_ifaddr, llt->llt_ifp);
-	IF_AFDATA_WUNLOCK(llt->llt_ifp);
+	LLTABLE_UNLOCK(llt);
 }
 
 /*
@@ -696,11 +695,11 @@ lltable_free(struct lltable *llt)
 	lltable_unlink(llt);
 
 	CK_LIST_INIT(&dchain);
-	IF_AFDATA_WLOCK(llt->llt_ifp);
+	LLTABLE_LOCK(llt);
 	/* Push all lles to @dchain */
 	lltable_foreach_lle(llt, lltable_free_cb, &dchain);
 	llentries_unlink(llt, &dchain);
-	IF_AFDATA_WUNLOCK(llt->llt_ifp);
+	LLTABLE_UNLOCK(llt);
 
 	CK_LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) {
 		llentry_free(lle);
@@ -722,24 +721,22 @@ lltable_delete_addr(struct lltable *llt, u_int flags,
     const struct sockaddr *l3addr)
 {
 	struct llentry *lle;
-	struct ifnet *ifp;
 
-	ifp = llt->llt_ifp;
-	IF_AFDATA_WLOCK(ifp);
+	LLTABLE_LOCK(llt);
 	lle = lla_lookup(llt, LLE_SF(l3addr->sa_family, LLE_EXCLUSIVE), l3addr);
 
 	if (lle == NULL) {
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(llt);
 		return (ENOENT);
 	}
 	if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) {
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(llt);
 		LLE_WUNLOCK(lle);
 		return (EPERM);
 	}
 
 	lltable_unlink_entry(llt, lle);
-	IF_AFDATA_WUNLOCK(ifp);
+	LLTABLE_UNLOCK(llt);
 
 	llt->llt_delete_entry(llt, lle);
 
@@ -798,10 +795,10 @@ lltable_delete_conditional(struct lltable *llt, llt_match_cb_t *func,
 	lmd.func = func;
 	lmd.farg = farg;
 
-	IF_AFDATA_WLOCK(llt->llt_ifp);
+	LLTABLE_LOCK(llt);
 	lltable_foreach_lle(llt, lltable_delete_conditional_cb, &lmd);
 	llentries_unlink(llt, &lmd.dchain);
-	IF_AFDATA_WUNLOCK(llt->llt_ifp);
+	LLTABLE_UNLOCK(llt);
 
 	CK_LIST_FOREACH_SAFE(lle, &lmd.dchain, lle_chain, next)
 		llt->llt_delete_entry(llt, lle);
@@ -817,6 +814,7 @@ lltable_allocate_htbl(uint32_t hsize)
 	llt->llt_hsize = hsize;
 	llt->lle_head = malloc(sizeof(struct llentries) * hsize,
 	    M_LLTABLE, M_WAITOK | M_ZERO);
+	mtx_init(&llt->llt_lock, "lltable", NULL, MTX_DEF);
 
 	for (i = 0; i < llt->llt_hsize; i++)
 		CK_LIST_INIT(&llt->lle_head[i]);
@@ -1021,13 +1019,13 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 
 		/* Try to link new entry */
 		lle_tmp = NULL;
-		IF_AFDATA_WLOCK(ifp);
+		LLTABLE_LOCK(llt);
 		LLE_WLOCK(lle);
 		lle_tmp = lla_lookup(llt, LLE_EXCLUSIVE, dst);
 		if (lle_tmp != NULL) {
 			/* Check if we are trying to replace immutable entry */
 			if ((lle_tmp->la_flags & LLE_IFADDR) != 0) {
-				IF_AFDATA_WUNLOCK(ifp);
+				LLTABLE_UNLOCK(llt);
 				LLE_WUNLOCK(lle_tmp);
 				lltable_free_entry(llt, lle);
 				return (EPERM);
@@ -1036,7 +1034,7 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 			lltable_unlink_entry(llt, lle_tmp);
 		}
 		lltable_link_entry(llt, lle);
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(llt);
 
 		if (lle_tmp != NULL) {
 			EVENTHANDLER_INVOKE(lle_event, lle_tmp,LLENTRY_EXPIRED);

sys/net/if_llatbl.h

@@ -174,6 +174,7 @@ struct lltable {
 	int			llt_maxentries;
 	struct llentries	*lle_head;
 	struct ifnet		*llt_ifp;
+	struct mtx		llt_lock;
 
 	llt_lookup_t		*llt_lookup;
 	llt_alloc_t		*llt_alloc_entry;
@@ -192,6 +193,12 @@ struct lltable {
 	llt_post_resolved_t	*llt_post_resolved;
 };
 
+#define	LLTABLE_LOCK(llt)	mtx_lock(&(llt)->llt_lock)
+#define	LLTABLE_UNLOCK(llt)	mtx_unlock(&(llt)->llt_lock)
+#define	LLTABLE_LOCK_ASSERT(llt)	mtx_assert(&(llt)->llt_lock, MA_OWNED)
+#define	LLTABLE_RLOCK_ASSERT(llt)	MPASS(in_epoch(net_epoch_preempt) || \
+					mtx_owned(&(llt)->llt_lock))
+
 MALLOC_DECLARE(M_LLTABLE);
 
 /*
@@ -261,7 +268,7 @@ void lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa);
 struct ifnet *lltable_get_ifp(const struct lltable *llt);
 int lltable_get_af(const struct lltable *llt);
 
-bool lltable_acquire_wlock(struct ifnet *ifp, struct llentry *lle);
+bool lltable_trylock(struct llentry *lle);
 
 int lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f,
     void *farg);

sys/netinet/if_ether.c

@@ -288,15 +288,15 @@ arptimer(void *arg)
 
 	/* XXX: LOR avoidance. We still have ref on lle. */
 	LLE_WUNLOCK(lle);
-	IF_AFDATA_LOCK(ifp);
+	LLTABLE_LOCK(LLTABLE(ifp));
 	LLE_WLOCK(lle);
 
 	/* Guard against race with other llentry_free(). */
 	if (lle->la_flags & LLE_LINKED) {
 		LLE_REMREF(lle);
 		lltable_unlink_entry(lle->lle_tbl, lle);
 	}
-	IF_AFDATA_UNLOCK(ifp);
+	LLTABLE_UNLOCK(LLTABLE(ifp));
 
 	size_t pkts_dropped = llentry_free(lle);
 
@@ -488,13 +488,13 @@ arpresolve_full(struct ifnet *ifp, int is_gw, int flags, struct mbuf *m,
 			return (EINVAL);
 		}
 
-		IF_AFDATA_WLOCK(ifp);
+		LLTABLE_LOCK(LLTABLE(ifp));
 		LLE_WLOCK(la);
 		la_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst);
 		/* Prefer ANY existing lle over newly-created one */
 		if (la_tmp == NULL)
 			lltable_link_entry(LLTABLE(ifp), la);
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(LLTABLE(ifp));
 		if (la_tmp != NULL) {
 			lltable_free_entry(LLTABLE(ifp), la);
 			la = la_tmp;
@@ -961,7 +961,7 @@ in_arpinput(struct mbuf *m)
 		lltable_set_entry_addr(ifp, la, linkhdr, linkhdrsize,
 		    lladdr_off);
 
-		IF_AFDATA_WLOCK(ifp);
+		LLTABLE_LOCK(LLTABLE(ifp));
 		LLE_WLOCK(la);
 		la_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst);
 
@@ -983,7 +983,7 @@ in_arpinput(struct mbuf *m)
 		 */
 		if (la_tmp == NULL)
 			lltable_link_entry(LLTABLE(ifp), la);
-		IF_AFDATA_WUNLOCK(ifp);
+		LLTABLE_UNLOCK(LLTABLE(ifp));
 
 		if (la_tmp == NULL) {
 			arp_mark_lle_reachable(la, ifp);
@@ -1301,15 +1301,15 @@ arp_add_ifa_lle(struct ifnet *ifp, const struct sockaddr *dst)
 		return;
 	}
 
-	IF_AFDATA_WLOCK(ifp);
+	LLTABLE_LOCK(LLTABLE(ifp));
 	LLE_WLOCK(lle);
 	/* Unlink any entry if exists */
 	lle_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst);
 	if (lle_tmp != NULL)
 		lltable_unlink_entry(LLTABLE(ifp), lle_tmp);
 
 	lltable_link_entry(LLTABLE(ifp), lle);
-	IF_AFDATA_WUNLOCK(ifp);
+	LLTABLE_UNLOCK(LLTABLE(ifp));
 
 	if (lle_tmp != NULL)
 		EVENTHANDLER_INVOKE(lle_event, lle_tmp, LLENTRY_EXPIRED);

sys/netinet/in.c

@@ -1546,7 +1546,7 @@ in_lltable_free_entry(struct lltable *llt, struct llentry *lle)
 
 	/* Unlink entry from table if not already */
 	if ((lle->la_flags & LLE_LINKED) != 0) {
-		IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp);
+		LLTABLE_LOCK_ASSERT(llt);
 		lltable_unlink_entry(llt, lle);
 	}
 
@@ -1728,7 +1728,7 @@ in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3add
 	const struct sockaddr_in *sin = (const struct sockaddr_in *)l3addr;
 	struct llentry *lle;
 
-	IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
+	LLTABLE_RLOCK_ASSERT(llt);
 	KASSERT(l3addr->sa_family == AF_INET,
 	    ("sin_family %d", l3addr->sa_family));
 	KASSERT((flags & (LLE_UNLOCKED | LLE_EXCLUSIVE)) !=

sys/netinet6/in6.c

@@ -2253,15 +2253,13 @@ in6_lltable_match_prefix(const struct sockaddr *saddr,
 static void
 in6_lltable_free_entry(struct lltable *llt, struct llentry *lle)
 {
-	struct ifnet *ifp __diagused;
 
 	LLE_WLOCK_ASSERT(lle);
 	KASSERT(llt != NULL, ("lltable is NULL"));
 
 	/* Unlink entry from table */
 	if ((lle->la_flags & LLE_LINKED) != 0) {
-		ifp = llt->llt_ifp;
-		IF_AFDATA_WLOCK_ASSERT(ifp);
+		LLTABLE_LOCK_ASSERT(llt);
 		lltable_unlink_entry(llt, lle);
 	}
 
@@ -2421,7 +2419,7 @@ in6_lltable_lookup(struct lltable *llt, u_int flags,
 	int family = flags >> 16;
 	struct llentry *lle;
 
-	IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
+	LLTABLE_RLOCK_ASSERT(llt);
 	KASSERT(l3addr->sa_family == AF_INET6,
 	    ("sin_family %d", l3addr->sa_family));
 	KASSERT((flags & (LLE_UNLOCKED | LLE_EXCLUSIVE)) !=
@@ -2445,7 +2443,7 @@ in6_lltable_lookup(struct lltable *llt, u_int flags,
 		LLE_RLOCK(lle);
 
 	/*
-	 * If the afdata lock is not held, the LLE may have been unlinked while
+	 * If the lltable lock is not held, the LLE may have been unlinked while
 	 * we were blocked on the LLE lock.  Check for this case.
 	 */
 	if (__predict_false((lle->la_flags & LLE_LINKED) == 0)) {
@@ -2743,9 +2741,9 @@ in6_purge_proxy_ndp(struct ifnet *ifp)
 		return;
 
 	llt = LLTABLE6(ifp);
-	IF_AFDATA_WLOCK(ifp);
+	LLTABLE_LOCK(llt);
 	need_purge = ((llt->llt_flags & LLT_ADDEDPROXY) != 0);
-	IF_AFDATA_WUNLOCK(ifp);
+	LLTABLE_UNLOCK(llt);
 
 	/*
 	 * Ever added proxy ndp entries, leave solicited node multicast

sys/netinet6/mld6.c

@@ -165,8 +165,8 @@ static int	sysctl_mld_ifinfo(SYSCTL_HANDLER_ARGS);
  *  scope ID is only used by MLD to select the outgoing interface.
  *
  *  During interface attach and detach, MLD will take MLD_LOCK *after*
- *  the IF_AFDATA_LOCK.
- *  As in6_setscope() takes IF_AFDATA_LOCK then SCOPE_LOCK, we can't call
+ *  the LLTABLE_LOCK.
+ *  As in6_setscope() takes LLTABLE_LOCK then SCOPE_LOCK, we can't call
  *  it with MLD_LOCK held without triggering an LOR. A netisr with indirect
  *  dispatch could work around this, but we'd rather not do that, as it
  *  can introduce other races.
@@ -182,7 +182,7 @@ static int	sysctl_mld_ifinfo(SYSCTL_HANDLER_ARGS);
  *  calls in6_setscope() internally whilst MLD_LOCK is held. This will
  *  trigger a LOR warning in WITNESS when the ifnet is detached.
  *
- *  The right answer is probably to make IF_AFDATA_LOCK an rwlock, given
+ *  The right answer is probably to make LLTABLE_LOCK an rwlock, given
  *  how it's used across the network stack. Here we're simply exploiting
  *  the fact that MLD runs at a similar layer in the stack to scope6.c.
  *
@@ -553,7 +553,7 @@ mld_ifdetach(struct ifnet *ifp, struct in6_multi_head *inmh)
  * Hook for domifdetach.
  * Runs after link-layer cleanup; free MLD state.
  *
- * SMPng: Normally called with IF_AFDATA_LOCK held.
+ * SMPng: Normally called with LLTABLE_LOCK held.
  */
 void
 mld_domifdetach(struct ifnet *ifp)