diff --git a/p2p/host/resource-manager/rcmgr.go b/p2p/host/resource-manager/rcmgr.go
index f5d60a44e0..76159d027d 100644
--- a/p2p/host/resource-manager/rcmgr.go
+++ b/p2p/host/resource-manager/rcmgr.go
@@ -161,6 +161,7 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager
 	for _, npLimit := range r.connLimiter.networkPrefixLimitV6 {
 		registeredConnLimiterPrefixes[npLimit.Network.String()] = struct{}{}
 	}
+	// Add network prefix limits from allowlist.allowedNetworks
 	for _, network := range allowlist.allowedNetworks {
 		prefix, err := netip.ParsePrefix(network.String())
 		if err != nil {
@@ -173,6 +174,25 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager
 				Network:   prefix,
 				ConnCount: r.limits.GetAllowlistedSystemLimits().GetConnTotalLimit(),
 			})
+			registeredConnLimiterPrefixes[prefix.String()] = struct{}{}
+		}
+	}
+	// Add network prefix limits from allowlist.allowedPeerByNetwork
+	for _, networks := range allowlist.allowedPeerByNetwork {
+		for _, network := range networks {
+			prefix, err := netip.ParsePrefix(network.String())
+			if err != nil {
+				log.Debugf("failed to parse prefix from allowlist %s, %s", network, err)
+				continue
+			}
+			if _, ok := registeredConnLimiterPrefixes[prefix.String()]; !ok {
+				// connlimiter doesn't know about this network. Let's fix that
+				r.connLimiter.addNetworkPrefixLimit(prefix.Addr().Is6(), NetworkPrefixLimit{
+					Network:   prefix,
+					ConnCount: r.limits.GetAllowlistedSystemLimits().GetConnTotalLimit(),
+				})
+				registeredConnLimiterPrefixes[prefix.String()] = struct{}{}
+			}
 		}
 	}
 	r.verifySourceAddressRateLimiter = newVerifySourceAddressRateLimiter(r.connLimiter)
diff --git a/p2p/host/resource-manager/rcmgr_test.go b/p2p/host/resource-manager/rcmgr_test.go
index 97756eaf6d..796f737ae5 100644
--- a/p2p/host/resource-manager/rcmgr_test.go
+++ b/p2p/host/resource-manager/rcmgr_test.go
@@ -1,6 +1,7 @@
 package rcmgr
 
 import (
+	"fmt"
 	"net"
 	"net/netip"
 	"testing"
@@ -1093,6 +1094,25 @@ func TestAllowlistAndConnLimiterPlayNice(t *testing.T) {
 		require.Equal(t, 8, rcmgr.(*resourceManager).connLimiter.networkPrefixLimitV6[0].ConnCount)
 	})
 
+	t.Run("IPv4 with peer ID", func(t *testing.T) {
+		peer, err := test.RandPeerID()
+		require.NoError(t, err)
+
+		rcmgr, err := NewResourceManager(NewFixedLimiter(limits), WithAllowlistedMultiaddrs([]multiaddr.Multiaddr{
+			multiaddr.StringCast(fmt.Sprintf("/ip4/1.2.3.0/ipcidr/24/p2p/%s", peer.String())),
+		}), WithNetworkPrefixLimit([]NetworkPrefixLimit{}, []NetworkPrefixLimit{}))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rcmgr.Close()
+
+		// The connLimiter should have the allowlisted network prefix
+		require.Equal(t, netip.MustParsePrefix("1.2.3.0/24"), rcmgr.(*resourceManager).connLimiter.networkPrefixLimitV4[0].Network)
+
+		// The connLimiter should use the limit from the allowlist
+		require.Equal(t, 8, rcmgr.(*resourceManager).connLimiter.networkPrefixLimitV4[0].ConnCount)
+	})
+
 	t.Run("Does not override if you set a limit directly", func(t *testing.T) {
 		rcmgr, err := NewResourceManager(NewFixedLimiter(limits), WithAllowlistedMultiaddrs([]multiaddr.Multiaddr{
 			multiaddr.StringCast("/ip4/1.2.3.0/ipcidr/24"),