Adding/Remove static arp entry for pod IP in VM (#273)

* add static arp entry in vm (for bridge mode and singletenancy) whenever pod created and remove when its deleted

Author: tamilmani1989

netlink/link.go

@@ -31,6 +31,11 @@ const (
 	IPVLAN_MODE_MAX
 )
 
+const (
+	ADD = iota
+	REMOVE
+)
+
 // Link represents a network interface.
 type Link interface {
 	Info() *LinkInfo
@@ -384,3 +389,42 @@ func SetLinkHairpin(bridgeName string, on bool) error {
 
 	return s.sendAndWaitForAck(req)
 }
+
+// AddOrRemoveStaticArp sets/removes static arp entry based on mode
+func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr) error {
+	s, err := getSocket()
+	if err != nil {
+		return err
+	}
+
+	var req *message
+	state := 0
+	if mode == ADD {
+		req = newRequest(unix.RTM_NEWNEIGH, unix.NLM_F_CREATE|unix.NLM_F_REPLACE|unix.NLM_F_ACK)
+		state = NUD_PERMANENT
+	} else {
+		req = newRequest(unix.RTM_DELNEIGH, unix.NLM_F_ACK)
+		state = NUD_INCOMPLETE
+	}
+
+	iface, err := net.InterfaceByName(name)
+	if err != nil {
+		return err
+	}
+
+	msg := neighMsg{
+		Family: uint8(unix.AF_INET),
+		Index:  uint32(iface.Index),
+		State:  uint16(state),
+	}
+	req.addPayload(&msg)
+
+	ipData := ipaddr.To4()
+	dstData := newRtAttr(NDA_DST, ipData)
+	req.addPayload(dstData)
+
+	hwData := newRtAttr(NDA_LLADDR, []byte(mac))
+	req.addPayload(hwData)
+
+	return s.sendAndWaitForAck(req)
+}

netlink/netlink_test.go

@@ -234,3 +234,28 @@ func TestSetLinkHairpin(t *testing.T) {
 		t.Errorf("DeleteLink failed: %+v", err)
 	}
 }
+
+func TestAddRemoveStaticArp(t *testing.T) {
+	_, err := addDummyInterface(ifName)
+	if err != nil {
+		t.Errorf("addDummyInterface failed: %v", err)
+	}
+
+	ip := net.ParseIP("192.168.0.2")
+	mac, _ := net.ParseMAC("aa:b3:4d:5e:e2:4a")
+
+	err = AddOrRemoveStaticArp(ADD, ifName, ip, mac)
+	if err != nil {
+		t.Errorf("ret val %v", err)
+	}
+
+	err = AddOrRemoveStaticArp(REMOVE, ifName, ip, mac)
+	if err != nil {
+		t.Errorf("ret val %v", err)
+	}
+
+	err = DeleteLink(ifName)
+	if err != nil {
+		t.Errorf("DeleteLink failed: %+v", err)
+	}
+}

netlink/protocol.go

@@ -13,6 +13,41 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+const (
+	NDA_UNSPEC = iota
+	NDA_DST
+	NDA_LLADDR
+	NDA_CACHEINFO
+	NDA_PROBES
+	NDA_VLAN
+	NDA_PORT
+	NDA_VNI
+	NDA_IFINDEX
+	NDA_MAX = NDA_IFINDEX
+)
+
+// Neighbor Cache Entry States.
+const (
+	NUD_NONE       = 0x00
+	NUD_INCOMPLETE = 0x01
+	NUD_REACHABLE  = 0x02
+	NUD_STALE      = 0x04
+	NUD_DELAY      = 0x08
+	NUD_PROBE      = 0x10
+	NUD_FAILED     = 0x20
+	NUD_NOARP      = 0x40
+	NUD_PERMANENT  = 0x80
+)
+
+// Neighbor Flags
+const (
+	NTF_USE    = 0x01
+	NTF_SELF   = 0x02
+	NTF_MASTER = 0x04
+	NTF_PROXY  = 0x08
+	NTF_ROUTER = 0x80
+)
+
 // Netlink protocol constants that are not already defined in unix package.
 const (
 	IFLA_INFO_KIND   = 1
@@ -30,6 +65,40 @@ type serializable interface {
 	length() int
 }
 
+//
+// Netlink message
+//
+
+// Generic netlink message
+type message struct {
+	unix.NlMsghdr
+	data    []byte
+	payload []serializable
+}
+
+// Generic netlink message attribute
+type attribute struct {
+	unix.NlAttr
+	value    []byte
+	children []serializable
+}
+
+// Neighbor entry message strutcure
+type neighMsg struct {
+	Family uint8
+	Index  uint32
+	State  uint16
+	Flags  uint8
+	Type   uint8
+}
+
+// rta attribute structure
+type rtAttr struct {
+	unix.RtAttr
+	Data     []byte
+	children []serializable
+}
+
 // Byte encoder
 var encoder binary.ByteOrder
 
@@ -43,17 +112,6 @@ func initEncoder() {
 	}
 }
 
-//
-// Netlink message
-//
-
-// Generic netlink message
-type message struct {
-	unix.NlMsghdr
-	data    []byte
-	payload []serializable
-}
-
 // Creates a new netlink message.
 func newMessage(msgType int, flags int) *message {
 	return &message{
@@ -127,14 +185,6 @@ func (msg *message) getAttributes(body serializable) []*attribute {
 //
 // Netlink message attribute
 //
-
-// Generic netlink message attribute
-type attribute struct {
-	unix.NlAttr
-	value    []byte
-	children []serializable
-}
-
 // Creates a new attribute.
 func newAttribute(attrType int, value []byte) *attribute {
 	return &attribute{
@@ -339,3 +389,70 @@ func (rt *rtMsg) serialize() []byte {
 func (rt *rtMsg) length() int {
 	return unix.SizeofRtMsg
 }
+
+// serialize neighbor message
+func (msg *neighMsg) serialize() []byte {
+	return (*(*[unsafe.Sizeof(*msg)]byte)(unsafe.Pointer(msg)))[:]
+}
+
+func (msg *neighMsg) length() int {
+	return int(unsafe.Sizeof(*msg))
+}
+
+// creates new rta attr message
+func newRtAttr(attrType int, data []byte) *rtAttr {
+	return &rtAttr{
+		RtAttr: unix.RtAttr{
+			Type: uint16(attrType),
+		},
+		children: []serializable{},
+		Data:     data,
+	}
+}
+
+// align rta attributes
+func rtaAlignOf(attrlen int) int {
+	return (attrlen + unix.RTA_ALIGNTO - 1) & ^(unix.RTA_ALIGNTO - 1)
+}
+
+// serialize rta message
+func (rta *rtAttr) serialize() []byte {
+	length := rta.length()
+	buf := make([]byte, rtaAlignOf(length))
+
+	next := 4
+	if rta.Data != nil {
+		copy(buf[next:], rta.Data)
+		next += rtaAlignOf(len(rta.Data))
+	}
+	if len(rta.children) > 0 {
+		for _, child := range rta.children {
+			childBuf := child.serialize()
+			copy(buf[next:], childBuf)
+			next += rtaAlignOf(len(childBuf))
+		}
+	}
+
+	if l := uint16(length); l != 0 {
+		encoder.PutUint16(buf[0:2], l)
+	}
+	encoder.PutUint16(buf[2:4], rta.Type)
+	return buf
+}
+
+func (rta *rtAttr) length() int {
+	if len(rta.children) == 0 {
+		return (unix.SizeofRtAttr + len(rta.Data))
+	}
+
+	l := 0
+	for _, child := range rta.children {
+		l += rtaAlignOf(child.length())
+	}
+	l += unix.SizeofRtAttr
+	return rtaAlignOf(l + len(rta.Data))
+}
+
+func (rta *rtAttr) addChild(attr serializable) {
+	rta.children = append(rta.children, attr)
+}

network/bridge_endpointclient_linux.go

@@ -72,6 +72,14 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo)
 		if err := ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, client.containerMac, ebtables.Append); err != nil {
 			return err
 		}
+
+		if client.mode != opModeTunnel {
+			log.Printf("[net] Adding static arp for IP address %v and MAC %v in VM", ipAddr.String(), client.containerMac.String())
+			netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac)
+			if err != nil {
+				log.Printf("Failed setting arp in vm: %v", err)
+			}
+		}
 	}
 
 	log.Printf("[net] Setting hairpin for hostveth %v", client.hostVethName)
@@ -99,6 +107,14 @@ func (client *LinuxBridgeEndpointClient) DeleteEndpointRules(ep *endpoint) {
 		if err != nil {
 			log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err)
 		}
+
+		if client.mode != opModeTunnel {
+			log.Printf("[net] Removing static arp for IP address %v and MAC %v from VM", ipAddr.String(), ep.MacAddress.String())
+			netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress)
+			if err != nil {
+				log.Printf("Failed removing arp from vm: %v", err)
+			}
+		}
 	}
 }