networking: New subproject.

Add scalar and NEON ones' complement checksumming implementations for
AArch64 and Armv7-A.

Author: Ola Liljedahl

Makefile

@@ -10,7 +10,7 @@ libdir = $(prefix)/lib
 includedir = $(prefix)/include
 
 # Configure these in config.mk, do not make changes in this file.
-SUBS = math string
+SUBS = math string networking
 HOST_CC = cc
 HOST_CFLAGS = -std=c99 -O2
 HOST_LDFLAGS =

README

@@ -17,6 +17,9 @@ math/           - math subproject sources.
 math/include/   - math library public headers.
 math/test/      - math test and benchmark related sources.
 math/tools/     - tools used for designing the algorithms.
+networking/     - networking subproject sources.
+networking/include/ - networking library public headers.
+networking/test/ - networking test and benchmark related sources.
 string/         - string routines subproject sources.
 string/include/ - string library public headers.
 string/test/    - string test and benchmark related sources.

config.mk.dist

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 # Subprojects to build
-SUBS = math string
+SUBS = math string networking
 
 # Target architecture: aarch64, arm or x86_64
 ARCH = aarch64
@@ -40,6 +40,7 @@ math-ldlibs =
 math-ulpflags =
 math-testflags =
 string-cflags =
+networking-cflags =
 
 # Use if mpfr is available on the target for ulp error checking.
 #math-ldlibs += -lmpfr -lgmp
@@ -58,3 +59,9 @@ math-cflags += -ffp-contract=fast -fno-math-errno
 # Disable fenv checks
 #math-ulpflags = -q -f
 #math-testflags = -nostatus
+
+# Enable assertion checks.
+#networking-cflags += -DWANT_ASSERT
+
+# Avoid auto-vectorization of scalar code and unroll loops
+networking-cflags += -O2 -fno-tree-vectorize -funroll-loops

networking/Dir.mk

@@ -0,0 +1,76 @@
+# Makefile fragment - requires GNU make
+#
+# Copyright (c) 2019-2020, Arm Limited.
+# SPDX-License-Identifier: MIT
+
+S := $(srcdir)/networking
+B := build/networking
+
+ifeq ($(ARCH),)
+all-networking check-networking install-networking clean-networking:
+	@echo "*** Please set ARCH in config.mk. ***"
+	@exit 1
+else
+
+networking-lib-srcs := $(wildcard $(S)/*.[cS]) $(wildcard $(S)/$(ARCH)/*.[cS])
+networking-test-srcs := $(wildcard $(S)/test/*.c)
+
+networking-includes := $(patsubst $(S)/%,build/%,$(wildcard $(S)/include/*.h))
+
+networking-libs := \
+	build/lib/libnetworking.so \
+	build/lib/libnetworking.a \
+
+networking-tools := \
+	build/bin/test/chksum
+
+networking-lib-objs := $(patsubst $(S)/%,$(B)/%.o,$(basename $(networking-lib-srcs)))
+networking-test-objs := $(patsubst $(S)/%,$(B)/%.o,$(basename $(networking-test-srcs)))
+
+networking-objs := \
+	$(networking-lib-objs) \
+	$(networking-lib-objs:%.o=%.os) \
+	$(networking-test-objs) \
+
+networking-files := \
+	$(networking-objs) \
+	$(networking-libs) \
+	$(networking-tools) \
+	$(networking-includes) \
+
+all-networking: $(networking-libs) $(networking-tools) $(networking-includes)
+
+$(networking-objs): $(networking-includes)
+$(networking-objs): CFLAGS_ALL += $(networking-cflags)
+
+build/lib/libnetworking.so: $(networking-lib-objs:%.o=%.os)
+	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -shared -o $@ $^
+
+build/lib/libnetworkinglib.a: $(networking-lib-objs)
+	rm -f $@
+	$(AR) rc $@ $^
+	$(RANLIB) $@
+
+build/bin/test/%: $(B)/test/%.o build/lib/libnetworkinglib.a
+	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -static -o $@ $^ $(LDLIBS)
+
+build/include/%.h: $(S)/include/%.h
+	cp $< $@
+
+build/bin/%.sh: $(S)/test/%.sh
+	cp $< $@
+
+check-networking: $(networking-tools)
+	$(EMULATOR) build/bin/test/chksum -i simple
+	$(EMULATOR) build/bin/test/chksum -i scalar
+	$(EMULATOR) build/bin/test/chksum -i simd || true # simd is not always available
+
+install-networking: \
+ $(networking-libs:build/lib/%=$(DESTDIR)$(libdir)/%) \
+ $(networking-includes:build/include/%=$(DESTDIR)$(includedir)/%)
+
+clean-networking:
+	rm -f $(networking-files)
+endif
+
+.PHONY: all-networking check-networking install-networking clean-networking

networking/aarch64/chksum_simd.c

@@ -0,0 +1,146 @@
+/*
+ * AArch64-specific checksum implementation using NEON
+ *
+ * Copyright (c) 2020, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "networking.h"
+#include "../chksum_common.h"
+
+#ifndef __ARM_NEON
+#pragma GCC target("+simd")
+#endif
+
+#include <arm_neon.h>
+
+always_inline
+static inline uint64_t
+slurp_head64(const void **pptr, uint32_t *nbytes)
+{
+    Assert(*nbytes >= 8);
+    uint64_t sum = 0;
+    uint32_t off = (uintptr_t) *pptr % 8;
+    if (likely(off != 0))
+    {
+	/* Get rid of bytes 0..off-1 */
+	const unsigned char *ptr64 = align_ptr(*pptr, 8);
+	uint64_t mask = ALL_ONES << (CHAR_BIT * off);
+	uint64_t val = load64(ptr64) & mask;
+	/* Fold 64-bit sum to 33 bits */
+	sum = val >> 32;
+	sum += (uint32_t) val;
+	*pptr = ptr64 + 8;
+	*nbytes -= 8 - off;
+    }
+    return sum;
+}
+
+always_inline
+static inline uint64_t
+slurp_tail64(uint64_t sum, const void *ptr, uint32_t nbytes)
+{
+    Assert(nbytes < 8);
+    if (likely(nbytes != 0))
+    {
+	/* Get rid of bytes 7..nbytes */
+	uint64_t mask = ALL_ONES >> (CHAR_BIT * (8 - nbytes));
+	Assert(__builtin_popcountl(mask) / CHAR_BIT == nbytes);
+	uint64_t val = load64(ptr) & mask;
+	sum += val >> 32;
+	sum += (uint32_t) val;
+	nbytes = 0;
+    }
+    Assert(nbytes == 0);
+    return sum;
+}
+
+unsigned short
+__chksum_aarch64_simd(const void *ptr, unsigned int nbytes)
+{
+    bool swap = (uintptr_t) ptr & 1;
+    uint64_t sum;
+
+    if (unlikely(nbytes < 50))
+    {
+	sum = slurp_small(ptr, nbytes);
+	swap = false;
+	goto fold;
+    }
+
+    /* 8-byte align pointer */
+    Assert(nbytes >= 8);
+    sum = slurp_head64(&ptr, &nbytes);
+    Assert(((uintptr_t) ptr & 7) == 0);
+
+    const uint32_t *may_alias ptr32 = ptr;
+
+    uint64x2_t vsum0 = { 0, 0 };
+    uint64x2_t vsum1 = { 0, 0 };
+    uint64x2_t vsum2 = { 0, 0 };
+    uint64x2_t vsum3 = { 0, 0 };
+
+    /* Sum groups of 64 bytes */
+    for (uint32_t i = 0; i < nbytes / 64; i++)
+    {
+	uint32x4_t vtmp0 = vld1q_u32(ptr32);
+	uint32x4_t vtmp1 = vld1q_u32(ptr32 + 4);
+	uint32x4_t vtmp2 = vld1q_u32(ptr32 + 8);
+	uint32x4_t vtmp3 = vld1q_u32(ptr32 + 12);
+	vsum0 = vpadalq_u32(vsum0, vtmp0);
+	vsum1 = vpadalq_u32(vsum1, vtmp1);
+	vsum2 = vpadalq_u32(vsum2, vtmp2);
+	vsum3 = vpadalq_u32(vsum3, vtmp3);
+	ptr32 += 16;
+    }
+    nbytes %= 64;
+
+    /* Fold vsum2 and vsum3 into vsum0 and vsum1 */
+    vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum2));
+    vsum1 = vpadalq_u32(vsum1, vreinterpretq_u32_u64(vsum3));
+
+    /* Add any trailing group of 32 bytes */
+    if (nbytes & 32)
+    {
+	uint32x4_t vtmp0 = vld1q_u32(ptr32);
+	uint32x4_t vtmp1 = vld1q_u32(ptr32 + 4);
+	vsum0 = vpadalq_u32(vsum0, vtmp0);
+	vsum1 = vpadalq_u32(vsum1, vtmp1);
+	ptr32 += 8;
+	nbytes -= 32;
+    }
+    Assert(nbytes < 32);
+
+    /* Fold vsum1 into vsum0 */
+    vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum1));
+
+    /* Add any trailing group of 16 bytes */
+    if (nbytes & 16)
+    {
+	uint32x4_t vtmp = vld1q_u32(ptr32);
+	vsum0 = vpadalq_u32(vsum0, vtmp);
+	ptr32 += 4;
+	nbytes -= 16;
+    }
+    Assert(nbytes < 16);
+
+    /* Add any trailing group of 8 bytes */
+    if (nbytes & 8)
+    {
+	uint32x2_t vtmp = vld1_u32(ptr32);
+	vsum0 = vaddw_u32(vsum0, vtmp);
+	ptr32 += 2;
+	nbytes -= 8;
+    }
+    Assert(nbytes < 8);
+
+    uint64_t val = vaddlvq_u32(vreinterpretq_u32_u64(vsum0));
+    sum += val >> 32;
+    sum += (uint32_t) val;
+
+    /* Handle any trailing 0..7 bytes */
+    sum = slurp_tail64(sum, ptr32, nbytes);
+
+fold:
+    return fold_and_swap(sum, swap);
+}