bsdkm: x86 crypto acceleration support.

Author: philljj

.gitignore

@@ -239,12 +239,17 @@ linuxkm/linuxkm
 linuxkm/src
 linuxkm/patches/src
 *.nds
+
+# Generated during FreeBSD kernel module build.
 bsdkm/export_syms
 bsdkm/i386
 bsdkm/libwolfssl.ko
 bsdkm/machine
 bsdkm/opt_global.h
 bsdkm/x86
+bsdkm/bus_if.h
+bsdkm/cryptodev_if.h
+bsdkm/device_if.h
 
 # autotools generated
 scripts/unit.test

Makefile.am

@@ -247,8 +247,9 @@ if BUILD_BSDKM
         EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \
         AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \
         AM_CCASFLAGS CCASFLAGS \
-        src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS
-
+        src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \
+        ENABLED_ASM ENABLED_AESNI \
+        ENABLED_KERNEL_BENCHMARKS
 endif
 
 

bsdkm/Makefile

@@ -1,39 +1,52 @@
 # wolfssl kernel module name and source, and root dir.
-KMOD=libwolfssl
-SRCS=wolfkmod.c
-WOLFSSL_DIR=../
+KMOD        = libwolfssl
+SRCS        = wolfkmod.c
+WOLFSSL_DIR = ../
 
 CFLAGS+=-I${WOLFSSL_DIR}
 CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
+
 #
 # debug options
 #   verbose printing:
-#   CFLAGS+=-DWOLFSSL_BSDKM_VERBOSE_DEBUG
+#   CFLAGS += -DWOLFSSL_BSDKM_VERBOSE_DEBUG
 #
 #   print memory mallocs / frees:
-#   CFLAGS+=-DWOLFSSL_BSDKM_MEMORY_DEBUG
+#   CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG
 #
-CFLAGS+=$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+
+.if defined(ENABLED_BSDKM_REGISTER)
+  # These device header files are generated during build.
+  SRCS += bus_if.h cryptodev_if.h device_if.h
+.endif
 
 # FreeBSD make does not support GNU make's patsubst and related. Filter
 # through sed instead.
 WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \
   sed 's|src_libwolfssl_la-||g' | sed 's|\.lo|.o|g' | \
   sed 's|wolfcrypt/src/|${WOLFSSL_DIR}/wolfcrypt/src/|g'
 
+# wolfcrypt test
 .if ${ENABLED_CRYPT_TESTS} == "yes"
     WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/test/test.o
 .else
-    CFLAGS+=-DNO_CRYPT_TEST
+    CFLAGS += -DNO_CRYPT_TEST
+.endif
+
+# wolfcrypt benchmark
+.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
+    WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o
+    CFLAGS+=-DWOLFSSL_NO_FLOAT_FMT
 .endif
 
 OBJS += ${WOLFSSL_OBJS}
 
 # Export no public symbols by default.
 .if !defined(BSDKM_EXPORT_SYMS)
-    EXPORT_SYMS=NO
+    EXPORT_SYMS = NO
 .else
-    EXPORT_SYMS=${BSDKM_EXPORT_SYMS}
+    EXPORT_SYMS = ${BSDKM_EXPORT_SYMS}
 .endif
 
 # Default to live kernel src tree makefile at
@@ -45,12 +58,37 @@ OBJS += ${WOLFSSL_OBJS}
 .endif
 .include "${SYSDIR}/conf/kmod.mk"
 
+# To use aesni in FreeBSD kernel we need to adjust build flags.
+# See these for reference:
+#   - /usr/src/sys/modules/aesni/Makefile
+#   - /usr/src/sys/conf/kern.mk
+.if ${ENABLED_ASM} == "yes"
+  CFLAGS.aes.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2
+  CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc}
+  CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+
+  CFLAGS.poly1305.c += -msse -msse2 -msse4.1 -maes -mpclmul
+  CFLAGS.poly1305.c := ${CFLAGS.aes.c:N-nostdinc}
+  CFLAGS.poly1305.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+
+  CFLAGS.chacha.c += -msse -msse2 -msse4.1 -maes -mpclmul
+  CFLAGS.chacha.c := ${CFLAGS.aes.c:N-nostdinc}
+  CFLAGS.chacha.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+
+  CFLAGS.sha.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2
+  CFLAGS.sha256.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2
+  CFLAGS.benchmark.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2
+  CFLAGS.benchmark.c := ${CFLAGS.aes.c:N-nostdinc}
+  CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+  .PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif
+
 # Smooth out a few inconsistencies between FreeBSD default compiler flags
 # in /usr/src/sys/conf/kern.mk, vs wolfssl harden flags in
 # m4/ax_harden_compiler_flags.m4. E.g. some FreeBSD header files shorten
 # 64 to 32 bit, and some wolfcrypt functions cast away const.
-CFLAGS+= -Wno-unused-function
-CFLAGS+= -Wno-cast-qual
-CFLAGS+= -Wno-error=cast-qual
-CFLAGS+= -Wno-shorten-64-to-32
-CFLAGS+= -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
+CFLAGS += -Wno-unused-function
+CFLAGS += -Wno-cast-qual
+CFLAGS += -Wno-error=cast-qual
+CFLAGS += -Wno-shorten-64-to-32
+CFLAGS += -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""

bsdkm/bsdkm_wc_port.h

@@ -61,13 +61,13 @@ static inline time_t wolfkmod_time(time_t * tloc) {
 #define WOLFSSL_DEBUG_PRINTF_FN printf
 
 /* str and char utility functions */
-#define XATOI(s) ({                                         \
-      char * endptr = NULL;                                 \
-      long   _xatoi_ret = strtol(s, &endptr, 10);           \
-      if ((s) == endptr || *endptr != '\0') {               \
-        _xatoi_ret = 0;                                     \
-      }                                                     \
-      (int)_xatoi_ret;                                      \
+#define XATOI(s) ({                                                          \
+      char * endptr = NULL;                                                  \
+      long   _xatoi_ret = strtol(s, &endptr, 10);                            \
+      if ((s) == endptr || *endptr != '\0') {                                \
+        _xatoi_ret = 0;                                                      \
+      }                                                                      \
+      (int)_xatoi_ret;                                                       \
 })
 
 #if !defined(XMALLOC_OVERRIDE)
@@ -103,6 +103,34 @@ extern struct malloc_type M_WOLFSSL[1];
     })
 #endif /* WOLFSSL_BSDKM_DEBUG_MEMORY */
 
+
+#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+    int  wolfkmod_vecreg_init(void);
+    void wolfkmod_vecreg_exit(void);
+    int  wolfkmod_vecreg_save(int flags_unused);
+    void wolfkmod_vecreg_restore(void);
+    /* wrapper defines for FPU_KERN(9).
+     * /usr/src/sys/amd64/amd64/fpu.c
+     * /usr/src/sys/amd64/include/pcb.h
+     * */
+    #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+        #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+    #endif
+
+
+    #define SAVE_VECTOR_REGISTERS(fail_clause) {                             \
+        int _svr_ret = wolfkmod_vecreg_save(0);                              \
+        if (_svr_ret != 0) {                                                 \
+            fail_clause                                                      \
+        }                                                                    \
+    }
+
+    #define SAVE_VECTOR_REGISTERS2() wolfkmod_vecreg_save(0)
+
+    #define RESTORE_VECTOR_REGISTERS() wolfkmod_vecreg_restore()
+
+#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS */
+
 #if !defined(SINGLE_THREADED)
     #define WC_MUTEX_OPS_INLINE
 
@@ -149,7 +177,8 @@ extern struct malloc_type M_WOLFSSL[1];
     typedef volatile int          wolfSSL_Atomic_Int;
     typedef volatile unsigned int wolfSSL_Atomic_Uint;
     #define WOLFSSL_ATOMIC_INITIALIZER(x) (x)
-    #define WOLFSSL_ATOMIC_LOAD(x)  (int)atomic_load_acq_int(&(x))
+    #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
+    #define WOLFSSL_ATOMIC_LOAD_UINT(x) atomic_load_acq_int(&(x))
     #define WOLFSSL_ATOMIC_STORE(x, v)  atomic_store_rel_int(&(x), (v))
     #define WOLFSSL_ATOMIC_OPS
 

bsdkm/include.am

@@ -2,8 +2,10 @@
 # included from Top Level Makefile.am
 # All paths should be given relative to the root
 
-EXTRA_DIST += m4/ax_bsdkm.m4 \
-	      bsdkm/Makefile \
-	      bsdkm/README.md \
-	      bsdkm/wolfkmod.c \
+EXTRA_DIST += m4/ax_bsdkm.m4       \
+	      bsdkm/Makefile       \
+	      bsdkm/README.md      \
+	      bsdkm/wolfkmod.c     \
+	      bsdkm/wolfkmod_aes.c \
+	      bsdkm/x86_vecreg.c   \
 	      bsdkm/bsdkm_wc_port.h