Merge pull request #33 from dgarske/docs

Updates to documentation for async

Author: cconlon

README-async.md

@@ -64,6 +64,14 @@ void wolfAsync_DevClose(int *devId)
 
 Closes the async device.
 
+### ```wolfAsync_DevCopy```
+
+```
+int wolfAsync_DevCopy(WC_ASYNC_DEV* src, WC_ASYNC_DEV* dst);
+```
+
+Copy async device memory safe (not pointers to old device).
+
 ### ```wolfAsync_DevCtxInit```
 ```
 int wolfAsync_DevCtxInit(WC_ASYNC_DEV* asyncDev, word32 marker, void* heap, int devId);
@@ -156,74 +164,60 @@ Stops hardware if internal `--start_count == 0`.
 
 ### TLS Server Example
 
-```
-#ifdef WOLFSSL_ASYNC_CRYPT
-    static int devId = INVALID_DEVID;
-
-    ret = wolfAsync_DevOpen(&devId);
-    if (ret != 0) {
-        err_sys("Async device open failed");
+```c
+int devId = INVALID_DEVID;
+
+ret = wolfAsync_DevOpen(&devId);
+if (ret != 0) {
+    err_sys("Async device open failed");
+}
+wolfSSL_CTX_SetDevId(ctx, devId);
+
+do {
+    err = 0; /* reset error */
+    ret = wolfSSL_accept(ssl, msg, msgSz, &msgSz);
+    if (ret <= 0) {
+        err = wolfSSL_get_error(ssl, 0);
+        if (err == WC_PENDING_E) {
+            ret = wolfSSL_AsyncPoll(ssl, WOLF_POLL_FLAG_CHECK_HW);
+            if (ret < 0) break;
+        }
     }
-    wolfSSL_CTX_UseAsync(ctx, devId);
-#endif /* WOLFSSL_ASYNC_CRYPT */
-
-	err = 0;
-	do {
-	#ifdef WOLFSSL_ASYNC_CRYPT
-	    if (err == WC_PENDING_E) {
-	       ret = wolfSSL_AsyncPoll(ssl);
-	       if (ret < 0) { break; } else if (ret == 0) { continue; }
-	    }
-	#endif
-
-	    ret = wolfSSL_accept(ssl);
-	    if (ret != SSL_SUCCESS) {
-	        err = wolfSSL_get_error(ssl, 0);
-	    }
-	} while (ret != SSL_SUCCESS && err == WC_PENDING_E);
+} while (err == WC_PENDING_E);
+if (ret != WOLFSSL_SUCCESS) {
+    err_sys("SSL_connect failed");
+}
 
-#ifdef WOLFSSL_ASYNC_CRYPT
-    wolfAsync_DevClose(&devId);
-#endif
+wolfAsync_DevClose(&devId);
 ```
 
 ### wolfCrypt RSA Example
 
-```
-#ifdef WOLFSSL_ASYNC_CRYPT
-    static int devId = INVALID_DEVID;
-
-    ret = wolfAsync_DevOpen(&devId);
-    if (ret != 0) {
-        err_sys("Async device open failed");
-    }
-#endif /* WOLFSSL_ASYNC_CRYPT */
+```c
+static int devId = INVALID_DEVID;
+RsaKey key;
 
-	RsaKey key;
-	ret = wc_InitRsaKey_ex(&key, HEAP_HINT, devId);
+ret = wolfAsync_DevOpen(&devId);
+if (ret != 0)
+    err_sys("Async device open failed");
 
-	ret = wc_RsaPrivateKeyDecode(tmp, &idx, &key, (word32)bytes);
-
-	do {
-#if defined(WOLFSSL_ASYNC_CRYPT)
+wc_InitRsaKey_ex(&key, HEAP_HINT, devId);
+if (ret == 0) {
+    ret = wc_RsaPrivateKeyDecode(tmp, &idx, &key, (word32)bytes);
+    do {
         ret = wc_AsyncWait(ret, &key.asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
-#endif
-        if (ret >= 0) {
+        if (ret >= 0)
             ret = wc_RsaPublicEncrypt(in, inLen, out, outSz, &key, &rng);
-        }
     } while (ret == WC_PENDING_E);
-    if (ret < 0) {
-        err_sys("RsaPublicEncrypt operation failed");
-    }
+    wc_FreeRsaKey(&key);
+}
 
-#ifdef WOLFSSL_ASYNC_CRYPT
-    wolfAsync_DevClose(&devId);
-#endif
+wolfAsync_DevClose(&devId);
 ```
 
 ## Build Options
 
-1. Async mult-threading can be disabled by defining `WC_NO_ASYNC_THREADING`.
+1. Async multi-threading can be disabled by defining `WC_NO_ASYNC_THREADING`. This only disables internal async threading functions. You are free to use other threading APIs or paradigms in your application.
 2. Software benchmarks can be disabled by defining `NO_SW_BENCH`.
 3. The `WC_ASYNC_THRESH_NONE` define can be used to disable the cipher thresholds, which are tunable values to determine at what size hardware should be used vs. software.
 4. Use `WOLFSSL_DEBUG_MEMORY` and `WOLFSSL_TRACK_MEMORY` to help debug memory issues. QAT also supports `WOLFSSL_DEBUG_MEMORY_PRINT`.
@@ -239,31 +233,53 @@ We have a full TLS client/server async examples here:
 
 * [https://github.com/wolfSSL/wolfssl-examples/blob/master/tls/client-tls-perf.c](https://github.com/wolfSSL/wolfssl-examples/blob/master/tls/client-tls-perf.c)
 
-#### Usage
+#### TLS Threaded epoll Example Building
 
-```
+```sh
 git clone [email protected]:wolfSSL/wolfssl-examples.git
 cd wolfssl-examples
 cd tls
+# For QuickAssist: Uncomment QAT lines at top of Makefile
 make
-sudo ./server-tls-epoll-perf
-sudo ./client-tls-perf
-```
-
-```
-Waiting for a connection...
-SSL cipher suite is TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-wolfSSL Client Benchmark 16384 bytes
-	Num Conns         :       100
-	Total             :   777.080 ms
-	Total Avg         :     7.771 ms
-	t/s               :   128.687
-	Accept            :   590.556 ms
-	Accept Avg        :     5.906 ms
-	Total Read bytes  :   1638400 bytes
-	Total Write bytes :   1638400 bytes
-	Read              :    73.360 ms (   21.299 MBps)
-	Write             :    74.535 ms (   20.963 MBps)
+```
+
+#### TLS Threaded epoll Example Usage
+
+```sh
+$ ./client-tls-perf -?
+perf 4.5.0 (NOTE: All files relative to wolfSSL home dir)
+-?          Help, print this usage
+-p <num>    Port to listen on, not 0, default 11111
+-v <num>    SSL version [0-3], SSLv3(0) - TLS1.2(3)), default 3
+-l <str>    Cipher suite list (: delimited)
+-c <file>   Certificate file,           default ../certs/client-cert.pem
+-k <file>   Key file,                   default ../certs/client-key.pem
+-A <file>   Certificate Authority file, default ../certs/ca-cert.pem
+-r          Resume session
+-n <num>    Benchmark <num> connections
+-N <num>    <num> concurrent connections
+-R <num>    <num> bytes read from client
+-W <num>    <num> bytes written to client
+-B <num>    Benchmark <num> written bytes
+```
+
+#### TLS Threaded epoll Example Output
+
+```sh
+$ sudo ./server-tls-epoll-threaded -n 10000
+$ sudo ./client-tls-perf -n 10000
+
+wolfSSL Server Benchmark 16384 bytes
+	Num Conns         :     10000
+	Total             : 18575.800 ms
+	Total Avg         :     1.858 ms
+	t/s               :   538.335
+	Accept            : 35848.428 ms
+	Accept Avg        :     3.585 ms
+	Total Read bytes  : 163840000 bytes
+	Total Write bytes : 163840000 bytes
+	Read              :   402.212 ms (  388.476 MBps)
+	Write             :   591.469 ms (  264.173 MBps)
 ```
 
 ## Change Log

wolfcrypt/src/port/cavium/README.md

@@ -11,12 +11,51 @@ Tested using `CNN55XX-Driver-Linux-KVM-XEN-PF-SDK-1.4.14.tar`
 
 ### Installation
 
+```sh
+$ cd CN55XX-SDK
+$ make clean
+$ make
+$ cd bin
+$ sudo perl ./init_nitrox.pl
+
+NITROX-V devices found: 1
+NITROX-V driver(nitrox_drv.ko) load: SUCCESS
+NITROX-V Device-0 part:  CNN5560-900BG676-C45-G
+
+Reading config file: ../microcode/ssl.conf
+Device count: 1  Config file device count: 2
+
+ NITROX Model: 0x1200 [ CNN55XX PASS 1.0 ]
+
+ Microcode Details:
+	Version : CNN5x-MC-AE-MAIN-0001
+	Core Count : 80
+	Code length : 9514
+	Block number: 0
+
+ Microcode Details:
+	Version : CNN5x-MC-SE-SSL-0004
+	Core Count : 64
+	Code length : 23738
+	Block number: 1
+
+ Microcode Load Succeed on device: 0
+
+ [ AE ] Microcode: CNN5x-MC-AE-MAIN-0001
+	Group : 0
+	Core Mask [Hi Low]: ffff ffffffffffffffff [ 80 ]
+
+ [ SE ] Microcode: CNN5x-MC-SE-SSL-0004
+	Group : 0
+	Core Mask : ffffffffffffffff [ 64 ]
+
+Microcode Load success
 ```
-cd CN55XX-SDK
-make clean
-make
-cd bin
-sudo perl ./init_nitrox.pl 
+
+```sh
+$ lspci | grep Cavium
+09:00.0 Network and computing encryption device: Cavium, Inc. Nitrox XL NPX (rev 01)
+81:00.0 Network and computing encryption device: Cavium, Inc. Device 0012
 ```
 
 #### Issues
@@ -25,7 +64,7 @@ sudo perl ./init_nitrox.pl
 
 a. Modify `include/vf_defs.h:120` -> `vf_config_mode_str()` function to:
 
-```
+```c
 static inline const char *vf_config_mode_str(vf_config_type_t vf_mode)
 {
 	const char *vf_mode_str;
@@ -38,14 +77,14 @@ c. In `include/linux/sysdep.h:46` rename `__BYTED_ORDER` to `__BYTE_ORDER`.
 
 2. If the CNN55XX driver is not extracted on the Linux box it can cause issues with the symbolic links in the microcode folder. Fix was to resolve the symbolic links in `./microcode`.
 
-```
+```sh
 NITROX Model: 0x1200 [ CNN55XX PASS 1.0 ]
 Invalid microcode
 ucode_dload: failed to initialize
 ```
 
 Resolve Links:
-```
+```sh
 cd microcode
 rm main_asym.out
 ln -s ./build/main_ae.out ./main_asym.out
@@ -58,7 +97,7 @@ ls -s ./build/main_ssl.out ./main_ssl.out
 
 ## Building wolfSSL
 
-```
+```sh
 ./configure --with-cavium-v=../CNN55XX-SDK --enable-asynccrypt --enable-aesni --enable-intelasm
 make
 sudo make install
@@ -82,7 +121,7 @@ Include the libnitrox static library:
 `LDFLAGS+= ../CNN55XX-SDK/lib/libnitrox.a`
 
 
-### Issues
+### wolfSSL Build Issues
 
 a. If building with debug `-g` and using an older binutils LD version 2.23 or less you may see a linker crash. Example of error: `BFD (GNU Binutils) 2.23.2 internal error, aborting at merge.c line 873 in _bfd_merged_section_offset`. Resolution is to use this in the CFLAGS `-g -fno-merge-debug-strings -fdebug-types-section`.
 
@@ -97,9 +136,9 @@ sudo ./wolfcrypt/test/testwolfcrypt
 ```
 
 
-## TLS Code Tempalte
+## TLS Code Template
 
-```
+```c
 /* GLOBAL DEVICE IDENTIFIER */
 #ifdef WOLFSSL_ASYNC_CRYPT
 	static int devId = INVALID_DEVID;
@@ -115,7 +154,6 @@ sudo ./wolfcrypt/test/testwolfcrypt
     wolfSSL_CTX_UseAsync(ctx, devId);
 #endif
 
-
 /* DONE IN YOUR WORKER LOOP IN WC_PENDING_E CASES AGAINST YOUR WOLFSSL_CTX */
 #ifdef WOLFSSL_ASYNC_CRYPT
 	int ret;
@@ -135,7 +173,6 @@ sudo ./wolfcrypt/test/testwolfcrypt
 	}
 #endif
 
-
 /* DONE AT CLEANUP */
 #ifdef WOLFSSL_ASYNC_CRYPT
     wolfAsync_DevClose(&devId);
@@ -150,7 +187,7 @@ CentOS: Kernel 3.10.0-514.16.1.el7.x86_64
 Single Thread
 
 ```
-./configure --with-cavium-v=../CNN55XX-SDK --enable-asynccrypt --enable-aesni --enable-intelasm CFLAGS="-DWC_NO_ASYNC_THREADING" && make
+./configure --with-cavium-v=../CNN55XX-SDK --enable-asynccrypt --enable-aesni --enable-intelasm --enable-sp --enable-sp-asm CFLAGS="-DWC_NO_ASYNC_THREADING" && make
 
 sudo ./wolfcrypt/benchmark/benchmark