Skip to content

Commit 030c6e0

Browse files
marktwtnmarktwtn
authored
Merge pull request #183 from DLTcollab/develop
Pre-release v0.4.1
2 parents 631d15c + da621cf commit 030c6e0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+517
-240
lines changed

‎LICENSE‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
Copyright (C) 2018-2019 BiiLabs Co., Ltd. and Contributors.
2+
Copyright (C) 2018 Ievgen Korokyi.
23
Copyright (C) 2017 IOTA AS, IOTA Foundation and Developers.
34
Copyright (C) 2016 Shinya Yagyu.
45

‎Makefile‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
VERSION = 0.4.0
1+
VERSION = 0.4.1
22

33
OUT ?= ./build
44
SRC := src
@@ -35,6 +35,9 @@ include mk/cpu-features.mk
3535
# Handle git submodule
3636
include mk/submodule.mk
3737

38+
# Board specific compiler flags
39+
include mk/board.mk
40+
3841
# Assign the hardware to CPU if no hardware is specified
3942
PLATFORMS := $(BUILD_AVX) $(BUILD_SSE) $(BUILD_GENERIC) $(BUILD_GPU) $(BUILD_FPGA_ACCEL)
4043
ENABLE_PLATFORMS := $(findstring 1,$(PLATFORMS))
@@ -99,7 +102,7 @@ LIBS := $(addprefix $(OUT)/, $(LIBS))
99102
JARS := dcurljni-$(VERSION).jar
100103
JARS := $(addprefix $(OUT)/, $(JARS))
101104

102-
PREQ := config $(TESTS) $(LIBS)
105+
PREQ := $(SUBS) config $(TESTS) $(LIBS)
103106
ifeq ("$(BUILD_JNI)","1")
104107
PREQ += $(JARS)
105108
endif
@@ -152,27 +155,31 @@ OBJS += \
152155
remote_common.o \
153156
remote_interface.o
154157

155-
WORKER_OBJS := $(addprefix $(OUT)/worker-,$(filter-out remote_interface.o, $(OBJS)))
158+
WORKER_EXCLUDE_OBJS := remote_interface.o
159+
ifeq ("$(BUILD_JNI)", "1")
160+
WORKER_EXCLUDE_OBJS += jni/iri-pearldiver-exlib.o
161+
endif
162+
WORKER_OBJS := $(addprefix $(OUT)/worker-,$(filter-out $(WORKER_EXCLUDE_OBJS), $(OBJS)))
156163
WORKER_CFLAGS := $(filter-out -DENABLE_REMOTE, $(CFLAGS))
157164
endif
158165

159166
OBJS := $(addprefix $(OUT)/, $(OBJS))
160167

161-
$(OUT)/test-%.o: tests/test-%.c $(LIBTUV_PATH)/include
168+
$(OUT)/test-%.o: tests/test-%.c
162169
$(VECHO) " CC\t$@\n"
163-
$(Q)$(CC) -o $@ $(CFLAGS) -I $(SRC) $(LIBTUV_INCLUDE) -c -MMD -MF $@.d $<
170+
$(Q)$(CC) -o $@ $(CFLAGS) -I $(SRC) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
164171

165-
$(OUT)/%.o: $(SRC)/%.c $(LIBTUV_PATH)/include $(LIBRABBITMQ_PATH)/build/include
172+
$(OUT)/%.o: $(SRC)/%.c $(SUB_OBJS)
166173
$(VECHO) " CC\t$@\n"
167-
$(Q)$(CC) -o $@ $(CFLAGS) $(LIBTUV_INCLUDE) $(LIBRABBITMQ_INCLUDE) -c -MMD -MF $@.d $<
174+
$(Q)$(CC) -o $@ $(CFLAGS) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
168175

169-
$(OUT)/test-%: $(OUT)/test-%.o $(OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
176+
$(OUT)/test-%: $(OUT)/test-%.o $(OBJS) $(SUB_OBJS)
170177
$(VECHO) " LD\t$@\n"
171-
$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
178+
$(Q)$(CC) -o $@ $^ $(LDFLAGS)
172179

173-
$(OUT)/libdcurl.so: $(OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
180+
$(OUT)/libdcurl.so: $(OBJS) $(SUB_OBJS)
174181
$(VECHO) " LD\t$@\n"
175-
$(Q)$(CC) -shared -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
182+
$(Q)$(CC) -shared -o $@ $^ $(LDFLAGS)
176183

177184
ifeq ("$(BUILD_REMOTE)", "1")
178185
include mk/remote.mk

‎README.md‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,16 @@ In addition, dcurl supports FPGA-accelerated PoW, described in [docs/fpga-accele
1313
dcurl can be regarded as the drop-in replacement for [ccurl](https://github.com/iotaledger/ccurl).
1414
IOTA Reference Implementation (IRI) adaptation is available to benefit from hardware-accelerated PoW.
1515

16-
1716
## Build Instructions
1817
Check [docs/build-n-test.md](docs/build-n-test.md) for details.
1918

20-
2119
## Performance
2220
After integrating dcurl into IRI, performance of [attachToTangle](https://iota.readme.io/reference#attachtotangle) is measured as following.
2321
* Each sampling is measured with 30 transaction trytes and total 200 samples are measured.
2422
* mwm = 14, 26 CPU threads to find nonce
2523
* Settings: enable 2 pow tasks in CPU, 1 pow tasks in GPU at the same time
2624
![](https://raw.githubusercontent.com/DLTcollab/dcurl/develop/docs/benchmark.png)
2725

28-
2926
## IRI Adaptation
3027
[Modified IRI accepting external PoW Library](https://github.com/DLTcollab/iri)
3128
Supported IRI version: 1.7.0
@@ -63,6 +60,27 @@ pull requests to dcurl.
6360
it faciliates dcurl to perform hardware-accelerated PoW operations on
6461
edge devices.
6562

63+
## FAQ
64+
- What is **binary encoded ternary**?
65+
66+
It is a skill to transform the ternary trit value to two separate bits value.\
67+
Hence multiple trits can be compressed to two separate data of the same data type and fully utilize the space.
68+
69+
- Can the project [Batch Binary Encoded Ternary Curl](https://github.com/luca-moser/bct_curl) be applied to dcurl?
70+
71+
The answer is no.\
72+
They both use the same skill, **binary encoded ternary**.\
73+
However, their purpose are totally different.
74+
- **bct_curl**:
75+
76+
Focus on hashing the multiple data of the same length at the same time.\
77+
It ends when the hashing is finished.
78+
79+
- **dcurl**:
80+
81+
Focus on trying the different values of one transaction at the same time to find the nonce value.\
82+
The procedure of finding the nonce value also does the hashing.\
83+
However, it ends when the nonce value is found, which means one of the values is acceptable.
6684

6785
## Licensing
6886
`dcurl` is freely redistributable under the MIT License.

‎docs/build-n-test.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@
1717
- ``BUILD_SSE``: build the Intel SSE-accelerated Curl backend.
1818
- ``BUILD_GPU``: build the OpenCL-based GPU accelerations.
1919
- ``BUILD_FPGA_ACCEL``: build the interface interacting with the Cyclone V FPGA based accelerator. Verified on DE10-nano board and Arrow SoCKit board.
20+
- ``BUILD_REMOTE``: build with the remote interface and create the remote worker executable for calculating PoW on remote devices.
2021
- ``BUILD_JNI``: build a JAR file including the shared library and the JAVA bytecode for IRI. The build system would generate JNI header file
2122
downloading from [latest JAVA source](https://github.com/DLTcollab/iri).
22-
- ``BUILD_COMPAT``: build extra cCurl compatible interface.
23+
- ``BUILD_COMPAT``: build extra cCurl compatible interface. **[deprecated]**
2324
- ``BUILD_STAT``: show the statistics of the PoW information.
2425
- ``BUILD_DEBUG``: dump verbose messages internally.
2526
Build with the corresponding Sanitizer to detect software potential bugs if the value is `address`, `undefined` or `thread`.

‎docs/remote-interface.md‎

Lines changed: 76 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,76 @@
1-
# Remote interface
2-
## Introduction
3-
4-
```
5-
+-----------------------------------------------+
6-
| remote interface |
7-
| +----------------------------------------+ |
8-
| +----------------------------------------+| |
9-
| +----------------------------------------+|+ |
10-
| | RabbitMQ-provided RPC |+ |
11-
| +----------------------------------------+ |
12-
| | ^ |
13-
+------------|---------------------|------------+
14-
+------------|---------------------|------------+
15-
| | RabbitMQ broker | |
16-
| | | |
17-
| | +------------------+ |
18-
| v +------------------+| |
19-
| +------------------+ +------------------+|| |
20-
| | incoming queue | | private queue ||| |
21-
| | | | ||| |
22-
| | - trytes | | - PoW result ||| |
23-
| | - mwm | | ||+ |
24-
| | | | |+ |
25-
| +------------------+ +------------------+ |
26-
| | ^ |
27-
+------------|---------------------|------------+
28-
v |
29-
+---------------------------------------------+
30-
+---------------------------------------------+|
31-
+---------------------------------------------+|+
32-
| remote worker |+
33-
+---------------------------------------------+
34-
```
35-
To support asynchronous remote procedure call, remote interface in dcurl provides an interface named as `Remote_ImplContext` to implement it. dcurl currently uses RabbitMQ C client to implement asynchronous RPC in remote interface. Remote interface provides thread management to support an asynchronous RPC per thread.
36-
37-
Here are detailed implementations of the RabbitMQ-provided RPC pattern as follows:
38-
* Asynchronous RPC requests are inserted into the message queue, `incoming_queue`, in RabbitMQ broker
39-
* Asynchronous RPCs with exclusive private queues (callback queues) with TTL = 10s property
40-
* Correlation ID is not used
41-
* An asynchronous RPC uses a connection to RabbitMQ broker
42-
* Remote workers can obtain requests from `incoming_queue` by default exchange of RabbitMQ broker
43-
44-
## How to test remote interface in localhost
45-
You need to open three terminals
46-
47-
Terminal 1: Run the RabbitMQ broker You can quickly use docker to run the RabbitMQ broker, rabbitmq
48-
```
49-
$ sudo docker run -d rabbitmq
50-
```
51-
52-
Terminal 2: Run remote workers
53-
```
54-
$ ./build/remote-worker
55-
```
56-
How to build remote worker on FPGA board
57-
```
58-
$ make BUILD_REMOTE=1 BUILD_FPGA_ACCEL=1 BOARD=de10nano
59-
```
60-
61-
Terminal 3: Run check
62-
```
63-
$ make BUILD_REMOTE=1 BUILD_DEBUG=1 check
64-
```
65-
66-
## Requirements
67-
Remote interface requires RabbitMQ broker
1+
# Remote interface
2+
## Introduction
3+
4+
```
5+
+-----------------------------------------------+
6+
| remote interface |
7+
| +----------------------------------------+ |
8+
| +----------------------------------------+| |
9+
| +----------------------------------------+|+ |
10+
| | RabbitMQ-provided RPC |+ |
11+
| +----------------------------------------+ |
12+
| | ^ |
13+
+------------|---------------------|------------+
14+
+------------|---------------------|------------+
15+
| | RabbitMQ broker | |
16+
| | | |
17+
| | +------------------+ |
18+
| v +------------------+| |
19+
| +------------------+ +------------------+|| |
20+
| | incoming queue | | private queue ||| |
21+
| | | | ||| |
22+
| | - trytes | | - PoW result ||| |
23+
| | - mwm | | ||+ |
24+
| | | | |+ |
25+
| +------------------+ +------------------+ |
26+
| | ^ |
27+
+------------|---------------------|------------+
28+
v |
29+
+---------------------------------------------+
30+
+---------------------------------------------+|
31+
+---------------------------------------------+|+
32+
| remote worker |+
33+
+---------------------------------------------+
34+
```
35+
To support asynchronous remote procedure call, remote interface in dcurl provides an interface named as `Remote_ImplContext` to implement it.\
36+
dcurl currently uses RabbitMQ C client to implement asynchronous RPC in remote interface.\
37+
Remote interface provides thread management to support an asynchronous RPC per thread.
38+
39+
Here are detailed implementations of the RabbitMQ-provided RPC pattern as follows:
40+
* Asynchronous RPC requests are inserted into the message queue, `incoming_queue`, in RabbitMQ broker
41+
* Asynchronous RPCs with exclusive private queues (callback queues) with TTL = 10s property
42+
* Correlation ID is not used
43+
* An asynchronous RPC uses a connection to RabbitMQ broker
44+
* Remote workers can obtain requests from `incoming_queue` by default exchange of RabbitMQ broker
45+
46+
## How to test remote interface in localhost
47+
You need to open three terminals
48+
49+
**Terminal 1:**\
50+
Run the RabbitMQ broker\
51+
You can quickly use docker to run the RabbitMQ broker, rabbitmq
52+
```
53+
$ sudo docker run -d -p 5672:5672 rabbitmq
54+
```
55+
56+
---
57+
58+
**Terminal 2:**\
59+
Build and run the remote worker
60+
```
61+
$ cd dcurl
62+
$ make BUILD_REMOTE=1 BUILD_DEBUG=1
63+
$ ./build/remote-worker
64+
```
65+
66+
---
67+
68+
**Terminal 3:**\
69+
Run the tests to send requests to remote worker
70+
```
71+
$ cd dcurl
72+
$ make BUILD_REMOTE=1 BUILD_DEBUG=1 check
73+
```
74+
75+
## Requirements
76+
Remote interface requires RabbitMQ broker

‎jni/iri-pearldiver-exlib.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
/*
2+
* Copyright (C) 2018-2019 BiiLabs Co., Ltd. and Contributors
3+
* All Rights Reserved.
4+
* This is free software; you can redistribute it and/or modify it under the
5+
* terms of the MIT license. A copy of the license can be found in the file
6+
* "LICENSE" at the root of this distribution.
7+
*/
8+
19
#include "iri-pearldiver-exlib.h"
210
#include <jni.h>
311
#include <stdint.h>

‎mk/board.mk‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ifeq ($(BOARD),de10nano)
2+
CFLAGS += -mcpu=cortex-a9 -mtune=cortex-a9 -mfloat-abi=hard -mfpu=neon
3+
endif

‎mk/common.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ NO_COLOR = \e[0m
2626
$(OUT)/test-%.done: $(OUT)/test-%
2727
$(Q)$(PRINTF) "*** Validating $< ***\n"
2828
$(Q)./$< && $(PRINTF) "\t$(PASS_COLOR)[ Verified ]$(NO_COLOR)\n"
29-
check: $(addsuffix .done, $(TESTS))
29+
check: $(SUBS) $(addsuffix .done, $(TESTS))
3030

3131
config: $(OUT)/config-timestamp
3232

‎mk/remote.mk‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Build remote-worker
2-
$(OUT)/worker-%.o: $(SRC)/%.c $(LIBTUV_PATH)/include $(LIBRABBITMQ_PATH)/build/include
2+
$(OUT)/worker-%.o: $(SRC)/%.c
33
$(VECHO) " CC\t$@\n"
4-
$(Q)$(CC) -o $@ $(WORKER_CFLAGS) $(LIBTUV_INCLUDE) $(LIBRABBITMQ_INCLUDE) -c -MMD -MF $@.d $<
4+
$(Q)$(CC) -o $@ $(WORKER_CFLAGS) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
55

6-
$(OUT)/remote-worker: $(OUT)/remote_worker.o $(WORKER_OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
6+
$(OUT)/remote-worker: $(OUT)/remote_worker.o $(WORKER_OBJS) $(SUB_OBJS)
77
$(VECHO) " LD\t$@\n"
8-
$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
8+
$(Q)$(CC) -o $@ $^ $(LDFLAGS)

‎mk/submodule.mk‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,42 +18,54 @@ LIBTUV_PATH = deps/libtuv
1818
LIBTUV_INCLUDE := -I $(LIBTUV_PATH)/include
1919
LIBTUV_PLATFORM := $(UNAME_M)-$(UNAME_S)
2020
LIBTUV_BOARD := $(BOARD)
21-
# PIC (Position-Independent-Code) library
22-
LIBTUV_LIBRARY := $(LIBTUV_PATH)/build/$(LIBTUV_PLATFORM)/release/lib/libtuv.o
21+
# PIC (Position-Independent-Code) object file
22+
LIBTUV_OBJS := $(LIBTUV_PATH)/build/$(LIBTUV_PLATFORM)/release/lib/libtuv.o
2323

2424
$(LIBTUV_PATH)/include:
2525
git submodule update --init $(LIBTUV_PATH)
2626

27-
$(LIBTUV_LIBRARY):
27+
$(LIBTUV_OBJS):
2828
$(MAKE) -C $(LIBTUV_PATH) TUV_BUILD_TYPE=release TUV_CREATE_PIC_LIB=yes TUV_PLATFORM=$(LIBTUV_PLATFORM) TUV_BOARD=$(LIBTUV_BOARD)
2929

3030
# librabbitmq related variables
3131
LIBRABBITMQ_PATH = deps/rabbitmq-c
3232
LIBRABBITMQ_INCLUDE := -I $(LIBRABBITMQ_PATH)/build/include
33-
LIBRABBITMQ_LIB_PATH := $(LIBRABBITMQ_PATH)/build/librabbitmq/
33+
LIBRABBITMQ_OBJS := $(LIBRABBITMQ_PATH)/build/librabbitmq/CMakeFiles/rabbitmq.dir/*.o
3434
ifeq ($(UNAME_S),darwin)
35-
# macOS
36-
LIBRABBITMQ_LINK := -Wl,-rpath,$(LIBRABBITMQ_LIB_PATH) -L$(LIBRABBITMQ_LIB_PATH) -lrabbitmq
37-
LIBRABBITMQ_LIBRARY := $(LIBRABBITMQ_LIB_PATH)/librabbitmq.dylib
38-
else
39-
LIBRABBITMQ_LINK := -Wl,-rpath=$(LIBRABBITMQ_LIB_PATH) -L$(LIBRABBITMQ_LIB_PATH) -lrabbitmq
40-
LIBRABBITMQ_LIBRARY := $(LIBRABBITMQ_LIB_PATH)/librabbitmq.so
35+
OPENSSL_PATH := /usr/local/opt/openssl
36+
LDFLAGS += -L$(OPENSSL_PATH)/lib -lcrypto -lssl
4137
endif
4238

43-
$(LIBRABBITMQ_PATH)/build/include:
39+
$(LIBRABBITMQ_PATH)/librabbitmq:
4440
git submodule update --init $(LIBRABBITMQ_PATH)
45-
mkdir $(LIBRABBITMQ_PATH)/build
41+
42+
$(LIBRABBITMQ_OBJS):
43+
mkdir -p $(LIBRABBITMQ_PATH)/build
4644
ifeq ($(UNAME_S),darwin)
4745
# macOS
4846
cd $(LIBRABBITMQ_PATH)/build && \
49-
cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl/ -DCMAKE_INSTALL_PREFIX=. .. && \
47+
cmake -DOPENSSL_ROOT_DIR=$(OPENSSL_PATH) -DCMAKE_INSTALL_PREFIX=. .. && \
5048
cmake --build . --target install
5149
else
5250
cd $(LIBRABBITMQ_PATH)/build && \
5351
cmake -DCMAKE_INSTALL_PREFIX=. .. && \
5452
cmake --build . --target install
5553
endif
56-
57-
$(LIBRABBITMQ_LIBRARY):
5854
cd $(LIBRABBITMQ_PATH)/build && \
5955
cmake --build .
56+
57+
# Submodules
58+
SUBS := $(LIBTUV_PATH)/include
59+
ifeq ($(BUILD_REMOTE),1)
60+
SUBS += $(LIBRABBITMQ_PATH)/librabbitmq
61+
endif
62+
# Submodule related objects
63+
SUB_OBJS := $(LIBTUV_OBJS)
64+
ifeq ($(BUILD_REMOTE),1)
65+
SUB_OBJS += $(LIBRABBITMQ_OBJS)
66+
endif
67+
# Submodule C flags for including header files
68+
SUB_INCLUDE := $(LIBTUV_INCLUDE)
69+
ifeq ($(BUILD_REMOTE),1)
70+
SUB_INCLUDE += $(LIBRABBITMQ_INCLUDE)
71+
endif

0 commit comments

Comments
 (0)