Skip to content

Commit 1441047

Browse files
jmwamplejmwample
authored
Infrastructure fixes (#181)
* inf improvements. incorrect path in cj-reg. prevent infinitely adding conjure iptables rules * typos * iptables chain jump order * ensure builds are working for docker * working on a station
1 parent affe724 commit 1441047

File tree

6 files changed

+114
-60
lines changed

6 files changed

+114
-60
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ libtapdance/genkey
66
cmd/registration-server/registration-server
77
target
88
bin/
9+
Cargo.lock

‎docker/Dockerfile‎

Lines changed: 23 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ touch /opt/conjure/src/lib.rs
5454
PATH="$HOME/.cargo/bin:$PATH" make rust
5555
EOF
5656

57-
RUN PATH="$HOME/.cargo/bin:$PATH" make conjure-sim && mv conjure conjure-sim
57+
RUN PATH="$HOME/.cargo/bin:$PATH" make conjure-sim && mv bin/conjure bin/conjure-sim
5858
RUN PATH="$HOME/.cargo/bin:$PATH" make conjure
5959

6060
COPY go.* /opt/conjure/
@@ -66,87 +66,77 @@ RUN PATH="$HOME/.go/bin/:$PATH" make app
6666
RUN PATH="$HOME/.go/bin/:$PATH" make registration-server
6767

6868
# Add default configs and launch scripts
69-
COPY sysconfig/ /opt/conjure/sysconfig
70-
COPY scripts/ /opt/conjure/bin
69+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
70+
COPY scripts/ /opt/conjure/scripts
7171

7272
# add application as default entrypoint for dev reasons.
73-
ENTRYPOINT /opt/conjure/application/application
73+
ENTRYPOINT /opt/conjure/scripts/start_application.sh
7474

7575

7676
# ------------------------------------------------------------------------------
7777
# Production image zbalance only
7878
# ------------------------------------------------------------------------------
7979
FROM base_pfring as conjure_zbalance
8080
# Add default configs and launch scripts
81-
COPY sysconfig/ /opt/conjure/sysconfig
82-
COPY scripts/ /opt/conjure/bin
83-
84-
ENTRYPOINT /opt/conjure/bin/start_zbalance_ipc.sh
81+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
82+
COPY scripts/start_zbalance_ipc.sh /opt/conjure/scripts/
8583

84+
ENTRYPOINT /opt/conjure/scripts/start_zbalance_ipc.sh
8685

8786
# ------------------------------------------------------------------------------
8887
# Production image detector only (kind of, requires pfring)
8988
# ------------------------------------------------------------------------------
9089
FROM base_pfring as conjure_det
9190
# Add default configs and launch scripts
92-
COPY sysconfig/ /opt/conjure/sysconfig
93-
COPY scripts/ /opt/conjure/bin
91+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig
92+
COPY scripts/ /opt/conjure/scripts
9493

9594
RUN apt update && apt install -yq libzmq3-dev
96-
COPY --from=dev_img /opt/conjure/conjure /opt/conjure/bin/
97-
95+
COPY --from=dev_img /opt/conjure/bin/conjure /opt/conjure/bin/
9896

9997
# ------------------------------------------------------------------------------
10098
# Production image application only
10199
# ------------------------------------------------------------------------------
102100
FROM ubuntu:20.04 as conjure_app
103101
# Add default configs and launch scripts
104-
COPY sysconfig/ /opt/conjure/sysconfig
105-
COPY scripts/ /opt/conjure/bin
106-
COPY application/ /opt/conjure/application
102+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
103+
COPY scripts/ /opt/conjure/scripts
107104

108105
RUN apt update && apt install -yq libzmq3-dev
109-
COPY --from=dev_img /opt/conjure/application/application /opt/conjure/bin/
110-
106+
COPY --from=dev_img /opt/conjure/bin/application /opt/conjure/bin/
111107

112108
# ------------------------------------------------------------------------------
113109
# Production image registration server only
114110
# ------------------------------------------------------------------------------
115111
FROM ubuntu:20.04 as conjure_reg
116112
# Add default configs and launch scripts
117-
COPY sysconfig/ /opt/conjure/sysconfig
118-
COPY scripts/ /opt/conjure/bin
113+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
114+
COPY scripts/ /opt/conjure/scripts/
119115

120116
RUN apt update && apt install -yq libzmq3-dev
121-
COPY --from=dev_img /opt/conjure/cmd/registration-server/registration-server /opt/conjure/bin/
122-
117+
COPY --from=dev_img /opt/conjure/bin/registration-server /opt/conjure/bin/
123118

124119
# ------------------------------------------------------------------------------
125120
# Simulation image (no pfring required)
126121
# ------------------------------------------------------------------------------
127122
FROM ubuntu:20.04 as conjure_sim
128123
# Add default configs and launch scripts
129-
COPY sysconfig/ /opt/conjure/sysconfig
130-
COPY scripts/ /opt/conjure/bin
124+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
125+
COPY scripts/ /opt/conjure/scripts/
131126

132127
RUN apt update && apt install -yq libzmq3-dev
133-
COPY --from=dev_img /opt/conjure/conjure-sim /opt/conjure/bin/conjure
134-
COPY --from=dev_img /opt/conjure/cmd/registration-server/registration-server /opt/conjure/bin/
135-
COPY --from=dev_img /opt/conjure/conjure /opt/conjure/bin/
136-
128+
COPY --from=dev_img /opt/conjure/bin/conjure-sim /opt/conjure/bin/conjure
129+
COPY --from=dev_img /opt/conjure/bin/registration-server /opt/conjure/bin/
137130

138131
# ------------------------------------------------------------------------------
139132
# Production image all (default)
140133
# ------------------------------------------------------------------------------
141134
FROM base_pfring as conjure
142135
# Add default configs and launch scripts
143-
COPY sysconfig/ /opt/conjure/sysconfig
144-
COPY scripts/ /opt/conjure/bin
145-
COPY application/ /opt/conjure/application
136+
COPY sysconfig/conjure.conf /opt/conjure/sysconfig/
137+
COPY scripts/ /opt/conjure/scripts/
146138

147139
RUN apt update && apt install -yq libzmq3-dev
148-
COPY --from=dev_img /opt/conjure/application/application /opt/conjure/bin/
149-
COPY --from=dev_img /opt/conjure/cmd/registration-server/registration-server /opt/conjure/bin/
150-
COPY --from=dev_img /opt/conjure/conjure /opt/conjure/bin/
140+
COPY --from=dev_img /opt/conjure/bin/ /opt/conjure/bin/
151141

152142
# ENTRYPOINT /bin/bash

‎docker/docker-compose.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ services:
2121
logging:
2222
driver: "journald"
2323
restart: "always"
24-
command: [ "/opt/conjure/bin/start_zbalance_ipc.sh" ]
24+
command: [ "/opt/conjure/scripts/start_zbalance_ipc.sh" ]
2525

2626
# # DEBUG PF_RING / zbalance
2727
# zcount:
@@ -43,7 +43,7 @@ services:
4343
[
4444
"bash",
4545
"-c",
46-
"sleep 10 && /opt/conjure/bin/start_detector.sh"
46+
"sleep 10 && /opt/conjure/scripts/start_detector.sh"
4747
]
4848

4949
conjure-app:
@@ -59,5 +59,5 @@ services:
5959
[
6060
"bash",
6161
"-c",
62-
"sleep 10 && /opt/conjure/bin/start_application.sh"
62+
"sleep 10 && /opt/conjure/scripts/start_application.sh"
6363
]

‎on-reboot.sh‎

Lines changed: 85 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Use this script once after reboot, and on configuration changes
44
# that affect variables in the config file.
55
# See README.md
6-
# Run as sud
6+
# Run as sudo
77

88
# if conjure path is unset (not empty -- unset) use the default.
99
if [ -z "${CJ_PATH+x}" ]; then
@@ -24,7 +24,74 @@ exit_msg() {
2424
exit 1
2525
}
2626

27-
# Otherwise, this section will require constant updating.
27+
build_or_rebuild_iptables() {
28+
if [[ $# -lt 3 ]]; then
29+
exit_msg "script broken, build_or_rebuild requires iptables table, chain, and source chain names"
30+
fi
31+
local table=$1
32+
local chain=$2
33+
local src_chain=$3
34+
iptables -t ${table} -n -L ${chain} >/dev/null 2>&1
35+
if [ "$?" -eq 0 ]; then
36+
# Chain already exists
37+
while [ "$?" -eq 0 ];
38+
do
39+
iptables -t ${table} -D ${src_chain} -j ${chain}
40+
done
41+
iptables -t ${table} -F ${chain}
42+
iptables -t ${table} -X ${chain}
43+
fi
44+
echo "building chain ${chain} in table ${table}"
45+
iptables -t ${table} -N ${chain}
46+
do_or_die "iptables -t ${table} -I ${src_chain} 1 -j ${chain}"
47+
48+
ip6tables -t ${table} -n -L ${chain} >/dev/null 2>&1
49+
if [ "$?" -eq 0 ]; then
50+
# Chain already exists
51+
while [ "$?" -eq 0 ];
52+
do
53+
ip6tables -t ${table} -D ${src_chain} -j ${chain}
54+
done
55+
ip6tables -t ${table} -F ${chain}
56+
ip6tables -t ${table} -X ${chain}
57+
fi
58+
ip6tables -t ${table} -N ${chain}
59+
do_or_die "ip6tables -t ${table} -I ${src_chain} 1 -j ${chain}"
60+
}
61+
62+
tun_setup_fn () {
63+
if [[ $# -lt 2 ]]; then
64+
exit_msg "script broken, tun_setup requires tun id and ip table name"
65+
fi
66+
67+
local N=$1
68+
local table=$2
69+
ip tuntap del mode tun tun${N}
70+
do_or_die "ip tuntap add mode tun tun${N}"
71+
do_or_die "sysctl -w net.ipv4.conf.tun${N}.rp_filter=0"
72+
73+
local rule_condition="iif tun${N} lookup ${table}"
74+
# Check if the rule exists
75+
local output=$(ip rule show | grep "$rule_condition")
76+
if ! [[ -n "$output" ]]; then
77+
# if not, add it
78+
do_or_die "ip rule add iif tun${N} lookup ${table}"
79+
fi
80+
81+
# not sure if we need to do rule below for every tun
82+
# `RTNETLink answers: File exists` means the route is already there; harmless, but can we avoid it?
83+
ip route add local 0.0.0.0/0 dev tun${N} table ${table}
84+
85+
86+
do_or_die "iptables -t nat -I CJ_PREROUTING 1 -p tcp -i tun${N} -j DNAT --to ${IP4_ADDR}:41245"
87+
do_or_die "iptables -t nat -I CJ_PREROUTING 1 -p udp -i tun${N} -j DNAT --to ${IP4_ADDR}:41245"
88+
do_or_die "ip6tables -t nat -I CJ_PREROUTING 1 -p tcp -i tun${N} -j DNAT --to ${IP6_ADDR}:41245"
89+
do_or_die "ip6tables -t nat -I CJ_PREROUTING 1 -p udp -i tun${N} -j DNAT --to ${IP6_ADDR}:41245"
90+
do_or_die "iptables -I CJ_INPUT 1 -i tun${N} -j ACCEPT"
91+
do_or_die "ip6tables -I CJ_INPUT 1 -i tun${N} -j ACCEPT"
92+
}
93+
94+
2895
if [ "x$PF_DRIVER" = "xe1000e" ]; then
2996
pf_ringcfg --configure-driver e1000e --rss-queues 1
3097
pf_ringcfg --list-interfaces
@@ -44,36 +111,32 @@ fi
44111
required_timeout=90
45112
nf_conntrack_tcp_timeout_last_ack=$(sysctl --values net.netfilter.nf_conntrack_tcp_timeout_last_ack)
46113
if [ "$nf_conntrack_tcp_timeout_last_ack" -lt "$required_timeout" ];then
47-
sysctl -w net.netfilter.nf_conntrack_tcp_timeout_last_ack=90
114+
sysctl -w net.netfilter.nf_conntrack_tcp_timeout_last_ack=90
115+
fi
116+
117+
# Internal Network Setup
118+
do_or_die "sysctl -w net.ipv4.conf.default.rp_filter=0"
119+
do_or_die "sysctl -w net.ipv4.conf.all.rp_filter=0"
120+
121+
rule_table_name="custom"
122+
rule_table_check=$(ip route show table "$rule_table_name" >/dev/null 2>&1)
123+
if [[ -z "$rule_table_check" ]]; then
124+
echo "adding routing table ${rule_table_name}"
125+
echo "200 ${rule_table_name}" >> /etc/iproute2/rt_tables
48126
fi
49127

128+
build_or_rebuild_iptables nat CJ_PREROUTING PREROUTING
129+
build_or_rebuild_iptables filter CJ_INPUT INPUT
130+
50131
# Create a tunnel for each core.
51132
# The tunnel numbers do not match the core index per the OS,
52133
# but instead match the count of cores being used by conjure.
53134
echo "Setting up devices tun{${OFFSET}..$((OFFSET + CORE_COUNT -1 ))}, adding rules for them, and turning off RP filters."
54-
do_or_die "sysctl -w net.ipv4.conf.all.rp_filter=0"
55-
# Or maybe just set up for all of them instead?
56135
for CORE in `seq $OFFSET $((OFFSET + CORE_COUNT -1 ))`
57136
do
58-
# echo "setting up tun${CORE} ${IP4_ADDR}, ${IP6_ADDR}"
59-
ip tuntap del mode tun tun${CORE}
60-
do_or_die "ip tuntap add mode tun tun${CORE}"
61-
do_or_die "sysctl -w net.ipv4.conf.tun${CORE}.rp_filter=0"
62-
do_or_die "ip rule add iif tun${CORE} lookup custom"
63-
64-
# not sure if we need to do rule below for every tun
65-
# `RTNETLink answers: File exists` means the route is already there; harmless, but can we avoid it?
66-
ip route add local 0.0.0.0/0 dev tun${CORE} table custom
67-
68-
do_or_die "iptables -t nat -I PREROUTING 1 -p tcp -i tun${CORE} -j DNAT --to ${IP4_ADDR}:41245"
69-
do_or_die "iptables -t nat -I PREROUTING 1 -p udp -i tun${CORE} -j DNAT --to ${IP4_ADDR}:41245"
70-
do_or_die "ip6tables -t nat -I PREROUTING 1 -p tcp -i tun${CORE} -j DNAT --to ${IP6_ADDR}:41245"
71-
do_or_die "ip6tables -t nat -I PREROUTING 1 -p udp -i tun${CORE} -j DNAT --to ${IP6_ADDR}:41245"
72-
do_or_die "iptables -I INPUT 1 -i tun${CORE} -j ACCEPT"
73-
do_or_die "ip6tables -I INPUT 1 -i tun${CORE} -j ACCEPT"
137+
tun_setup_fn ${CORE} ${rule_table_name}
74138
done
75139

76-
77140
echo "Setting up hugepages"
78141
if [ ! -d "/mnt/hugepages" ]; then
79142
echo "Creating /mnt/hugepages"

‎sysconfig/conjure-reg.service‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Description=Conjure Registration Server
44
[Service]
55
Type=simple
66

7-
WorkingDirectory=/opt/conjure/cmd/registration-server/
7+
WorkingDirectory=/opt/conjure/
88

99
SyslogIdentifier=conjure
1010
# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf

‎sysconfig/zbalance.service‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ SyslogIdentifier=zbalance
1212
# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
1313

1414
# makes if binary doesn't exist
15-
#ExecStartPre=/usr/bin/make zbalance
15+
ExecStartPre=/opt/conjure/bin/on-reboot.sh
1616

1717
ExecStart=/bin/bash /opt/conjure/scripts/start_zbalance_ipc.sh
1818

0 commit comments

Comments
 (0)