Skip to content

Commit ae0952d

Browse files
osnrosnr
authored
Merge pull request #217
folk2 (parallel Folk evaluator)
2 parents 23aa0cc + 83479b1 commit ae0952d

File tree

847 files changed

+765188
-31053
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

847 files changed

+765188
-31053
lines changed

‎.gitignore‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
folk
2+
folk.pid
3+
*.o
4+
folk.dSYM
5+
CFLAGS
16
.DS_Store
2-
statements.pdf
3-
statements.dot
4-
trie.pdf
5-
trie.dot
6-
7+
play
8+
.vscode/settings.json
9+
vendor/jimtcl/conftest__.dSYM/Contents/Info.plist
10+
vendor/jimtcl/conftest__.dSYM/Contents/Resources/Relocations/aarch64/conftest__.yml

‎.gitmodules‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
[submodule "live-build/config/includes.chroot_after_packages/etc/skel/apriltag"]
2-
path = live-build/config/includes.chroot_after_packages/home/folk/apriltag
3-
url = https://github.com/FolkComputer/apriltag
1+
[submodule "vendor/tracy"]
2+
path = vendor/tracy
3+
url = https://github.com/wolfpld/tracy.git

‎Makefile‎

Lines changed: 165 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,184 @@
1-
start:
2-
tclsh8.6 main.tcl
3-
debug:
4-
gdb --args tclsh8.6 main.tcl
5-
remote-debug: sync
6-
ssh -tt folk@$(FOLK_SHARE_NODE) -- 'sudo systemctl stop folk && make -C /home/folk/folk debug'
7-
remote-valgrind: sync
8-
ssh -tt folk@$(FOLK_SHARE_NODE) -- 'cd folk; sudo systemctl stop folk && valgrind --leak-check=yes tclsh8.6 main.tcl'
1+
ifeq ($(shell uname -s),Linux)
2+
override BUILTIN_CFLAGS += -Wl,--export-dynamic
3+
endif
94

10-
FOLK_SHARE_NODE := $(shell tclsh8.6 hosts.tcl shareNode)
5+
ifneq (,$(filter -DTRACY_ENABLE,$(CFLAGS)))
6+
# Tracy is enabled
7+
TRACY_TARGET = vendor/tracy/public/TracyClient.o
8+
override CPPFLAGS += -std=c++20 -DTRACY_ENABLE
9+
LINKER := c++
10+
else
11+
LINKER := cc
12+
endif
1113

12-
sync:
13-
rsync --delete --timeout=5 -e "ssh -o StrictHostKeyChecking=no" -a --no-links . folk@$(FOLK_SHARE_NODE):/home/folk/folk
14+
folk: workqueue.o db.o trie.o sysmon.o epoch.o folk.o \
15+
vendor/c11-queues/mpmc_queue.o vendor/c11-queues/memory.o \
16+
vendor/jimtcl/libjim.a $(TRACY_TARGET) CFLAGS
17+
18+
$(LINKER) -g -fno-omit-frame-pointer $(if $(ASAN_ENABLE),-fsanitize=address -fsanitize-recover=address,) -o$@ \
19+
$(CFLAGS) $(BUILTIN_CFLAGS) \
20+
-L./vendor/jimtcl \
21+
$(filter %.o %.a,$^) \
22+
-ljim -lm -lssl -lcrypto -lz
23+
if [ "$$(uname)" = "Darwin" ]; then \
24+
dsymutil $@; \
25+
fi
26+
# Hack for the gadget trigger button.
27+
if [ "$$(uname)" = "Linux" ]; then \
28+
(sudo -n true 2>/dev/null && sudo setcap cap_sys_rawio+ep $@) || true; \
29+
fi
1430

15-
sync-restart: sync
16-
ssh -tt folk@$(FOLK_SHARE_NODE) -- 'sudo systemctl restart folk'
31+
%.o: %.c trie.h workqueue.h CFLAGS
32+
cc -c -O2 -g -fno-omit-frame-pointer $(if $(ASAN_ENABLE),-fsanitize=address -fsanitize-recover=address,) -o$@ \
33+
-D_GNU_SOURCE $(CFLAGS) $(BUILTIN_CFLAGS) \
34+
$< -I./vendor/jimtcl -I./vendor/tracy/public
1735

18-
test:
19-
for testfile in test/*.tcl; do echo; echo $${testfile}; echo --------; make FOLK_ENTRY=$${testfile}; done
36+
.PHONY: test clean deps
37+
test: folk
38+
@count=1; \
39+
total=$$(ls test/*.folk | wc -l | tr -d ' '); \
40+
for test in test/*.folk; do \
41+
echo "Running test: $$test ($${count}/$$total)"; \
42+
echo "--------------------"; \
43+
./folk $$test; \
44+
result=$$?; \
45+
if [ $$result -eq 0 ]; then \
46+
echo "Ran test: $$test ($${count}/$$total): ✅ passed"; \
47+
else \
48+
echo "Ran test: $$test ($${count}/$$total): ❌ failed"; \
49+
fi; \
50+
echo ""; \
51+
count=$$((count + 1)); \
52+
done
53+
test/%: test/%.folk folk
54+
./folk $<
55+
debug-test/%: test/%.folk folk
56+
if [ "$$(uname)" = "Darwin" ]; then \
57+
lldb -o "process handle -p true -s false SIGUSR1" -- ./folk $<; \
58+
else \
59+
gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" --args ./folk $<; \
60+
fi
2061

21-
test/%.debug:
22-
FOLK_ENTRY=test/$*.tcl lldb -- tclsh8.6 main.tcl
62+
debug: folk
63+
if [ "$$(uname)" = "Darwin" ]; then \
64+
lldb -o "process handle -p true -s false SIGUSR1" -- ./folk; \
65+
else \
66+
gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" ./folk; \
67+
fi
2368

24-
test/%:
25-
make FOLK_ENTRY=$@.tcl
69+
clean:
70+
rm -f folk *.o vendor/tracy/public/TracyClient.o vendor/c11-queues/*.o
71+
distclean: clean
72+
make -C vendor/jimtcl distclean
73+
make -C vendor/apriltag clean
2674

27-
repl:
28-
tclsh8.6 replmain.tcl
75+
remote-clean: sync
76+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make clean'
77+
remote-distclean: sync
78+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make distclean'
79+
deps:
80+
if [ ! -f vendor/jimtcl/Makefile ]; then \
81+
cd vendor/jimtcl && ./configure CFLAGS='-g -fno-omit-frame-pointer'; \
82+
fi
83+
make -C vendor/jimtcl
84+
make -C vendor/apriltag libapriltag.so
85+
if [ ! -f vendor/wslay/Makefile ]; then \
86+
cd vendor/wslay && autoreconf -i && automake && autoconf && ./configure; \
87+
fi
88+
make -C vendor/wslay
89+
if [ "$$(uname)" = "Darwin" ]; then \
90+
install_name_tool -id @executable_path/vendor/apriltag/libapriltag.so vendor/apriltag/libapriltag.so; \
91+
install_name_tool -id @executable_path/vendor/wslay/lib/.libs/libwslay.0.dylib vendor/wslay/lib/.libs/libwslay.0.dylib; \
92+
fi
2993

30-
journal:
31-
ssh folk@$(FOLK_SHARE_NODE) -- journalctl -f -n 100 -u folk
94+
kill-folk:
95+
sudo systemctl stop folk
96+
if [ -f folk.pid ]; then \
97+
OLD_PID=`cat folk.pid`; \
98+
sudo pkill -9 --pgroup $$OLD_PID; \
99+
while sudo pkill -0 --pgroup $$OLD_PID; do sleep 0.2; done; \
100+
fi
32101

33-
ssh:
34-
ssh folk@$(FOLK_SHARE_NODE)
102+
FOLK_REMOTE_NODE ?= folk-live
103+
104+
sync:
105+
ssh $(FOLK_REMOTE_NODE) -t \
106+
'cd ~/folk && git init > /dev/null && git ls-files --exclude-standard -oi --directory' \
107+
> .git/ignores.tmp || true
108+
git ls-files --exclude-standard -oi --directory >> .git/ignores.tmp
109+
rsync --timeout=15 -e "ssh -o StrictHostKeyChecking=no" \
110+
--archive --delete --itemize-changes \
111+
--exclude='/.git' \
112+
--exclude-from='.git/ignores.tmp' \
113+
--include='vendor/tracy/public/***' \
114+
--exclude='vendor/tracy/*' \
115+
./ $(FOLK_REMOTE_NODE):~/folk/
116+
117+
remote-setup:
118+
ssh-copy-id $(FOLK_REMOTE_NODE)
119+
make sync
120+
ssh $(FOLK_REMOTE_NODE) -- 'sudo usermod -a -G tty folk && chmod +rwx ~/folk-calibration-poses; sudo apt update && sudo apt install libssl-dev gdb libwslay-dev google-perftools libgoogle-perftools-dev linux-perf && cd folk/vendor/jimtcl && make distclean; ./configure CFLAGS="-g -fno-omit-frame-pointer"'
121+
122+
remote: sync
123+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make start CFLAGS="$(CFLAGS)" ASAN_ENABLE=$(ASAN_ENABLE)'
124+
sudo-remote: sync
125+
ssh -tt $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && sudo HOME=/home/folk TRACY_SAMPLING_HZ=31000 ./folk'
126+
debug-remote: sync
127+
ssh -tt $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" ./folk'
128+
debug-sudo-remote: sync
129+
ssh -tt $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && sudo HOME=/home/folk TRACY_SAMPLING_HZ=31000 gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" ./folk'
130+
valgrind-remote: sync
131+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make && valgrind --leak-check=yes ./folk'
132+
heapprofile-remote: sync
133+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && env LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so HEAPPROFILE=/tmp/folk.hprof PERFTOOLS_VERBOSE=-1 ./folk'
134+
debug-heapprofile-remote: sync
135+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && env LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so HEAPPROFILE=/tmp/folk.hprof PERFTOOLS_VERBOSE=-1 gdb -ex "handle SIGUSR1 nostop" ./folk'
136+
heapprofile-remote-show:
137+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; google-pprof --text folk $(HEAPPROFILE)'
138+
heapprofile-remote-svg:
139+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; google-pprof --svg folk $(HEAPPROFILE)' > out.svg
35140

36-
FLAMEGRAPH_TID := $(shell pgrep tclsh8.6 | head -1)
37141
flamegraph:
38-
sudo perf record -F 997 --tid=$(FLAMEGRAPH_TID) -g -- sleep 30
142+
sudo perf record --freq=997 --call-graph lbr --pid=$(shell cat folk.pid) -g -- sleep 30
39143
sudo perf script -f > out.perf
40144
~/FlameGraph/stackcollapse-perf.pl out.perf > out.folded
41145
~/FlameGraph/flamegraph.pl out.folded > out.svg
42146

43-
# You can use the Web server to check the pid of display.folk,
44-
# apriltags.folk, camera.folk, etc.
45147
remote-flamegraph:
46-
ssh -t folk@$(FOLK_SHARE_NODE) -- make -C /home/folk/folk flamegraph $(if $(REMOTE_FLAMEGRAPH_TID),FLAMEGRAPH_TID=$(REMOTE_FLAMEGRAPH_TID),)
47-
scp folk@$(FOLK_SHARE_NODE):~/folk/out.svg .
48-
scp folk@$(FOLK_SHARE_NODE):~/folk/out.perf .
49-
50-
backup-printed-programs:
51-
cd ~/folk-printed-programs && timestamp=$$(date '+%Y-%m-%d_%H-%M-%S%z') && tar -zcvf ~/"folk-printed-programs_$$timestamp.tar.gz" . && echo "Saved to: ~/folk-printed-programs_$$timestamp.tar.gz"
148+
ssh -t $(FOLK_REMOTE_NODE) -- make -C folk flamegraph
149+
scp $(FOLK_REMOTE_NODE):~/folk/out.svg .
150+
scp $(FOLK_REMOTE_NODE):~/folk/out.perf .
52151

53-
.PHONY: test sync start journal repl enable-pubkey install-deps
152+
start: folk
153+
@if [ -n "$$(systemctl list-unit-files | grep folk.service)" ] && \
154+
[ -n "$$(systemctl cat folk.service | grep "ExecStart.*$$(pwd)")" ] && \
155+
[ -z "$(ENABLE_ASAN)" ] && \
156+
[ -z "$$INVOCATION_ID" ] && \
157+
[ -z "$(CFLAGS)" ] ; then \
158+
sudo systemctl start folk.service; \
159+
journalctl --output=cat -f -u folk.service; \
160+
else \
161+
$(if $(ENABLE_ASAN),ASAN_OPTIONS=detect_leaks=1:halt_on_error=0,) ./folk; \
162+
fi
54163

55-
enable-pubkey:
56-
ssh folk-live -- 'sudo sed -i "s/.*PubkeyAuthentication.*/PubkeyAuthentication yes/g" /etc/ssh/sshd_config && sudo systemctl restart ssh'
57-
sleep 1
58-
ssh-copy-id folk-live
164+
run-tracy:
165+
vendor/tracy/profiler/build/tracy-profiler
166+
tracy-remote:
167+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk'
168+
vendor/tracy/profiler/build/tracy-profiler -a `ssh -G $(FOLK_REMOTE_NODE) | awk '$$1 == "hostname" { print $$2 }'` & \
169+
make remote CFLAGS=-DTRACY_ENABLE FOLK_REMOTE_NODE=$(FOLK_REMOTE_NODE)
170+
sudo-tracy-remote:
171+
ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk'
172+
vendor/tracy/profiler/build/tracy-profiler -a `ssh -G $(FOLK_REMOTE_NODE) | awk '$$1 == "hostname" { print $$2 }'` & \
173+
make sudo-remote CFLAGS=-DTRACY_ENABLE FOLK_REMOTE_NODE=$(FOLK_REMOTE_NODE)
59174

60-
install-deps:
61-
sudo apt install console-data
175+
# From https://stackoverflow.com/a/26147844 to force rebuild if CFLAGS
176+
# changes (in particular, so we rebuild if we want to use Tracy)
177+
define DEPENDABLE_VAR
178+
.PHONY: phony
179+
$1: phony
180+
@if [ "`cat $1 2>&1`" != '$($1)' ]; then \
181+
/bin/echo -n $($1) > $1 ; \
182+
fi
183+
endef
184+
$(eval $(call DEPENDABLE_VAR,CFLAGS))

0 commit comments

Comments
 (0)