* add p2p multi-processes mode

* reduce compilation time
* reduce the number of threads in process mode
* improvements of building

Author: vyloy

.github/workflows/container.yml

@@ -40,14 +40,16 @@ jobs:
 
     - name: package-server
       run: |-
-        sudo apt-get update && sudo apt-get install make git gn ninja-build python3 python3-pip libgtk-3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu -y
+        sudo apt-get update && sudo apt-get install make git gn ninja-build python3 python3-pip libgtk-3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu rustc -y
+        rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu
         GOARCH=amd64 make release_server && mv release/linux-amd64-server release/linux-x86_64-server
         TARGET=aarch64-linux-gnu GOOS=linux GOARCH=arm64 make release_server && mv release/linux-arm64-server release/linux-aarch64-server
         ls -al release
 
     - name: package-client
       run: |-
-        sudo apt-get update && sudo apt-get install make git gn ninja-build python3 python3-pip libgtk-3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu -y
+        sudo apt-get update && sudo apt-get install make git gn ninja-build python3 python3-pip libgtk-3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu rustc -y
+        rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu
         GOARCH=amd64 make release_client && mv release/linux-amd64-client release/linux-x86_64-client
         TARGET=aarch64-linux-gnu GOOS=linux GOARCH=arm64 make release_client && mv release/linux-arm64-client release/linux-aarch64-client
         ls -al release

.gitmodules

@@ -17,6 +17,8 @@
 	url = https://git.mirror.iscas.ac.cn/ao-space/google-webrtc.git
 	branch = main
 	shallow = true
+	ignore = dirty
 [submodule "dep/_msquic"]
 	path = dep/_msquic
 	url = https://github.com/microsoft/msquic
+	ignore = dirty

Dockerfile

@@ -14,20 +14,26 @@
 
 FROM golang:1.20-bookworm
 
+# 安装 gnu 编译链
+# 安装 msquic 依赖
+# 安装 nodejs 20
 RUN apt update && \
-    apt install xz-utils bzip2 sudo lsb-release ninja-build generate-ninja file patch -y
+    apt install -y xz-utils bzip2 sudo lsb-release ninja-build generate-ninja file patch \
+        gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \
+        cmake build-essential liblttng-ust-dev lttng-tools libssl-dev \
+        ca-certificates curl gnupg
 
-# 安装nodejs 20
-RUN apt-get install -y ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
+RUN mkdir -p /etc/apt/keyrings && \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
     NODE_MAJOR=20 && \
     echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install nodejs -y
+    apt update && \
+    apt install nodejs -y
 
-# 安装msquic依赖
-RUN apt-get install -y cmake build-essential liblttng-ust-dev lttng-tools libssl-dev
+# 安装 rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    . "$HOME/.cargo/env" && \
+    rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu
 
 # golang 切换国内源并且提前安装好依赖
 ENV GO111MODULE=on
@@ -37,9 +43,6 @@ RUN cd /go/src/github.com/isrc-cas/gt && \
     go mod download && \
     rm -rf /go/src/github.com/isrc-cas/gt
 
-# 安装 gnu 编译链
-RUN apt install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu
-
 # 安装 google webrtc 依赖
 ADD ./dep/_google-webrtc/src/build /root/build
 RUN	sed -i 's/egrep -q "i686|x86_64"/true/g' /root/build/install-build-deps.sh && \

Makefile

@@ -40,6 +40,9 @@ TARGET_OS=$(shell echo $(TARGET) | awk -F '-' '{print $$2}')
 ifeq ($(TARGET_OS), native)
 	TARGET_OS=
 endif
+ifeq ($(TARGET_OS), linux)
+	RUST_TARGET?=$(shell echo $(TARGET) | awk -F '-' '{print $$1 "-unknown-" $$2 "-" $$3}')
+endif
 TARGET_CPU=$(shell echo $(TARGET) | awk -F '-' '{print $$1}')
 ifeq ($(TARGET_CPU), native)
 	TARGET_CPU=
@@ -136,8 +139,12 @@ set_safe_directories:
 docker_create_image: update_submodule
 	docker images | grep -cim1 -E "^gtbuild\s+?v1" || docker build -t gtbuild:v1 .
 
-docker_build_linux_amd64: docker_build_linux_amd64_server docker_build_linux_amd64_client
-docker_release_linux_amd64: docker_release_linux_amd64_server docker_release_linux_amd64_client
+docker_build_linux_amd64: docker_create_image
+	$(eval MAKE_ENV=TARGET=x86_64-linux-gnu GOOS=linux GOARCH=amd64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
+	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make build'
+docker_release_linux_amd64: docker_create_image
+	$(eval MAKE_ENV=TARGET=x86_64-linux-gnu GOOS=linux GOARCH=amd64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
+	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make release'
 docker_build_linux_amd64_client: docker_create_image
 	$(eval MAKE_ENV=TARGET=x86_64-linux-gnu GOOS=linux GOARCH=amd64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
 	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make build_client'
@@ -151,8 +158,12 @@ docker_release_linux_amd64_server: docker_create_image
 	$(eval MAKE_ENV=TARGET=x86_64-linux-gnu GOOS=linux GOARCH=amd64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
 	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make release_server'
 
-docker_build_linux_arm64: docker_build_linux_arm64_server docker_build_linux_arm64_client
-docker_release_linux_arm64: docker_release_linux_arm64_server docker_release_linux_arm64_client
+docker_build_linux_arm64: docker_create_image
+	$(eval MAKE_ENV=TARGET=aarch64-linux-gnu GOOS=linux GOARCH=arm64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
+	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make build'
+docker_release_linux_arm64: docker_create_image
+	$(eval MAKE_ENV=TARGET=aarch64-linux-gnu GOOS=linux GOARCH=arm64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
+	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make release'
 docker_build_linux_arm64_client: docker_create_image
 	$(eval MAKE_ENV=TARGET=aarch64-linux-gnu GOOS=linux GOARCH=arm64 STATIC_LINK=$(STATIC_LINK) RACE_CHECK=$(RACE_CHECK) WITH_OFFICIAL_WEBRTC=$(WITH_OFFICIAL_WEBRTC))
 	docker run --rm -v $(PWD):/go/src/github.com/isrc-cas/gt -w /go/src/github.com/isrc-cas/gt gtbuild:v1 sh -c '$(MAKE_ENV) make build_client'
@@ -168,12 +179,14 @@ docker_release_linux_arm64_server: docker_create_image
 
 build: build_server build_client
 release: release_server release_client
-build_client: $(SOURCES) Makefile compile_msquic compile_webrtc
+build_client: $(SOURCES) Makefile compile_msquic compile_webrtc compile_p2p
 	$(eval CGO_CXXFLAGS+=-O0 -g -ggdb)
+	$(eval CGO_LDFLAGS+=$(PWD)/dep/p2p/target/$(RUST_TARGET)/release/libp2p.a)
 	$(eval NAME=$(GOOS)-$(GOARCH)-client)
 	go build $(DEBUG_OPTIONS) -o build/$(NAME)$(EXE) ./cmd/client
-release_client: $(SOURCES) Makefile compile_msquic compile_webrtc release_web_client
+release_client: $(SOURCES) Makefile compile_msquic compile_webrtc compile_p2p release_web_client
 	$(eval CGO_CXXFLAGS+=-O3)
+	$(eval CGO_LDFLAGS+=$(PWD)/dep/p2p/target/$(RUST_TARGET)/release/libp2p.a)
 	$(eval NAME=$(GOOS)-$(GOARCH)-client)
 	go build $(RELEASE_OPTIONS) -o release/$(NAME)$(EXE) ./cmd/client
 build_server: $(SOURCES) Makefile compile_msquic compile_webrtc
@@ -203,7 +216,7 @@ duplicate_dist_server:
 duplicate_dist_client:
 	cp -r $(FRONTEND_DIR)/dist $(CLIENT_FRONT_DIR)/dist
 
-clean: clean_web
+clean: clean_web clean_p2p
 	rm -rf build/* release/*
 	rm -rf dep/_google-webrtc/src/out/*
 
@@ -277,4 +290,14 @@ compile_msquic: check_msquic_dependencies update_submodule
 	cmake -B./dep/_msquic/$(TARGET) -S./dep/_msquic -DQUIC_BUILD_SHARED=OFF -DCMAKE_TARGET_ARCHITECTURE=$(TARGET_CPU)
 	make -C./dep/_msquic/$(TARGET) -j$(shell nproc)
 	@renameSymbols=$$(objdump -t ./dep/_msquic/$(TARGET)/bin/Release/libmsquic.a | awk -v RS= '/_YB80VJ/{next}1' | grep -E 'g +(F|O) ' | grep -Evi ' (ms){0,1}quic' | awk '{print " --redefine-sym " $$NF "=" $$NF "_YB80VJ"}') && \
-    		$(TARGET)-objcopy $$renameSymbols ./dep/_msquic/$(TARGET)/bin/Release/libmsquic.a
+    		$(TARGET)-objcopy $$renameSymbols ./dep/_msquic/$(TARGET)/bin/Release/libmsquic.a
+
+compile_p2p:
+	cd ./dep/p2p && \
+    . "$$HOME/.cargo/env" && \
+	cargo build -r --target $(RUST_TARGET)
+
+clean_p2p:
+	cd ./dep/p2p && \
+    . "$$HOME/.cargo/env" && \
+	cargo clean

client/client.go

@@ -89,7 +89,7 @@ func New(args []string, out io.Writer) (c *Client, err error) {
 	c = &Client{
 		Logger:  l,
 		tunnels: make(map[*conn]struct{}),
-		peers:   make(map[uint32]*peerTask),
+		peers:   make(map[uint32]PeerTask),
 	}
 	c.config.Store(&conf)
 	c.tunnelsCond = sync.NewCond(c.tunnelsRWMtx.RLocker())

client/config.go

@@ -66,10 +66,11 @@ type Options struct {
 	SentryDebug       bool                 `yaml:"sentryDebug,omitempty" json:",omitempty" usage:"Sentry debug mode, the debug information is printed to help you understand what sentry is doing"`
 
 	WebRTCConnectionIdleTimeout config.Duration `yaml:"webrtcConnectionIdleTimeout,omitempty" usage:"The timeout of WebRTC connection. Supports values like '30s', '5m'"`
-	WebRTCRemoteConnections     uint          `yaml:"webrtcConnections" usage:"The max number of webrtc connections. Valid value is 1 to 50"`
+	WebRTCRemoteConnections     uint            `yaml:"webrtcConnections" usage:"The max number of webrtc connections. Valid value is 1 to 50"`
 	WebRTCLogLevel              string          `yaml:"webrtcLogLevel,omitempty" json:",omitempty" usage:"WebRTC log level: verbose, info, warning, error"`
 	WebRTCMinPort               uint16          `yaml:"webrtcMinPort,omitempty" json:",omitempty" usage:"The min port of WebRTC peer connection"`
 	WebRTCMaxPort               uint16          `yaml:"webrtcMaxPort,omitempty" json:",omitempty" usage:"The max port of WebRTC peer connection"`
+	WebRTCThread                bool            `yaml:"webrtcThreadMode,omitempty" json:",omitempty" usage:"Use thread mode of WebRTC peer connection"`
 
 	TCPForwardAddr        string `yaml:"tcpForwardAddr,omitempty" json:",omitempty" usage:"The address of TCP forward"`
 	TCPForwardHostPrefix  string `yaml:"tcpForwardHostPrefix,omitempty" json:",omitempty" usage:"The host prefix of TCP forward"`
@@ -168,6 +169,8 @@ type service struct {
 	LocalURL           clientURL       `yaml:"local,omitempty" json:",omitempty"`
 	LocalTimeout       config.Duration `yaml:"localTimeout,omitempty" json:",omitempty"`
 	UseLocalAsHTTPHost bool            `yaml:"useLocalAsHTTPHost,omitempty" json:",omitempty"`
+
+	remoteTCPPort uint32
 }
 
 func (s *service) String() string {

client/conn.go

@@ -461,9 +461,10 @@ func (c *conn) processData(taskID uint32, r *bufio.LimitedReader) (readErr, writ
 				})
 				return
 			}
-			_, err := r.WriteTo(pt.apiConn.PipeWriter)
+			_, err := r.WriteTo(pt.APIWriter())
 			if err != nil {
-				pt.Logger.Error().Err(err).Msg("processP2P WriteTo failed")
+				c.Logger.Info().
+					Uint32("peerTask", taskID).Msg("got closed because task with same id is received")
 			}
 			return
 		}
@@ -494,17 +495,65 @@ func (c *conn) processData(taskID uint32, r *bufio.LimitedReader) (readErr, writ
 }
 
 func (c *conn) processP2P(id uint32, r *bufio.LimitedReader) {
-	t, ok := c.newPeerTask(id)
+	var t PeerTask
+	var ok bool
+	if c.client.Config().WebRTCThread {
+		t, ok = c.newPeerTask(id)
+	} else {
+		t, ok = c.newPeerProcTask(id)
+	}
 	if !ok {
 		return
 	}
 
-	c.client.apiServer.Listener.AcceptCh() <- t.apiConn
-	t.Logger.Info().Msg("peer task started")
-	_, err := r.WriteTo(t.apiConn.PipeWriter)
+	c.client.apiServer.Listener.AcceptCh() <- t.APIConn()
+	c.Logger.Info().
+		Uint32("peerTask", id).Msg("peer task started")
+	_, err := r.WriteTo(t.APIWriter())
 	if err != nil {
-		t.Logger.Error().Err(err).Msg("processP2P WriteTo failed")
+		c.Logger.Error().
+			Uint32("peerTask", id).Err(err).Msg("processP2P WriteTo failed")
+	}
+}
+
+func (c *conn) newPeerProcTask(id uint32) (t *peerProcessTask, ok bool) {
+	c.client.peersRWMtx.Lock()
+	defer c.client.peersRWMtx.Unlock()
+	if !predef.Debug {
+		l := uint(len(c.client.peers))
+		if l >= c.client.Config().WebRTCRemoteConnections {
+			respAndClose(id, c, [][]byte{
+				[]byte("HTTP/1.1 403 Forbidden\r\nConnection: Closed\r\n\r\n"),
+			})
+			return
+		}
+	}
+
+	t = &peerProcessTask{}
+	t.id = id
+	t.tunnel = c
+	t.apiConn = api.NewConn(id, "", c)
+	t.apiConn.ProcessOffer = t.processOffer
+	t.apiConn.GetOffer = t.getOffer
+	t.apiConn.ProcessAnswer = t.processAnswer
+	t.data = pool.BytesPool.Get().([]byte)
+	t.Logger = c.Logger.With().
+		Uint32("peerTask", id).
+		Logger()
+	t.timer = time.AfterFunc(120*time.Second, func() {
+		t.Logger.Info().Msg("peer task timeout")
+		t.CloseWithLock()
+	})
+
+	ot, ok := c.client.peers[id]
+	if ok && ot != nil {
+		ot.CloseWithLock()
+		c.Logger.Info().
+			Uint32("peerTask", id).Msg("got closed because task with same id is received")
 	}
+	c.client.peers[id] = t
+	ok = true
+	return
 }
 
 func (c *conn) newPeerTask(id uint32) (t *peerTask, ok bool) {
@@ -542,7 +591,8 @@ func (c *conn) newPeerTask(id uint32) (t *peerTask, ok bool) {
 	ot, ok := c.client.peers[id]
 	if ok && ot != nil {
 		ot.CloseWithLock()
-		ot.Logger.Info().Msg("got closed because task with same id is received")
+		c.Logger.Info().
+			Uint32("peerTask", id).Msg("got closed because task with same id is received")
 	}
 	c.client.peers[id] = t
 	ok = true

client/debug.go

@@ -34,7 +34,7 @@ type Client struct {
 	closing             uint32
 	tunnels             map[*conn]struct{}
 	tunnelsRWMtx        sync.RWMutex
-	peers               map[uint32]*peerTask
+	peers               map[uint32]PeerTask
 	peersRWMtx          sync.RWMutex
 	tunnelsCond         *sync.Cond
 	idleManager         *idleManager

client/peer.go

@@ -55,6 +55,14 @@ type peerTask struct {
 	waitNegotiationNeeded chan struct{}
 }
 
+func (pt *peerTask) APIConn() *api.Conn {
+	return pt.apiConn
+}
+
+func (pt *peerTask) APIWriter() *std.PipeWriter {
+	return pt.apiConn.PipeWriter
+}
+
 func (pt *peerTask) OnSignalingChange(state webrtc.SignalingState) {
 	pt.Logger.Info().Str("state", state.String()).Msg("signaling state changed")
 }
@@ -492,7 +500,7 @@ func (pt *peerTask) processAnswer(r *http.Request, writer http.ResponseWriter) {
 			err = errors.New("invalid task id")
 			return
 		}
-		task = pt
+		task = pt.(*peerTask)
 	}
 	task.process(r.Body, writer, func() (err error) {
 		var answer webrtc.SessionDescription