Skip to content

Commit 20adc21

Browse files
Ethereal-OPengzna
Ethereal-O
and
Pengzna
authored
fix(vermeer): avoid binary file include in source code (#340)
* introducing auto-download .so and makefile * remove >800k files * fix(ui/Makefile): add ui dirs; change Makefile * fix(ui header): add header --------- Co-authored-by: Peng Junzhi <[email protected]>
1 parent 7e40c8a commit 20adc21

File tree

14 files changed

+619
-654
lines changed

14 files changed

+619
-654
lines changed

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
target/
22
**.db
33
logs/
4-
ui
54
node_modules
65
upload-files/
76
demo*

‎vermeer/.gitignore‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,14 @@ node_modules/
8787
# Others #
8888
######################
8989
test/case/
90+
91+
# Downloaded binaries (should be downloaded via scripts/download_binaries.sh) #
92+
######################
93+
tools/supervisord/*/supervisord*
94+
tools/protoc/*/protoc
95+
tools/protoc/*/bin/
96+
tools/protoc/*/include/
97+
98+
# Generated files (should be generated via go generate) #
99+
######################
100+
asset/assets_vfsdata.go

‎vermeer/Makefile‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
.PHONY: all init download-binaries generate-assets build clean help
19+
20+
# Default target
21+
all: generate-assets build
22+
23+
# Initialize project (first time setup)
24+
init: download-binaries
25+
@echo "Installing Go dependencies..."
26+
go mod download
27+
@echo "Project initialized successfully!"
28+
29+
# Download binary dependencies (supervisord, protoc)
30+
download-binaries:
31+
@echo "Downloading binary dependencies..."
32+
@./scripts/download_binaries.sh || (echo "Failed to download binaries" && exit 1)
33+
34+
# Generate assets (vfsdata.go for web UI)
35+
generate-assets:
36+
@echo "Generating assets..."
37+
@cd asset && go generate || (echo "Failed to generate assets" && exit 1)
38+
@echo "Assets generated successfully!"
39+
40+
# Build vermeer binary
41+
build:
42+
@echo "Building vermeer..."
43+
@go build -o vermeer
44+
@echo "Build completed: ./vermeer"
45+
46+
# Build for specific platform
47+
build-linux-amd64:
48+
@echo "Building for linux/amd64..."
49+
@CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o vermeer
50+
51+
build-linux-arm64:
52+
@echo "Building for linux/arm64..."
53+
@CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o vermeer
54+
55+
# Clean generated files and binaries
56+
clean:
57+
@echo "Cleaning..."
58+
@rm -f vermeer
59+
@rm -f asset/assets_vfsdata.go
60+
@echo "Clean completed!"
61+
62+
# Clean including downloaded binaries
63+
clean-all: clean
64+
@echo "Cleaning downloaded binaries..."
65+
@rm -rf tools/supervisord/*/supervisord
66+
@rm -rf tools/protoc/*/protoc
67+
@rm -rf tools/protoc/*/bin
68+
@rm -rf tools/protoc/*/include
69+
@echo "All clean completed!"
70+
71+
# Help
72+
help:
73+
@echo "Vermeer Build System"
74+
@echo ""
75+
@echo "Usage:"
76+
@echo " make init - First time setup (download binaries + go mod download)"
77+
@echo " make download-binaries - Download supervisord and protoc binaries for your platform"
78+
@echo " make generate-assets - Generate assets_vfsdata.go from web UI (required before build)"
79+
@echo " make build - Build vermeer for current platform (default: local architecture)"
80+
@echo " make build-linux-amd64 - Build for Linux AMD64 (for deployment)"
81+
@echo " make build-linux-arm64 - Build for Linux ARM64 (for ARM servers)"
82+
@echo " make clean - Remove generated files and binaries (keep downloaded tools)"
83+
@echo " make clean-all - Remove everything including downloaded tools"
84+
@echo " make all - Generate assets and build (default target)"
85+
@echo " make help - Show this help message"

‎vermeer/README.md‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -48,42 +48,73 @@ Configuration file reference config/supervisor.conf
4848
./supervisord -c supervisor.conf -d
4949
````
5050

51-
## Compile
52-
Required
53-
* go 1.23
51+
## Build from Source
5452

55-
### Install dependencies
53+
### Requirements
54+
* Go 1.23 or later
55+
* `curl` and `unzip` utilities (for downloading dependencies)
56+
* Internet connection (for first-time setup)
5657

58+
### Quick Start
59+
60+
**Recommended**: Use Makefile for building:
61+
62+
```bash
63+
# First time setup (downloads binary dependencies)
64+
make init
65+
66+
# Build vermeer
67+
make
5768
```
58-
go mod tidy
69+
70+
**Alternative**: Use the build script:
71+
72+
```bash
73+
# For AMD64
74+
./build.sh amd64
75+
76+
# For ARM64
77+
./build.sh arm64
5978
```
6079

61-
### Local compile
80+
# The script will:
81+
# - Auto-detect your OS and architecture if no parameter is provided
82+
# - Download required tools if not present
83+
# - Generate assets and build the binary
84+
# - Exit with error message if any step fails
85+
86+
### Build Targets
6287

88+
```bash
89+
make build # Build for current platform
90+
make build-linux-amd64 # Build for Linux AMD64
91+
make build-linux-arm64 # Build for Linux ARM64
92+
make clean # Clean generated files
93+
make help # Show all available targets
6394
```
64-
go build
95+
96+
### Development Build
97+
98+
For development with hot-reload of web UI:
99+
100+
```bash
101+
go build -tags=dev
65102
```
66103

67104
---
68105

69-
### install grpc protobuf dependencies
70-
````
71-
go install google.golang.org/protobuf/cmd/[email protected] \
72-
go install google.golang.org/grpc/cmd/[email protected]
73-
````
74-
75-
### protobuf build
76-
````
77-
../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
78-
````
106+
### Protobuf Development
79107

108+
If you need to regenerate protobuf files:
80109

81-
### Cross Compile
110+
```bash
111+
# Install protobuf Go plugins
112+
go install google.golang.org/protobuf/cmd/[email protected]
113+
go install google.golang.org/grpc/cmd/[email protected]
82114

83-
````
84-
linux: GOARCH=amd64 GOOS=linux go build
85-
CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin
86-
````
115+
# Generate protobuf files
116+
tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
117+
```
87118

88119
---
89120

‎vermeer/README.zh-CN.md‎

Lines changed: 51 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -48,44 +48,73 @@ worker: ./vermeer --env=worker01
4848
./supervisord -c supervisor.conf -d
4949
````
5050

51-
## 编译
51+
## 从源码编译
5252

53-
* Go 1.23
53+
### 环境要求
54+
* Go 1.23 或更高版本
55+
* `curl` 和 `unzip` 工具(用于下载依赖)
56+
* 互联网连接(首次构建时需要)
5457

55-
### 安装依赖项
58+
### 快速开始
5659

57-
```
58-
go mod tidy
60+
**推荐**: 使用 Makefile 进行构建:
61+
62+
```bash
63+
# 首次设置(下载二进制依赖)
64+
make init
65+
66+
# 构建 vermeer
67+
make
5968
```
6069

61-
### 本地编译
70+
**替代方案**: 使用构建脚本:
6271

72+
```bash
73+
# AMD64 架构
74+
./build.sh amd64
75+
76+
# ARM64 架构
77+
./build.sh arm64
6378
```
64-
go build
79+
80+
构建过程会自动:
81+
1. 自动检测操作系统和架构（如果未提供参数）
82+
2. 下载所需的二进制工具(supervisord, protoc)
83+
3. 生成 Web UI 资源文件
84+
4. 构建 vermeer 二进制文件
85+
86+
### 构建目标
87+
88+
```bash
89+
make build # 为当前平台构建
90+
make build-linux-amd64 # 为 Linux AMD64 构建
91+
make build-linux-arm64 # 为 Linux ARM64 构建
92+
make clean # 清理生成的文件
93+
make help # 显示所有可用目标
6594
```
6695

67-
### grpc protobuf 依赖项安装
68-
````
69-
go install google.golang.org/protobuf/cmd/[email protected] \
70-
go install google.golang.org/grpc/cmd/[email protected]
71-
````
96+
### 开发构建
7297

98+
如需在开发环境中热重载 Web UI:
7399

100+
```bash
101+
go build -tags=dev
102+
```
74103

75-
### protobuf build
76-
生成protobuf文件
77-
````
78-
../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
79-
````
104+
---
80105

106+
### Protobuf 开发
81107

108+
如需重新生成 protobuf 文件:
82109

83-
### 交叉编译
110+
```bash
111+
# 安装 protobuf Go 插件
112+
go install google.golang.org/protobuf/cmd/[email protected]
113+
go install google.golang.org/grpc/cmd/[email protected]
84114

85-
````
86-
linux: GOARCH=amd64 GOOS=linux go build
87-
CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin
88-
````
115+
# 生成 protobuf 文件
116+
tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
117+
```
89118

90119
---
91120

‎vermeer/asset/assets_vfsdata.go‎

Lines changed: 0 additions & 609 deletions
This file was deleted.

‎vermeer/build.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ go env -w GONOSUMDB=\* ## 目前有一些代码库还
2828
#go env -w CXX=/opt/compiler/gcc-8.2/bin/g++
2929

3030
go mod download
31+
32+
# Download binary dependencies if not exist
33+
echo "Checking binary dependencies..."
34+
./scripts/download_binaries.sh
35+
36+
# Generate assets if not exist
37+
if [ ! -f "asset/assets_vfsdata.go" ]; then
38+
echo "Generating assets..."
39+
cd asset && go generate && cd ..
40+
fi
3141
ARCH=$1
3242
CGO_ENABLED=0 GOOS=linux GOARCH="$ARCH" go build
3343

0 commit comments

Comments
 (0)