Skip to content

Commit 73ffcf7

Browse files
TempoTianTempoTian
committed
Merge branch 'feature/add_build_flow' into 'main'
Fix CI build issues See merge request adf/esp-webrtc-solution!18
2 parents b0a58fe + cc2ba7c commit 73ffcf7

File tree

7 files changed

+141
-99
lines changed

7 files changed

+141
-99
lines changed

‎.gitlab-ci.yml

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,28 @@ variables:
1111
IDF_SKIP_CHECK_SUBMODULES: 1
1212
GIT_DEPTH: 1
1313

14-
before_script:
15-
- source ${CI_PROJECT_DIR}/tools/ci/utils.sh
16-
- add_gitlab_ssh_keys
14+
.common_before_scripts: &common-before_scripts |
15+
source tools/ci/utils.sh
16+
17+
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
18+
# must use after setup_tools_except_target_test
19+
# otherwise the export.sh won't work properly
20+
21+
# Install esp-clang if necessary
22+
if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
23+
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
24+
fi
25+
26+
source $IDF_PATH/export.sh
27+
28+
.before_script:build:
29+
before_script:
30+
- *common-before_scripts
31+
- *setup_tools_and_idf_python_venv
1732

1833
include:
1934
- '.gitlab/ci/rules.yml'
2035
- '.gitlab/ci/common.yml'
2136
# - '.gitlab/ci/pre_check.yml'
22-
# - '.gitlab/ci/build_webrtc_solutions.yml'
37+
- '.gitlab/ci/build_webrtc_solutions.yml'
2338
- '.gitlab/ci/deploy.yml'

‎.gitlab/ci/build_webrtc_solutions.yml

Lines changed: 49 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,57 @@
1-
.build_template:
1+
.build_templates: &build_templates
2+
stage: build
3+
tags:
4+
- build
5+
image: espressif/idf:latest
6+
variables:
7+
# Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
8+
IDF_CCACHE_ENABLE: "1"
9+
BATCH_BUILD: "1"
10+
V: "0"
11+
WARNING_STR: ""
12+
13+
.build_solutions:
14+
extends:
15+
- .build_templates
216
artifacts:
317
expire_in: 4 days
418
paths:
5-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/size.json
6-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/build_log.txt
7-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/*.bin
8-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/*.elf
9-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/flasher_args.json
10-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/flash_project_args
11-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/config/sdkconfig.json
12-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/sdkconfig
13-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/bootloader/*.bin
14-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/partition_table/*.bin
15-
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/*.py
16-
before_script:
17-
- export OPENAI_API_KEY=FAKE_KEY_FOR_BUILD_ONLY
19+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/size.json
20+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/build_log.txt
21+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/*.bin
22+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/*.elf
23+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/flasher_args.json
24+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/flash_project_args
25+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/config/sdkconfig.json
26+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/sdkconfig
27+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/bootloader/*.bin
28+
- ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER/build*/partition_table/*.bin
1829
script:
19-
- run_cmd python ${BASE_FRAMEWORK_PATH}/tools/ci/ci_build_apps.py ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER
20-
-vv -t $IDF_TARGET --pytest-apps
21-
build_doorbell_demo_esp32p4:
22-
extends: .build_template
23-
stage: build
24-
variables:
25-
CI_BUILD_FOLDER: doorbell_demo
26-
IDF_TARGET: esp32p4
27-
build_doorbell_demo_esp32s3:
28-
extends: .build_template
29-
stage: build
30-
variables:
31-
CI_BUILD_FOLDER: doorbell_demo
32-
IDF_TARGET: esp32s3
33-
build_openai_demo_esp32s3:
34-
extends: .build_template
35-
stage: build
36-
variables:
37-
CI_BUILD_FOLDER: openai_demo
38-
IDF_TARGET: esp32s3
39-
build_peer_demo_esp32:
40-
extends: .build_template
41-
stage: build
42-
variables:
43-
CI_BUILD_FOLDER: peer_demo
44-
IDF_TARGET: esp32
45-
build_peer_demo_esp32s2:
46-
extends: .build_template
47-
stage: build
48-
variables:
49-
CI_BUILD_FOLDER: peer_demo
50-
IDF_TARGET: esp32s2
51-
build_peer_demo_esp32s3:
52-
extends: .build_template
53-
stage: build
54-
variables:
55-
CI_BUILD_FOLDER: peer_demo
56-
IDF_TARGET: esp32s3
30+
- export OPENAI_API_KEY=FAKE_KEY_FOR_BUILD_ONLY
31+
- cd $IDF_PATH
32+
- perl ${CI_PROJECT_DIR}/tools/build_apps.pl ${CI_PROJECT_DIR}/solutions/$CI_BUILD_FOLDER $IDF_TARGET
33+
34+
build_matrix:
35+
extends: .build_solutions
36+
parallel:
37+
matrix:
38+
- CI_BUILD_FOLDER: ["doorbell_demo"]
39+
IDF_TARGET: ["esp32p4", "esp32s3"]
40+
- CI_BUILD_FOLDER: ["openai_demo"]
41+
IDF_TARGET: ["esp32s3"]
42+
- CI_BUILD_FOLDER: ["peer_demo"]
43+
IDF_TARGET: ["esp32", "esp32s2", "esp32s3"]
44+
- CI_BUILD_FOLDER: ["videocall_demo"]
45+
IDF_TARGET: ["esp32p4"]
46+
- CI_BUILD_FOLDER: ["whip_demo"]
47+
IDF_TARGET: ["esp32p4"]
48+
5749
stages:
58-
- build
50+
- build
51+
5952
variables:
60-
BASE_FRAMEWORK: $IDF_REPOSITORY
61-
BASE_FRAMEWORK_PATH: $CI_PROJECT_DIR/esp-idf
6253
DOCKER_IMAGE: ${CI_DOCKER_REGISTRY}/esp-env-v5.4:1
63-
IDF_TAG_FLAG: false
54+
BASE_FRAMEWORK_PATH: "$CI_PROJECT_DIR/esp-idf"
55+
BASE_FRAMEWORK: "ssh://[email protected]:27227/espressif/esp-idf.git"
6456
IDF_VERSION_TAG: v5.4
57+
IDF_TAG_FLAG: false

‎.gitlab/ci/common.yml

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,24 @@
11
variables:
2-
DOCKER_IMAGE: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1
2+
DOCKER_IMAGE: ${CI_DOCKER_REGISTRY}/esp-env-v5.4:1
33
BASE_FRAMEWORK_PATH: "$CI_PROJECT_DIR/esp-idf"
4-
BASE_FRAMEWORK: $IDF_REPOSITORY
5-
IDF_VERSION_TAG: v5.3
4+
BASE_FRAMEWORK: "ssh://[email protected]:27227/espressif/esp-idf.git"
5+
IDF_VERSION_TAG: v5.4
66
IDF_TAG_FLAG: false
77

88
.update_source: &update_source
99
- source ${CI_PROJECT_DIR}/tools/ci/utils.sh
1010
- add_gitlab_ssh_keys
11-
- git submodule update --init --recursive
12-
- git clone $BASE_FRAMEWORK --depth 1
13-
- cd $BASE_FRAMEWORK_PATH
14-
- git submodule update --init
15-
- git log -2
11+
- echo ${IDF_PATH}
12+
- cd ${IDF_PATH}
13+
- idf.py --version
1614
- fetch_idf_branch ${IDF_VERSION_TAG}
1715
- common_before_scripts
1816
- setup_tools_and_idf_python_venv
1917
- set_env_variable
2018

2119
.build_template:
2220
stage: build
23-
image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1
21+
image: ${CI_DOCKER_REGISTRY}/esp-env-v5.4:1
2422
tags:
2523
- multimedia_build
2624
before_script:

‎.gitlab/ci/pre_check.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
.pre_check_template:
22
stage: pre_check
3-
image: "${CI_DOCKER_REGISTRY}/esp-env-v5.3:1"
3+
image: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:1"
44
tags:
55
- host_test
66
dependencies: []

‎tools/build_all.pl

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎tools/build_apps.pl

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
use strict;
2+
use Cwd qw(cwd);
3+
use Term::ANSIColor;
4+
5+
my %target = (
6+
"peer_demo" => [qw/esp32 esp32s3 esp32s2/],
7+
"doorbell_demo" => [qw/esp32p4 esp32s3/],
8+
"openai_demo" => [qw/esp32s3/],
9+
"videocall_demo" => [qw/esp32p4/],
10+
"whip_demo" => [qw/esp32p4/]
11+
);
12+
13+
my $cur = cwd();
14+
my ($app_folder, $target);
15+
my $build_all = 0;
16+
if (@ARGV >= 2) {
17+
$app_folder = $ARGV[0];
18+
$target = $ARGV[1];
19+
build_target($app_folder, $target);
20+
} else {
21+
$build_all = 1;
22+
for my $t(keys %target) {
23+
my @board = @{$target{$t}};
24+
print "Start build for $t @board\n";
25+
for (@board) {
26+
build_target("$cur/$t", $_);
27+
}
28+
}
29+
}
30+
31+
sub filter_build {
32+
my $target = shift;
33+
my $build_cmd = "idf.py set-target $target 2>&1; idf.py build 2>&1";
34+
open(my $fh, '-|', $build_cmd) or die "Failed to run build command: $!";
35+
my @error_patterns = (
36+
qr/\b(error|warning)\b/i,
37+
qr/failed to build/i,
38+
qr/link error/i
39+
);
40+
while (my $line = <$fh>) {
41+
chomp $line;
42+
if (grep { $line =~ $_ } @error_patterns) {
43+
print colored($line, 'red'), "\n";
44+
}
45+
}
46+
close($fh);
47+
}
48+
49+
sub build_target {
50+
my ($app_dir, $target) = @_;
51+
chdir($app_dir);
52+
my $app = $app_dir;
53+
print "Build for $app_dir $target\n";
54+
$app =~ s/.*\///;
55+
`rm -rf $_` for (qw/dependencies.lock build managed_components sdkconfig/);
56+
filter_build($target);
57+
my $f = <build/*.elf>;
58+
if ($f) {
59+
print colored("Build for $app target $target success\n", 'green');
60+
} else {
61+
print colored("Fail to build for $app target $target\n", 'red');
62+
exit(-1) if ($build_all == 0);
63+
}
64+
}

‎tools/ci/utils.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ function common_before_scripts() {
6363
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
6464
}
6565

66-
function setup_tools_and_idf_python_venv() {
66+
function setup_tools_and_idf_python_venv0() {
6767
# must use after setup_tools_except_target_test
6868
# otherwise the export.sh won't work properly
6969
pushd ${IDF_PATH} 2>/dev/null

0 commit comments

Comments
 (0)