ci: limit CI to only changed architecture

If files are changed for architecture code, then only run CI on those
architectures.

Combine all scripts into one and make it simpler.

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

scripts/ci/get_modified_boards.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2021 Intel Corporation
+
+# A script to generate twister options based on modified files.
+
+import re, os
+import sh
+import argparse
+import glob
+
+if "ZEPHYR_BASE" not in os.environ:
+    exit("$ZEPHYR_BASE environment variable undefined.")
+
+repository_path = os.environ['ZEPHYR_BASE']
+sh_special_args = {
+    '_tty_out': False,
+    '_cwd': repository_path
+}
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+                description="Generate twister argument files based on modified file")
+    parser.add_argument('-c', '--commits', default=None,
+            help="Commit range in the form: a..b")
+    return parser.parse_args()
+
+def find_archs(files):
+    # we match both arch/<arch>/* and include/arch/<arch> and skip common.
+    # Some architectures like riscv require special handling, i.e. riscv
+    # directory covers 2 architectures known to twister: riscv32 and riscv64.
+    archs = set()
+
+    for f in files:
+        p = re.match(r"^arch\/([^/]+)\/", f)
+        if not p:
+            p = re.match(r"^include\/arch\/([^/]+)\/", f)
+        if p:
+            if p.group(1) != 'common':
+                if p.group(1) == 'riscv':
+                    archs.add('riscv32')
+                    archs.add('riscv64')
+                else:
+                    archs.add(p.group(1))
+
+    if archs:
+        with open("modified_archs.args", "w") as fp:
+            fp.write("-a\n%s" %("\n-a\n".join(archs)))
+
+def find_boards(files):
+    boards = set()
+    all_boards = set()
+
+    for f in files:
+        if f.endswith(".rst") or f.endswith(".png") or f.endswith(".jpg"):
+            continue
+        p = re.match(r"^boards\/[^/]+\/([^/]+)\/", f)
+        if p and p.groups():
+            boards.add(p.group(1))
+
+    for b in boards:
+        suboards = glob.glob("boards/*/%s/*.yaml" %(b))
+        for subboard in suboards:
+            name = os.path.splitext(os.path.basename(subboard))[0]
+            if name:
+                all_boards.add(name)
+
+    if all_boards:
+        with open("modified_boards.args", "w") as fp:
+            fp.write("-p\n%s" %("\n-p\n".join(all_boards)))
+
+def find_tests(files):
+    tests = set()
+    for f in files:
+        if f.endswith(".rst"):
+            continue
+        d = os.path.dirname(f)
+        while d:
+            if os.path.exists(os.path.join(d, "testcase.yaml")) or \
+                os.path.exists(os.path.join(d, "sample.yaml")):
+                tests.add(d)
+                break
+            else:
+                d = os.path.dirname(d)
+
+    if tests:
+        with open("modified_tests.args", "w") as fp:
+            fp.write("-T\n%s\n--all" %("\n-T\n".join(tests)))
+
+if __name__ == "__main__":
+
+    args = parse_args()
+    if not args.commits:
+        exit(1)
+
+    # pylint does not like the 'sh' library
+    # pylint: disable=too-many-function-args,unexpected-keyword-arg
+    commit = sh.git("diff", "--name-only", args.commits, **sh_special_args)
+    files = commit.split("\n")
+
+    find_boards(files)
+    find_archs(files)
+    find_tests(files)

scripts/ci/get_modified_tests.py

@@ -22,7 +22,7 @@ set -xe
 
 twister_options=" --inline-logs -N -v --integration"
 export BSIM_OUT_PATH="${BSIM_OUT_PATH:-/opt/bsim/}"
-if [ ! -d "${BSIM_OUT_PATH}" ]; then
+if [ ! -d "${BSIM_OUT_PATH}x" ]; then
         unset BSIM_OUT_PATH
 fi
 export BSIM_COMPONENTS_PATH="${BSIM_OUT_PATH}/components/"
@@ -104,16 +104,21 @@ function run_bsim_bt_tests() {
 
 function get_tests_to_run() {
 	./scripts/zephyr_module.py --twister-out module_tests.args
-	./scripts/ci/get_modified_tests.py --commits ${commit_range} > modified_tests.args
-	./scripts/ci/get_modified_boards.py --commits ${commit_range} > modified_boards.args
+	./scripts/ci/get_twister_opt.py --commits ${commit_range}
 
 	if [ -s modified_boards.args ]; then
-		${twister} ${twister_options} +modified_boards.args --save-tests test_file_1.txt || exit 1
+		${twister} ${twister_options} +modified_boards.args \
+			--save-tests test_file_1.txt || exit 1
 	fi
 	if [ -s modified_tests.args ]; then
-		${twister} ${twister_options} +modified_tests.args --save-tests test_file_2.txt || exit 1
+		${twister} ${twister_options} +modified_tests.args \
+			--save-tests test_file_2.txt || exit 1
 	fi
-	rm -f modified_tests.args modified_boards.args
+	if [ -s modified_archs.args ]; then
+		${twister} ${twister_options} +modified_archs.args \
+			--save-tests test_file_3.txt || exit 1
+	fi
+	rm -f modified_tests.args modified_boards.args modified_archs.args
 }
 
 
@@ -209,6 +214,7 @@ if [ -n "$main_ci" ]; then
 # https://stackoverflow.com/questions/3398258/edit-shell-script-while-its-running
 		git rebase $remote/${branch}
 	else
+		echo "Full Run"
 		SC="full"
 	fi
 	$short_git_log
@@ -226,7 +232,7 @@ if [ -n "$main_ci" ]; then
 
 	# cleanup
 	rm -f test_file.txt
-	touch test_file_1.txt test_file_2.txt
+	touch test_file_1.txt test_file_2.txt test_file_3.txt
 
 	# In a pull-request see if we have changed any tests or board definitions
 	if [ -n "${pull_request_nr}" -o -n "${local_run}"  ]; then
@@ -235,16 +241,18 @@ if [ -n "$main_ci" ]; then
 
 	if [ "$SC" == "full" ]; then
 		# Save list of tests to be run
-		${twister} ${twister_options} --save-tests test_file_3.txt || exit 1
+		${twister} ${twister_options} --save-tests test_file_4.txt || exit 1
 	else
-		echo "test,arch,platform,status,extra_args,handler,handler_time,ram_size,rom_size" > test_file_3.txt
+		echo "test,arch,platform,status,extra_args,handler,handler_time,ram_size,rom_size" \
+			> test_file_4.txt
 	fi
 
 	# Remove headers from all files but the first one to generate one
 	# single file with only one header row
+	tail -n +2 test_file_3.txt > test_file_3_in.txt
 	tail -n +2 test_file_2.txt > test_file_2_in.txt
 	tail -n +2 test_file_1.txt > test_file_1_in.txt
-	cat test_file_3.txt test_file_2_in.txt test_file_1_in.txt > test_file.txt
+	cat test_file_4.txt test_file_3_in.txt test_file_2_in.txt test_file_1_in.txt > test_file.txt
 
 	echo "+++ run twister"
 

scripts/ci/get_twister_opt.py

@@ -23,6 +23,22 @@ Makefile
 tests/*
 samples/*
 boards/*/*/*
+arch/xtensa/*
+arch/x86/*
+arch/posix/*
+arch/arc/*
+arch/sparc/*
+arch/arm/*
+arch/nios2/*
+arch/riscv/*
+include/arch/xtensa/*
+include/arch/x86/*
+include/arch/posix/*
+include/arch/arc/*
+include/arch/sparc/*
+include/arch/arm/*
+include/arch/nios2/*
+include/arch/riscv/*
 doc/*
 # GH action have no impact on code
 .github/*