actions: twister: determine nodes in python script

Improve calculation of matrix and move calculations from workflow to the
testplan script. We now generate a file that can be parsed by the action
with the data needed to start twister with the right number of nodes.

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

.github/workflows/twister.yaml

@@ -76,39 +76,26 @@ jobs:
             echo "Your branch is not up to date, you need to rebase on top of latest HEAD of main branch"
             exit 1
           )
-          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request
-
-          # get number of tests
-          if [ -s testplan.csv ]; then
-            lines=$(wc -l < testplan.csv)
-          else
-            lines=1
-          fi
-          if [ "$lines" = 1 ]; then
-            # no tests, so we need 0 nodes
-            nodes=0
+          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
+          if [ -s .testplan ]; then
+            cat .testplan >> $GITHUB_ENV
           else
-            nodes=$(( ${lines} / ${TESTS_PER_BUILDER}))
-            if [ ${nodes} -gt 4 -a ${nodes} -lt 10 ]; then
-              nodes=${MATRIX_SIZE}
-            elif [ "${nodes}" = 0 ]; then
-              # for less than TESTS_PER_BUILDER, we take at least 1 node
-              nodes=1
-            fi
+            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
           fi
-          echo "::set-output name=calculated_matrix_size::${nodes}";
-          rm -f testplan.csv
+
+
+          rm -f testplan.csv .testplan
 
       - name: Determine matrix size
         id: output-services
         run: |
           if [ "${{github.event_name}}" = "pull_request_target" ]; then
-            if [ -n "${{steps.test-plan.outputs.calculated_matrix_size}}" ]; then
-              subset="[$(seq -s',' 1 ${{steps.test-plan.outputs.calculated_matrix_size}})]"
+            if [ -n "${TWISTER_NODES}" ]; then
+              subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
             else
               subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
             fi
-            size=${{ steps.test-plan.outputs.calculated_matrix_size }}
+            size=${TWISTER_NODES}
           elif [ "${{github.event_name}}" = "push" ]; then
             subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
             size=${MATRIX_SIZE}

scripts/ci/test_plan.py

@@ -284,6 +284,10 @@ def parse_args():
             help="This is a pull request")
     parser.add_argument('-p', '--platform', action="append",
             help="Limit this for a platform or a list of platforms.")
+    parser.add_argument('-t', '--tests_per_builder', default=700, type=int,
+            help="Number of tests per builder")
+    parser.add_argument('-n', '--default-matrix', default=10, type=int,
+            help="Number of tests per builder")
 
     return parser.parse_args()
 
@@ -317,6 +321,18 @@ def parse_args():
             dup_free_set.add(tuple(x))
 
     logging.info(f'Total tests to be run: {len(dup_free)}')
+    with open(".testplan", "w") as tp:
+        total_tests = len(dup_free)
+        nodes = round(total_tests / args.tests_per_builder)
+        if total_tests % args.tests_per_builder != total_tests:
+            nodes = nodes + 1
+
+        if nodes > 5:
+            nodes = args.default_matrix
+
+        tp.write(f"TWISTER_TESTS={total_tests}\n")
+        tp.write(f"TWISTER_NODES={nodes}\n")
+
     header = ['test', 'arch', 'platform', 'status', 'extra_args', 'handler',
             'handler_time', 'ram_size', 'rom_size']