[CI] Update CI scripts (#13410)

Update CI scripts. Some refactors to place all the code related
to filtering out packages in the same function. Avoid creating
kind clusters in CI build testing with Serverless. Remove some
output in the trigger step.

Author: mrodm

.buildkite/scripts/common.sh

@@ -570,9 +570,9 @@ prepare_serverless_stack() {
 }
 
 is_spec_3_0_0() {
-    local pkg_spec
+    local pkg_spec=""
     pkg_spec=$(cat manifest.yml | yq '.format_version')
-    local major_version
+    local major_version=""
     major_version=$(echo "$pkg_spec" | cut -d '.' -f 1)
 
     if [ "${major_version}" -ge 3 ]; then
@@ -670,10 +670,8 @@ get_to_changeset() {
 
 is_pr_affected() {
     local package="${1}"
-    local from=${2:-""}
-    local to=${3:-""}
-
-    echoerr "[${package}] Original commits: from '${from}' - to: '${to}'"
+    local from="${2}"
+    local to="${3}"
 
     if ! is_supported_stack ; then
         echo "[${package}] PR is not affected: unsupported stack (${STACK_VERSION})"
@@ -689,30 +687,31 @@ is_pr_affected() {
             echo "[${package}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
             return 1
         fi
+        if [[ "${package}" == "fleet_server" ]]; then
+            echoerr "fleet_server not supported. Skipped"
+            echo "[${package}] not supported"
+            return 1
+        fi
+        if ! is_spec_3_0_0 ; then
+            echoerr "Not v3 spec version. Skipped"
+            echo "[${package}] spec <3.0.0"
+            return 1
+        fi
     fi
 
     if [[ "${FORCE_CHECK_ALL}" == "true" ]];then
         echo "[${package}] PR is affected: \"force_check_all\" parameter enabled"
         return 0
     fi
 
-    if [[ "${from}" == ""  || "${to}" == "" ]]; then
-        echo "[${package}] Calculating commits: from '${from}' - to: '${to}'"
-        # setting range of changesets to check differences
-        from="$(get_from_changeset)"
-        to="$(get_to_changeset)"
-    fi
-
-    echo "[${package}]: commits: from: '${from}' - to: '${to}'"
-
-    echo "[${package}] git-diff: check non-package files"
     commit_merge=$(git merge-base "${from}" "${to}")
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
+    echoerr "[${package}] git-diff: check non-package files (${commit_merge}..${to})"
+    if git diff --name-only "${commit_merge}" "${to}" | grep -q -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi
-    echo "[${package}] git-diff: check package files"
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E "^packages/${package}/" ; then
+    echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})"
+    if git diff --name-only "${commit_merge}" "${to}" | grep -q -E "^packages/${package}/" ; then
         echo "[${package}] PR is affected: found package files"
         return 0
     fi
@@ -728,7 +727,11 @@ is_pr() {
 }
 
 kubernetes_service_deployer_used() {
-    find . -type d | grep -E '_dev/deploy/k8s$'
+    # Not able to use -q in parameter
+    # as set -o pipefail is defined, when adding "-q" parameter, grep finishes with its first match
+    # but find still is writing to the pipe causing the SIGPIPE
+    # https://tldp.org/LDP/lpg/node20.html
+    find . -type d | grep -E "_dev/deploy/k8s$" > /dev/null
 }
 
 teardown_serverless_test_package() {
@@ -954,54 +957,35 @@ upload_safe_logs_from_package() {
 
 # Helper to run all tests and checks for a package
 process_package() {
-    local package="$1"
-    local from=${2:-""}
-    local to=${3:-""}
+    local package="${1}"
+    local failed_packages_file="${2:-""}"
     local exit_code=0
 
     echo "--- Package ${package}: check"
     pushd "${package}" > /dev/null
 
     clean_safe_logs
 
-    if is_serverless ; then
-        if [[ "${package}" == "fleet_server" ]]; then
-            echo "fleet_server not supported. Skipped"
-            echo "- [${package}] not supported" >> "${SKIPPED_PACKAGES_FILE_PATH}"
-            popd > /dev/null
-            return
-        fi
-        if ! is_spec_3_0_0 ; then
-            echo "Not v3 spec version. Skipped"
-            echo "- [${package}] spec <3.0.0" >> "${SKIPPED_PACKAGES_FILE_PATH}"
-            popd > /dev/null
-            return
-        fi
-    fi
-
-    if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then
-        echo "${reason}"
-        echo "- ${reason}" >> "${SKIPPED_PACKAGES_FILE_PATH}"
-        popd > /dev/null
-        return
-    fi
-
-    echo "${reason}"
-
     use_kind=0
-    if kubernetes_service_deployer_used ; then
-        echo "Kubernetes service deployer is used. Creating Kind cluster"
-        use_kind=1
-        if ! create_kind_cluster ; then
-            popd > /dev/null
-            return 1
+    if ! is_serverless ; then
+        # TODO: in serverless system tests are not triggered,
+        # there is no need to create the kind cluster in these CI builds.
+        if kubernetes_service_deployer_used ; then
+            echo "Kubernetes service deployer is used. Creating Kind cluster"
+            use_kind=1
+            if ! create_kind_cluster ; then
+                popd > /dev/null
+                return 1
+            fi
         fi
     fi
 
     if ! run_tests_package "${package}" ; then
         exit_code=1
         echo "[${package}] run_tests_package failed"
-        echo "- ${package}" >> "${FAILED_PACKAGES_FILE_PATH}"
+        if [[ "${failed_packages_file}" != "" ]]; then
+            echo "- ${package}" >> "${failed_packages_file}"
+        fi
     fi
 
     if ! is_serverless ; then

.buildkite/scripts/test_integrations_with_serverless.sh

@@ -61,13 +61,32 @@ echo "Done."
 
 # setting range of changesets to check differences
 from="$(get_from_changeset)"
+if [[ "${from}" == "" ]]; then
+    echo "Missing \"from\" changset".
+    exit 1
+fi
 to="$(get_to_changeset)"
+if [[ "${to}" == "" ]]; then
+    echo "Missing \"to\" changset".
+    exit 1
+fi
+echo "Checking with commits: from: '${from}' to: '${to}'"
 
 any_package_failing=0
 
 pushd packages > /dev/null
 for package in $(list_all_directories); do
-    if ! process_package "${package}" "${from}" "${to}"; then
+    pushd "${package}" > /dev/null
+    if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then
+        echo "${reason}"
+        echo "- ${reason}" >> "${SKIPPED_PACKAGES_FILE_PATH}"
+        popd > /dev/null
+        continue
+    fi
+    echo "${reason}"
+    popd > /dev/null
+
+    if ! process_package "${package}" "${FAILED_PACKAGES_FILE_PATH}" ; then
         any_package_failing=1
     fi
 done

.buildkite/scripts/test_one_package.sh

@@ -4,17 +4,8 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-
-# used in common.sh
-SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
-FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
-
 # package name
 package="$1"
-# changesets
-from=${2:-""}
-to=${3:-""}
-
 
 if [ ! -d packages ]; then
     echo "Missing packages folder"
@@ -36,7 +27,7 @@ use_elastic_package
 
 pushd packages > /dev/null
 exit_code=0
-if ! process_package "${package}" "${from}" "${to}"; then
+if ! process_package "${package}" ; then
     echo "[${package}] failed"
     exit_code=1
 fi

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -25,9 +25,17 @@ EOF
 
 # setting range of changesets to check differences
 from="$(get_from_changeset)"
+if [[ "${from}" == "" ]]; then
+    echo "Missing \"from\" changset".
+    exit 1
+fi
 to="$(get_to_changeset)"
+if [[ "${to}" == "" ]]; then
+    echo "Missing \"to\" changset".
+    exit 1
+fi
 
-echo "[DEBUG] Checking with commits: from: '${from}' to: '${to}'"
+echo "Checking with commits: from: '${from}' to: '${to}'"
 
 # This variable does not exist in builds triggered automatically
 GITHUB_PR_TRIGGER_COMMENT="${GITHUB_PR_TRIGGER_COMMENT:-""}"