Skip to content

Commit 604c606

Browse files
authored
[system testing] Add provision for system testing negative / false-positive scenarios in packages (#1333)
1 parent 8a95967 commit 604c606

File tree

23 files changed

+823
-6
lines changed

23 files changed

+823
-6
lines changed

.buildkite/hooks/pre-command

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact
3333
# Secrets must be redacted
3434
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
3535

36-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
36+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
3737
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
3838
fi
3939

.buildkite/pipeline.trigger.integration.tests.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,22 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
4949
fi
5050
done
5151

52+
pushd test/packages/false_positives > /dev/null
53+
for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
54+
package_name=$(basename ${package})
55+
echo " - label: \":go: Running integration test (false positive): ${package_name}\""
56+
echo " key: \"integration-false_positives-${package_name}\""
57+
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
58+
echo " env:"
59+
echo " UPLOAD_SAFE_LOGS: 1"
60+
echo " agents:"
61+
echo " provider: \"gcp\""
62+
echo " artifact_paths:"
63+
echo " - build/test-results/*.xml"
64+
done
65+
66+
popd > /dev/null
67+
5268
pushd test/packages/parallel > /dev/null
5369
for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
5470
package_name=$(basename ${package})

.buildkite/scripts/integration_tests.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ source .buildkite/scripts/install_deps.sh
2424
source .buildkite/scripts/tooling.sh
2525

2626
PARALLEL_TARGET="test-check-packages-parallel"
27+
FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
2728
KIND_TARGET="test-check-packages-with-kind"
2829
TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
2930
GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
@@ -108,7 +109,7 @@ if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
108109
fi
109110

110111
echo "--- Run integration test ${TARGET}"
111-
if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
112+
if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
112113
make install
113114

114115
# allow to fail this command, to be able to upload safe logs

Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,17 @@ test-stack-command-8x:
6969

7070
test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
7171

72-
test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks
72+
test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives
7373

7474
test-check-packages-with-kind:
7575
PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
7676

7777
test-check-packages-other:
7878
PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
7979

80+
test-check-packages-false-positives:
81+
PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
82+
8083
test-check-packages-benchmarks:
8184
PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
8285

docs/howto/system_testing.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,23 @@ to indexing generated data from the integration's data streams into Elasticsearc
568568
elastic-package test system --generate
569569
```
570570

571+
### System testing negative or false-positive scenarios
572+
573+
The system tests support packages to be tested for negative scenarios. An example would be to test that the `assert.hit_count` is verified when all the docs are ingested rather than just finding enough docs for the testcase.
574+
575+
There are some special rules for testing negative scenarios
576+
577+
- The negative / false-positive test packages are added under `test/packages/false_positives`
578+
- It is required to have a file `<package_name>.expected_errors` with the lines needed for every package added under `test/packages/false_positives`.
579+
- One line per error, taking into account that all `\n` were removed, meaning it is just one line for everything.
580+
- As it is used `grep` with `-E` some kind of regexes can be used.
581+
582+
Example `expected_errors` file content:
583+
584+
```xml
585+
<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
586+
```
587+
571588
## Continuous Integration
572589

573590
`elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes.

internal/testrunner/runners/system/runner.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
620620
// (TODO in future) Optionally exercise service to generate load.
621621
logger.Debug("checking for expected data in data stream...")
622622
var hits *hits
623+
oldHits := 0
623624
passed, err := waitUntilTrue(func() (bool, error) {
624625
if signal.SIGINT() {
625626
return true, errors.New("SIGINT: cancel waiting for policy assigned")
@@ -629,11 +630,21 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
629630
hits, err = r.getDocs(dataStream)
630631

631632
if config.Assert.HitCount > 0 {
632-
return hits.size() >= config.Assert.HitCount, err
633-
}
633+
if hits.size() < config.Assert.HitCount {
634+
return false, err
635+
}
636+
637+
ret := hits.size() == oldHits
638+
if !ret {
639+
oldHits = hits.size()
640+
time.Sleep(4 * time.Second)
641+
}
634642

643+
return ret, err
644+
}
635645
return hits.size() > 0, err
636646
}, waitForDataTimeout)
647+
637648
if err != nil {
638649
return result.WithError(err)
639650
}
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/bin/bash
2+
3+
set -euxo pipefail
4+
5+
cleanup() {
6+
r=$?
7+
8+
# Dump stack logs
9+
elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}"
10+
11+
# Take down the stack
12+
elastic-package stack down -v
13+
14+
# Clean used resources
15+
for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
16+
(
17+
cd $d
18+
elastic-package clean -v
19+
)
20+
done
21+
22+
# This is a false positive scenario and tests that the test case failure is a success scenario
23+
if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then
24+
if [ $r == 1 ]; then
25+
EXPECTED_ERRORS_FILE="test/packages/false_positives/${PACKAGE_UNDER_TEST}.expected_errors"
26+
if [ ! -f ${EXPECTED_ERRORS_FILE} ]; then
27+
echo "Error: Missing expected errors file: ${EXPECTED_ERRORS_FILE}"
28+
fi
29+
RESULTS_NO_SPACES="build/test-results-no-spaces.xml"
30+
cat build/test-results/*.xml | tr -d '\n' > ${RESULTS_NO_SPACES}
31+
32+
# check number of expected errors
33+
number_errors=$(cat build/test-results/*.xml | grep "<failure>" | wc -l)
34+
expected_errors=$(cat ${EXPECTED_ERRORS_FILE} | wc -l)
35+
36+
if [ ${number_errors} -ne ${expected_errors} ]; then
37+
echo "Error: There are unexpected errors in ${PACKAGE_UNDER_TEST}"
38+
exit 1
39+
fi
40+
41+
# check whether or not the expected errors exist in the xml files
42+
while read -r line; do
43+
cat ${RESULTS_NO_SPACES} | grep -E "${line}"
44+
done < ${EXPECTED_ERRORS_FILE}
45+
rm -f build/test-results/*.xml
46+
rm -f ${RESULTS_NO_SPACES}
47+
exit 0
48+
elif [ $r == 0 ]; then
49+
echo "Error: Expected to fail tests, but there was none failing"
50+
exit 1
51+
fi
52+
fi
53+
54+
exit $r
55+
}
56+
57+
trap cleanup EXIT
58+
59+
export ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
60+
61+
OLDPWD=$PWD
62+
# Build/check packages
63+
for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
64+
(
65+
cd $d
66+
elastic-package check -v
67+
)
68+
done
69+
cd -
70+
71+
# Update the stack
72+
elastic-package stack update -v
73+
74+
# Boot up the stack
75+
elastic-package stack up -d -v
76+
77+
elastic-package stack status
78+
79+
# Run package tests
80+
eval "$(elastic-package stack shellinit)"
81+
82+
for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
83+
(
84+
cd $d
85+
elastic-package install -v
86+
87+
# defer-cleanup is set to a short period to verify that the option is available
88+
elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage
89+
)
90+
cd -
91+
done

scripts/test-check-packages.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ cleanup() {
2929
elastic-package clean -v
3030
)
3131
done
32-
32+
3333
exit $r
3434
}
3535

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dependencies:
2+
ecs:
3+
reference: [email protected]

0 commit comments

Comments
 (0)