diff --git a/.buildkite/scripts/buildkite-k8s-integration-tests.sh b/.buildkite/scripts/buildkite-k8s-integration-tests.sh index 892ba110ad7..93054278d14 100755 --- a/.buildkite/scripts/buildkite-k8s-integration-tests.sh +++ b/.buildkite/scripts/buildkite-k8s-integration-tests.sh @@ -95,10 +95,14 @@ EOF pod_logs_base="${PWD}/build/${fully_qualified_group_name}.pod_logs_dump" set +e - K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${group_name}" -integration.sudo="false" + K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration/k8s -v -args -integration.groups="${group_name}" -integration.sudo="false" exit_status=$? set -e + if [[ $exit_status -ne 0 ]]; then + echo "^^^ +++" + fi + if [[ $TESTS_EXIT_STATUS -eq 0 && $exit_status -ne 0 ]]; then TESTS_EXIT_STATUS=$exit_status fi diff --git a/.buildkite/scripts/steps/k8s-extended-tests.sh b/.buildkite/scripts/steps/k8s-extended-tests.sh index 8eba5ea667f..85064ebbab9 100755 --- a/.buildkite/scripts/steps/k8s-extended-tests.sh +++ b/.buildkite/scripts/steps/k8s-extended-tests.sh @@ -26,7 +26,7 @@ else fi SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package -TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix +TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:testKubernetesMatrix TESTS_EXIT_STATUS=$? set -e diff --git a/NOTICE.txt b/NOTICE.txt index 0a84f84c1fb..4123939dfe8 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -19156,222 +19156,11 @@ Contents of probable licence file $GOMODCACHE/sigs.k8s.io/e2e-framework@v0.4.0/L -------------------------------------------------------------------------------- Dependency : sigs.k8s.io/kustomize/api -Version: v0.13.5-0.20230601165947-6ce0bf390ce3 +Version: v0.18.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kustomize/api@v0.13.5-0.20230601165947-6ce0bf390ce3/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : sigs.k8s.io/kustomize/kyaml -Version: v0.14.3-0.20230601165947-6ce0bf390ce3 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kustomize/kyaml@v0.14.3-0.20230601165947-6ce0bf390ce3/LICENSE: +Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kustomize/api@v0.18.0/LICENSE: Apache License Version 2.0, January 2004 @@ -96495,45 +96284,6 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/proto/otlp@v1. limitations under the License. --------------------------------------------------------------------------------- -Dependency : go.starlark.net -Version: v0.0.0-20230525235612-a134d8f9ddca -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.starlark.net@v0.0.0-20230525235612-a134d8f9ddca/LICENSE: - -Copyright (c) 2017 The Bazel Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the - distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : go.uber.org/atomic Version: v1.11.0 @@ -101963,6 +101713,217 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : sigs.k8s.io/kustomize/kyaml +Version: v0.18.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kustomize/kyaml@v0.18.1/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : sigs.k8s.io/structured-merge-diff/v4 Version: v4.4.1 diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 0aaa1b9808c..a3964a96a9b 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -65,9 +65,9 @@ The test are run with mage using the `integration` namespace: - `mage integration:matrix` to run all tests on the complete matrix of supported operating systems and architectures of the Elastic Agent. -- `mage integration:kubernetes` to run kubernetes tests for the default image on the default version of kubernetes (all previous commands will not run any kubernetes tests). +- `mage integration:testKubernetes` to run kubernetes tests for the default image on the default version of kubernetes (all previous commands will not run any kubernetes tests). -- `mage integration:kubernetesMatrix` to run a matrix of kubernetes tests for all image types and supported versions of kubernetes. +- `mage integration:testKubernetesMatrix` to run a matrix of kubernetes tests for all image types and supported versions of kubernetes. #### Selecting specific platform @@ -83,7 +83,7 @@ between, and it can be very specific or not very specific. - `TEST_PLATFORMS="linux/amd64/ubuntu/20.04 mage integration:test` to execute tests only on Ubuntu 20.04 ARM64. - `TEST_PLATFORMS="windows/amd64/2022 mage integration:test` to execute tests only on Windows Server 2022. - `TEST_PLATFORMS="linux/amd64 windows/amd64/2022 mage integration:test` to execute tests on Linux AMD64 and Windows Server 2022. -- `TEST_PLATFORMS="kubernetes/arm64/1.31.0/wolfi" mage integration:kubernetes` to execute kubernetes tests on Kubernetes version 1.31.0 with wolfi docker variant. +- `INSTANCE_PROVISIONER="kind" TEST_PLATFORMS="kubernetes/arm64/1.33.0/wolfi" mage integration:testKubernetes` to execute kubernetes tests on Kubernetes version 1.33.0 with wolfi docker variant under kind cluster. > **_NOTE:_** This only filters down the tests based on the platform. It will not execute a tests on a platform unless > the test defines as supporting it. @@ -369,7 +369,7 @@ not cause already provisioned resources to be replaced with an instance created ### Kind Instance Provisioner Use only when running Kubernetes tests. Uses local installed kind to create Kubernetes clusters on the fly. -- `INSTANCE_PROVISIONER="kind" mage integration:kubernetes` +- `INSTANCE_PROVISIONER="kind" mage integration:testKubernetes` ## Troubleshooting Tips diff --git a/go.mod b/go.mod index a827e0cdec4..8f47fdfdb23 100644 --- a/go.mod +++ b/go.mod @@ -90,8 +90,7 @@ require ( k8s.io/client-go v0.31.3 kernel.org/pub/linux/libs/security/libcap/cap v1.2.70 sigs.k8s.io/e2e-framework v0.4.0 - sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 - sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 + sigs.k8s.io/kustomize/api v0.18.0 ) require ( @@ -577,7 +576,6 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect go.opentelemetry.io/otel/trace v1.34.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect @@ -608,6 +606,7 @@ require ( oras.land/oras-go v1.2.5 // indirect sigs.k8s.io/controller-runtime v0.19.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index c84fbca8a01..60552d94a9d 100644 --- a/go.sum +++ b/go.sum @@ -1696,8 +1696,6 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -1942,7 +1940,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= @@ -2246,10 +2243,10 @@ sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY= sigs.k8s.io/e2e-framework v0.4.0/go.mod h1:JilFQPF1OL1728ABhMlf9huse7h+uBJDXl9YeTs49A8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= +sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= +sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= +sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= +sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/magefile.go b/magefile.go index b9b61304576..389f1e008b9 100644 --- a/magefile.go +++ b/magefile.go @@ -1921,6 +1921,7 @@ func (Integration) Check() error { define.ValidateDir("testing/integration"), define.ValidateDir("testing/integration/serverless"), define.ValidateDir("testing/integration/leak"), + define.ValidateDir("testing/integration/k8s"), ) } @@ -2001,24 +2002,34 @@ func (Integration) TestServerless(ctx context.Context) error { return integRunner(ctx, "testing/integration/serverless", false, "") } -// Kubernetes runs kubernetes integration tests -func (Integration) Kubernetes(ctx context.Context) error { +// TestKubernetes runs kubernetes integration tests +func (Integration) TestKubernetes(ctx context.Context) error { // invoke integration tests if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil { return err } - return integRunner(ctx, "testing/integration", false, "") + return integRunner(ctx, "testing/integration/k8s", false, "") } -// KubernetesMatrix runs a matrix of kubernetes integration tests -func (Integration) KubernetesMatrix(ctx context.Context) error { +// TestKubernetesSingle runs single k8s integration test +func (Integration) TestKubernetesSingle(ctx context.Context, testName string) error { // invoke integration tests if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil { return err } - return integRunner(ctx, "testing/integration", true, "") + return integRunner(ctx, "testing/integration/k8s", false, testName) +} + +// TestKubernetesMatrix runs a matrix of kubernetes integration tests +func (Integration) TestKubernetesMatrix(ctx context.Context) error { + // invoke integration tests + if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil { + return err + } + + return integRunner(ctx, "testing/integration/k8s", true, "") } // UpdateVersions runs an update on the `.agent-versions.yml` fetching diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go index 989f12d2feb..cc1f662634a 100644 --- a/pkg/testing/define/define.go +++ b/pkg/testing/define/define.go @@ -37,6 +37,7 @@ var osInfo *types.OSInfo var osInfoErr error var osInfoOnce sync.Once var noSpecialCharsRegexp = regexp.MustCompile("[^a-zA-Z0-9]+") +var kubernetesSupported = false // Require defines what this test requires for it to be run by the test runner. // @@ -46,6 +47,12 @@ func Require(t *testing.T, req Requirements) *Info { return defineAction(t, req) } +// SetKubernetesSupported sets the kubernetesSupported flag to true +// to allow kubernetes tests to be run. +func SetKubernetesSupported() { + kubernetesSupported = true +} + type Info struct { // ESClient is the elasticsearch client to communicate with elasticsearch. // This is only present if you say a cloud is required in the `define.Require`. @@ -139,7 +146,7 @@ func findProjectRoot() (string, error) { } } -func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Info { +func runOrSkip(t *testing.T, req Requirements, local bool) *Info { // always validate requirement is valid if err := req.Validate(); err != nil { panic(fmt.Sprintf("test %s has invalid requirements: %s", t.Name(), err)) @@ -165,7 +172,7 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf return nil } for _, o := range req.OS { - if o.Type == Kubernetes && !kubernetes { + if o.Type == Kubernetes && !kubernetesSupported { t.Skip("test requires kubernetes") return nil } diff --git a/pkg/testing/define/define_all.go b/pkg/testing/define/define_all.go index 53aa5c02acc..febc52786d1 100644 --- a/pkg/testing/define/define_all.go +++ b/pkg/testing/define/define_all.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -//go:build !define && !local && !kubernetes +//go:build !define && !local package define @@ -11,5 +11,5 @@ import ( ) func defineAction(t *testing.T, req Requirements) *Info { - return runOrSkip(t, req, false, false) + return runOrSkip(t, req, false) } diff --git a/pkg/testing/define/define_kubernetes.go b/pkg/testing/define/define_kubernetes.go deleted file mode 100644 index cf39e7f20a6..00000000000 --- a/pkg/testing/define/define_kubernetes.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -//go:build kubernetes && !define && !local - -package define - -import ( - "testing" -) - -func defineAction(t *testing.T, req Requirements) *Info { - return runOrSkip(t, req, false, true) -} diff --git a/pkg/testing/define/define_local.go b/pkg/testing/define/define_local.go index 5ae211b8e7e..270b7000281 100644 --- a/pkg/testing/define/define_local.go +++ b/pkg/testing/define/define_local.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -//go:build local && !define && !kubernetes +//go:build local && !define package define @@ -11,5 +11,5 @@ import ( ) func defineAction(t *testing.T, req Requirements) *Info { - return runOrSkip(t, req, true, false) + return runOrSkip(t, req, true) } diff --git a/testing/integration/common.go b/testing/integration/common.go new file mode 100644 index 00000000000..4874be40549 --- /dev/null +++ b/testing/integration/common.go @@ -0,0 +1,53 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package integration + +import ( + "fmt" + "net/url" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/testing/estools" +) + +func GetESHost() (string, error) { + fixedESHost := os.Getenv("ELASTICSEARCH_HOST") + parsedES, err := url.Parse(fixedESHost) + if err != nil { + return "", err + } + if parsedES.Port() == "" { + fixedESHost = fmt.Sprintf("%s:443", fixedESHost) + } + return fixedESHost, nil +} + +// FindESDocs runs `findFn` until at least one document is returned and there is no error +func FindESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents { + var docs estools.Documents + require.Eventually( + t, + func() bool { + var err error + docs, err = findFn() + if err != nil { + t.Logf("got an error querying ES, retrying. Error: %s", err) + return false + } + + return docs.Hits.Total.Value != 0 + }, + 3*time.Minute, + 15*time.Second, + ) + + return docs +} diff --git a/testing/integration/k8s/common.go b/testing/integration/k8s/common.go new file mode 100644 index 00000000000..65667ed2aa5 --- /dev/null +++ b/testing/integration/k8s/common.go @@ -0,0 +1,587 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package k8s + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/gofrs/uuid/v5" + "github.com/stretchr/testify/require" + helmKube "helm.sh/helm/v3/pkg/kube" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + cliResource "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/api/krusty" + + "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent-libs/testing/estools" + aclient "github.com/elastic/elastic-agent/pkg/control/v2/client" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" + "github.com/elastic/elastic-agent/testing/integration" + "github.com/elastic/go-elasticsearch/v8" +) + +var noSpecialCharsRegexp = regexp.MustCompile("[^a-zA-Z0-9]+") + +// k8sContext contains all the information needed to run a k8s test +type k8sContext struct { + client klient.Client + clientSet *kubernetes.Clientset + // logsBasePath is the path that will be used to store the pod logs in a case a test fails + logsBasePath string + // agentImage is the full image of elastic-agent to use in the test + agentImage string + // agentImageRepo is the repository of elastic-agent image to use in the test + agentImageRepo string + // agentImageTag is the tag of elastic-agent image to use in the test + agentImageTag string + // esHost is the host of the elasticsearch to use in the test + esHost string + // esAPIKey is the API key of the elasticsearch to use in the test + esAPIKey string + // esEncodedAPIKey is the encoded API key of the elasticsearch to use in the test + esEncodedAPIKey string + // enrollParams contains the information needed to enroll an agent with Fleet in the test + enrollParams *fleettools.EnrollParams + // createdAt is the time when the k8sContext was created + createdAt time.Time +} + +// getNamespace returns a unique namespace for the current test +func (k k8sContext) getNamespace(t *testing.T) string { + if ns := os.Getenv("K8S_TESTS_NAMESPACE"); ns != "" { + return ns + } + + nsUUID, err := uuid.NewV4() + if err != nil { + t.Fatalf("error generating namespace UUID: %v", err) + } + hasher := sha256.New() + hasher.Write([]byte(nsUUID.String())) + testNamespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil))) + return noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") +} + +// k8sGetContext performs all the necessary checks to get a k8sContext for the current test +func k8sGetContext(t *testing.T, info *define.Info) k8sContext { + agentImage := os.Getenv("AGENT_IMAGE") + require.NotEmpty(t, agentImage, "AGENT_IMAGE must be set") + + agentImageParts := strings.SplitN(agentImage, ":", 2) + require.Len(t, agentImageParts, 2, "AGENT_IMAGE must be in the form ':'") + agentImageRepo := agentImageParts[0] + agentImageTag := agentImageParts[1] + + client, err := info.KubeClient() + require.NoError(t, err) + require.NotNil(t, client) + + clientSet, err := kubernetes.NewForConfig(client.RESTConfig()) + require.NoError(t, err) + require.NotNil(t, clientSet) + + testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") + require.NotEmpty(t, testLogsBasePath, "K8S_TESTS_POD_LOGS_BASE must be set") + + err = os.MkdirAll(testLogsBasePath, 0o755) + require.NoError(t, err, "failed to create test logs directory") + + esHost, err := integration.GetESHost() + require.NoError(t, err, "cannot parse ELASTICSEARCH_HOST") + + esAPIKey, err := generateESAPIKey(info.ESClient, info.Namespace) + require.NoError(t, err, "failed to generate ES API key") + require.NotEmpty(t, esAPIKey, "failed to generate ES API key") + + enrollParams, err := fleettools.NewEnrollParams(context.Background(), info.KibanaClient) + require.NoError(t, err, "failed to create fleet enroll params") + + return k8sContext{ + client: client, + clientSet: clientSet, + agentImage: agentImage, + agentImageRepo: agentImageRepo, + agentImageTag: agentImageTag, + logsBasePath: testLogsBasePath, + esHost: esHost, + esAPIKey: esAPIKey.APIKey, + esEncodedAPIKey: esAPIKey.Encoded, + enrollParams: enrollParams, + createdAt: time.Now(), + } +} + +// generateESAPIKey generates an API key for the given Elasticsearch. +func generateESAPIKey(esClient *elasticsearch.Client, keyName string) (estools.APIKeyResponse, error) { + return estools.CreateAPIKey(context.Background(), esClient, estools.APIKeyRequest{Name: keyName, Expiration: "1d"}) +} + +// int64Ptr returns a pointer to the given int64 +func int64Ptr(val int64) *int64 { + valPtr := val + return &valPtr +} + +// k8sCheckAgentStatus checks that the agent reports healthy. +func k8sCheckAgentStatus(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer, + namespace string, agentPodName string, containerName string, componentPresence map[string]bool, +) error { + command := []string{"elastic-agent", "status", "--output=json"} + stopCheck := errors.New("stop check") + + // we will wait maximum 120 seconds for the agent to report healthy + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + checkStatus := func() error { + pod := corev1.Pod{} + if err := client.Resources(namespace).Get(ctx, agentPodName, namespace, &pod); err != nil { + return err + } + + for _, container := range pod.Status.ContainerStatuses { + if container.Name != containerName { + continue + } + + if restarts := container.RestartCount; restarts != 0 { + return fmt.Errorf("container %q of pod %q has restarted %d times: %w", containerName, agentPodName, restarts, stopCheck) + } + } + + status := atesting.AgentStatusOutput{} // clear status output + stdout.Reset() + stderr.Reset() + if err := client.Resources().ExecInPod(ctx, namespace, agentPodName, containerName, command, stdout, stderr); err != nil { + return err + } + + if err := json.Unmarshal(stdout.Bytes(), &status); err != nil { + return err + } + + var err error + // validate that the components defined are also healthy if they should exist + for component, shouldBePresent := range componentPresence { + compState, ok := getAgentComponentState(status, component) + if shouldBePresent { + if !ok { + // doesn't exist + err = errors.Join(err, fmt.Errorf("required component %s not found", component)) + } else if compState != int(aclient.Healthy) { + // not healthy + err = errors.Join(err, fmt.Errorf("required component %s is not healthy", component)) + } + } else if ok { + // should not be present + err = errors.Join(err, fmt.Errorf("component %s should not be present", component)) + } + } + return err + } + for { + err := checkStatus() + if err == nil { + return nil + } else if errors.Is(err, stopCheck) { + return err + } + if ctx.Err() != nil { + // timeout waiting for agent to become healthy + return errors.Join(err, errors.New("timeout waiting for agent to become healthy")) + } + time.Sleep(100 * time.Millisecond) + } +} + +// k8sGetAgentID returns the agent ID for the given agent pod +func k8sGetAgentID(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer, + namespace string, agentPodName string, containerName string, +) (string, error) { + command := []string{"elastic-agent", "status", "--output=json"} + + status := atesting.AgentStatusOutput{} // clear status output + stdout.Reset() + stderr.Reset() + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + err := client.Resources().ExecInPod(ctx, namespace, agentPodName, containerName, command, stdout, stderr) + cancel() + if err != nil { + return "", err + } + + if err := json.Unmarshal(stdout.Bytes(), &status); err != nil { + return "", err + } + + return status.Info.ID, nil +} + +// getAgentComponentState returns the component state for the given component name and a bool indicating if it exists. +func getAgentComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { + for _, comp := range status.Components { + if comp.Name == componentName { + return comp.State, true + } + } + return -1, false +} + +// k8sKustomizeAdjustObjects adjusts the namespace of given k8s objects and calls the given callbacks for the containers and the pod +func k8sKustomizeAdjustObjects(objects []k8s.Object, namespace string, containerName string, cbContainer func(container *corev1.Container), cbPod func(pod *corev1.PodSpec)) { + // Update the agent image and image pull policy as it is already loaded in kind cluster + for _, obj := range objects { + obj.SetNamespace(namespace) + var podSpec *corev1.PodSpec + switch objWithType := obj.(type) { + case *appsv1.DaemonSet: + podSpec = &objWithType.Spec.Template.Spec + case *appsv1.StatefulSet: + podSpec = &objWithType.Spec.Template.Spec + case *appsv1.Deployment: + podSpec = &objWithType.Spec.Template.Spec + case *appsv1.ReplicaSet: + podSpec = &objWithType.Spec.Template.Spec + case *batchv1.Job: + podSpec = &objWithType.Spec.Template.Spec + case *batchv1.CronJob: + podSpec = &objWithType.Spec.JobTemplate.Spec.Template.Spec + default: + continue + } + + if cbPod != nil { + cbPod(podSpec) + } + + for idx, container := range podSpec.Containers { + if container.Name != containerName { + continue + } + if cbContainer != nil { + cbContainer(&podSpec.Containers[idx]) + } + } + } +} + +// k8sRenderKustomize renders the given kustomize directory to YAML +func k8sRenderKustomize(kustomizePath string) ([]byte, error) { + // Create a file system pointing to the kustomize directory + fSys := filesys.MakeFsOnDisk() + + // Create a kustomizer + k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + + // Run the kustomizer on the given directory + resMap, err := k.Run(fSys, kustomizePath) + if err != nil { + return nil, err + } + + // Convert the result to YAML + renderedManifest, err := resMap.AsYaml() + if err != nil { + return nil, err + } + + return renderedManifest, nil +} + +// k8sDeleteOpts contains options for deleting k8s objects +type k8sDeleteOpts struct { + // wait for the objects to be deleted + wait bool + // timeout for waiting for the objects to be deleted + waitTimeout time.Duration +} + +// k8sDeleteObjects deletes the given k8s objects and waits for them to be deleted if wait is true. +func k8sDeleteObjects(ctx context.Context, client klient.Client, opts k8sDeleteOpts, objects ...k8s.Object) error { + if len(objects) == 0 { + return nil + } + + // Delete the objects + for _, obj := range objects { + _ = client.Resources(obj.GetNamespace()).Delete(ctx, obj) + } + + if !opts.wait { + // no need to wait + return nil + } + + if opts.waitTimeout == 0 { + // default to 20 seconds + opts.waitTimeout = 20 * time.Second + } + + timeoutCtx, timeoutCancel := context.WithTimeout(ctx, opts.waitTimeout) + defer timeoutCancel() + for _, obj := range objects { + for { + if timeoutCtx.Err() != nil { + return errors.New("timeout waiting for k8s objects to be deleted") + } + + err := client.Resources().Get(timeoutCtx, obj.GetName(), obj.GetNamespace(), obj) + if err != nil { + // object has been deleted + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + return nil +} + +// k8sCreateOpts contains options for k8sCreateObjects +type k8sCreateOpts struct { + // namespace is the namespace to create the objects in + namespace string + // wait specifies whether to wait for the objects to be ready + wait bool + // waitTimeout is the timeout for waiting for the objects to be ready if wait is true + waitTimeout time.Duration +} + +// k8sCreateObjects creates k8s objects and waits for them to be ready if specified in opts. +// Note that if opts.namespace is not empty, all objects will be created and updated to reference +// the given namespace. +func k8sCreateObjects(ctx context.Context, client klient.Client, opts k8sCreateOpts, objects ...k8s.Object) error { + // Create the objects + for _, obj := range objects { + if opts.namespace != "" { + // update the namespace + obj.SetNamespace(opts.namespace) + + // special case for ClusterRoleBinding and RoleBinding + // update the subjects to reference the given namespace + switch objWithType := obj.(type) { + case *rbacv1.ClusterRoleBinding: + for idx := range objWithType.Subjects { + objWithType.Subjects[idx].Namespace = opts.namespace + } + case *rbacv1.RoleBinding: + for idx := range objWithType.Subjects { + objWithType.Subjects[idx].Namespace = opts.namespace + } + } + } + if err := client.Resources().Create(ctx, obj); err != nil { + return fmt.Errorf("failed to create object %s: %w", obj.GetName(), err) + } + } + + if !opts.wait { + // no need to wait + return nil + } + + if opts.waitTimeout == 0 { + // default to 120 seconds + opts.waitTimeout = 120 * time.Second + } + + return k8sWaitForReady(ctx, client, opts.waitTimeout, objects...) +} + +// k8sWaitForReady waits for the given k8s objects to be ready +func k8sWaitForReady(ctx context.Context, client klient.Client, waitDuration time.Duration, objects ...k8s.Object) error { + // use ready checker from helm kube + clientSet, err := kubernetes.NewForConfig(client.RESTConfig()) + if err != nil { + return fmt.Errorf("error creating clientset: %w", err) + } + readyChecker := helmKube.NewReadyChecker(clientSet, func(s string, i ...interface{}) {}) + + ctxTimeout, cancel := context.WithTimeout(ctx, waitDuration) + defer cancel() + + waitFn := func(ri *cliResource.Info) error { + // here we wait for the k8s object (e.g. deployment, daemonset, pod) to be ready + for { + ready, readyErr := readyChecker.IsReady(ctxTimeout, ri) + if ready { + // k8s object is ready + return nil + } + // k8s object is not ready yet + readyErr = errors.Join(fmt.Errorf("k8s object %s is not ready", ri.Name), readyErr) + + if ctxTimeout.Err() != nil { + // timeout + return errors.Join(fmt.Errorf("timeout waiting for k8s object %s to be ready", ri.Name), readyErr) + } + time.Sleep(100 * time.Millisecond) + } + } + + for _, o := range objects { + // convert k8s.Object to resource.Info for ready checker + runtimeObj, ok := o.(runtime.Object) + if !ok { + return fmt.Errorf("unable to convert k8s.Object %s to runtime.Object", o.GetName()) + } + + if err := waitFn(&cliResource.Info{ + Object: runtimeObj, + Name: o.GetName(), + Namespace: o.GetNamespace(), + }); err != nil { + return err + } + // extract pod label selector for all k8s objects that have underlying pods + oPodsLabelSelector, err := helmKube.SelectorsForObject(runtimeObj) + if err != nil { + // k8s object does not have pods + continue + } + + podList, err := clientSet.CoreV1().Pods(o.GetNamespace()).List(ctx, metav1.ListOptions{ + LabelSelector: oPodsLabelSelector.String(), + }) + if err != nil { + return fmt.Errorf("error listing pods: %w", err) + } + + // here we wait for the all pods to be ready + for _, pod := range podList.Items { + if err := waitFn(&cliResource.Info{ + Object: &pod, + Name: pod.Name, + Namespace: pod.Namespace, + }); err != nil { + return err + } + } + } + + return nil +} + +func k8sSchedulableNodeCount(ctx context.Context, kCtx k8sContext) (int, error) { + nodeList := corev1.NodeList{} + err := kCtx.client.Resources().List(ctx, &nodeList) + if err != nil { + return 0, err + } + + totalSchedulableNodes := 0 + + for _, node := range nodeList.Items { + if node.Spec.Unschedulable { + continue + } + + hasNoScheduleTaint := false + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule { + hasNoScheduleTaint = true + break + } + } + + if hasNoScheduleTaint { + continue + } + + totalSchedulableNodes++ + } + + return totalSchedulableNodes, err +} + +// GetAgentResponse extends kibana.GetAgentResponse and includes the EnrolledAt field +type GetAgentResponse struct { + kibana.GetAgentResponse `json:",inline"` + EnrolledAt time.Time `json:"enrolled_at"` +} + +// kibanaGetAgent essentially re-implements kibana.GetAgent to extract also GetAgentResponse.EnrolledAt +func kibanaGetAgent(ctx context.Context, kc *kibana.Client, id string) (*GetAgentResponse, error) { + apiURL := fmt.Sprintf("/api/fleet/agents/%s", id) + r, err := kc.Connection.SendWithContext(ctx, http.MethodGet, apiURL, nil, nil, nil) + if err != nil { + return nil, fmt.Errorf("error calling get agent API: %w", err) + } + defer r.Body.Close() + var agentResp struct { + Item GetAgentResponse `json:"item"` + } + b, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + if r.StatusCode != http.StatusOK { + return nil, fmt.Errorf("error calling get agent API: %s", string(b)) + } + err = json.Unmarshal(b, &agentResp) + if err != nil { + return nil, fmt.Errorf("unmarshalling response json: %w", err) + } + return &agentResp.Item, nil +} + +func queryK8sNamespaceDataStream(dsType, dataset, datastreamNamespace, k8snamespace string) map[string]any { + return map[string]any{ + "_source": []string{"message"}, + "query": map[string]any{ + "bool": map[string]any{ + "filter": []any{ + map[string]any{ + "term": map[string]any{ + "data_stream.dataset": dataset, + }, + }, + map[string]any{ + "term": map[string]any{ + "data_stream.namespace": datastreamNamespace, + }, + }, + map[string]any{ + "term": map[string]any{ + "data_stream.type": dsType, + }, + }, + map[string]any{ + "term": map[string]any{ + "resource.attributes.k8s.namespace.name": k8snamespace, + }, + }, + }, + }, + }, + } +} diff --git a/testing/integration/kubernetes_agent_service_test.go b/testing/integration/k8s/kubernetes_agent_service_test.go similarity index 98% rename from testing/integration/kubernetes_agent_service_test.go rename to testing/integration/k8s/kubernetes_agent_service_test.go index 6c2049c5106..d14f42b8750 100644 --- a/testing/integration/kubernetes_agent_service_test.go +++ b/testing/integration/k8s/kubernetes_agent_service_test.go @@ -4,7 +4,7 @@ //go:build integration -package integration +package k8s import ( "context" diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/k8s/kubernetes_agent_standalone_test.go similarity index 68% rename from testing/integration/kubernetes_agent_standalone_test.go rename to testing/integration/k8s/kubernetes_agent_standalone_test.go index db8191e7655..9d9adeabb86 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/k8s/kubernetes_agent_standalone_test.go @@ -4,66 +4,44 @@ //go:build integration -package integration +package k8s import ( "archive/tar" "bufio" "bytes" "context" - "crypto/sha256" - "encoding/base64" "encoding/json" - "errors" "fmt" "io" "os" "path/filepath" - "regexp" - "strings" "testing" "time" - "github.com/gofrs/uuid/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/elastic-agent-libs/testing/estools" - "github.com/elastic/go-elasticsearch/v8" - - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - cliResource "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/kubernetes" "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/k8s" - "sigs.k8s.io/kustomize/api/krusty" - "sigs.k8s.io/kustomize/kyaml/filesys" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - helmKube "helm.sh/helm/v3/pkg/kube" - aclient "github.com/elastic/elastic-agent/pkg/control/v2/client" - atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" testK8s "github.com/elastic/elastic-agent/pkg/testing/kubernetes" - "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" ) const ( - agentK8SKustomize = "../../deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone" - agentK8SHelm = "../../deploy/helm/elastic-agent" + agentK8SKustomize = "../../../deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone" + agentK8SHelm = "../../../deploy/helm/elastic-agent" ) -var noSpecialCharsRegexp = regexp.MustCompile("[^a-zA-Z0-9]+") - func TestKubernetesAgentStandaloneKustomize(t *testing.T) { info := define.Require(t, define.Requirements{ Stack: &define.Stack{}, @@ -475,88 +453,6 @@ func TestKubernetesAgentHelm(t *testing.T) { } } -// k8sCheckAgentStatus checks that the agent reports healthy. -func k8sCheckAgentStatus(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer, - namespace string, agentPodName string, containerName string, componentPresence map[string]bool, -) error { - command := []string{"elastic-agent", "status", "--output=json"} - stopCheck := errors.New("stop check") - - // we will wait maximum 120 seconds for the agent to report healthy - ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) - defer cancel() - - checkStatus := func() error { - pod := corev1.Pod{} - if err := client.Resources(namespace).Get(ctx, agentPodName, namespace, &pod); err != nil { - return err - } - - for _, container := range pod.Status.ContainerStatuses { - if container.Name != containerName { - continue - } - - if restarts := container.RestartCount; restarts != 0 { - return fmt.Errorf("container %q of pod %q has restarted %d times: %w", containerName, agentPodName, restarts, stopCheck) - } - } - - status := atesting.AgentStatusOutput{} // clear status output - stdout.Reset() - stderr.Reset() - if err := client.Resources().ExecInPod(ctx, namespace, agentPodName, containerName, command, stdout, stderr); err != nil { - return err - } - - if err := json.Unmarshal(stdout.Bytes(), &status); err != nil { - return err - } - - var err error - // validate that the components defined are also healthy if they should exist - for component, shouldBePresent := range componentPresence { - compState, ok := getAgentComponentState(status, component) - if shouldBePresent { - if !ok { - // doesn't exist - err = errors.Join(err, fmt.Errorf("required component %s not found", component)) - } else if compState != int(aclient.Healthy) { - // not healthy - err = errors.Join(err, fmt.Errorf("required component %s is not healthy", component)) - } - } else if ok { - // should not be present - err = errors.Join(err, fmt.Errorf("component %s should not be present", component)) - } - } - return err - } - for { - err := checkStatus() - if err == nil { - return nil - } else if errors.Is(err, stopCheck) { - return err - } - if ctx.Err() != nil { - // timeout waiting for agent to become healthy - return errors.Join(err, errors.New("timeout waiting for agent to become healthy")) - } - time.Sleep(100 * time.Millisecond) - } -} - -// getAgentComponentState returns the component state for the given component name and a bool indicating if it exists. -func getAgentComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { - for _, comp := range status.Components { - if comp.Name == componentName { - return comp.State, true - } - } - return -1, false -} - // k8sDumpPods creates an archive that contains logs of all pods in the given namespace and kube-system to the given target directory func k8sDumpPods(t *testing.T, ctx context.Context, client klient.Client, testName string, namespace string, targetDir string, testStartTime time.Time) { // Create the tar file @@ -703,370 +599,6 @@ func k8sDumpPods(t *testing.T, ctx context.Context, client klient.Client, testNa } } -// k8sKustomizeAdjustObjects adjusts the namespace of given k8s objects and calls the given callbacks for the containers and the pod -func k8sKustomizeAdjustObjects(objects []k8s.Object, namespace string, containerName string, cbContainer func(container *corev1.Container), cbPod func(pod *corev1.PodSpec)) { - // Update the agent image and image pull policy as it is already loaded in kind cluster - for _, obj := range objects { - obj.SetNamespace(namespace) - var podSpec *corev1.PodSpec - switch objWithType := obj.(type) { - case *appsv1.DaemonSet: - podSpec = &objWithType.Spec.Template.Spec - case *appsv1.StatefulSet: - podSpec = &objWithType.Spec.Template.Spec - case *appsv1.Deployment: - podSpec = &objWithType.Spec.Template.Spec - case *appsv1.ReplicaSet: - podSpec = &objWithType.Spec.Template.Spec - case *batchv1.Job: - podSpec = &objWithType.Spec.Template.Spec - case *batchv1.CronJob: - podSpec = &objWithType.Spec.JobTemplate.Spec.Template.Spec - default: - continue - } - - if cbPod != nil { - cbPod(podSpec) - } - - for idx, container := range podSpec.Containers { - if container.Name != containerName { - continue - } - if cbContainer != nil { - cbContainer(&podSpec.Containers[idx]) - } - } - } -} - -// k8sRenderKustomize renders the given kustomize directory to YAML -func k8sRenderKustomize(kustomizePath string) ([]byte, error) { - // Create a file system pointing to the kustomize directory - fSys := filesys.MakeFsOnDisk() - - // Create a kustomizer - k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) - - // Run the kustomizer on the given directory - resMap, err := k.Run(fSys, kustomizePath) - if err != nil { - return nil, err - } - - // Convert the result to YAML - renderedManifest, err := resMap.AsYaml() - if err != nil { - return nil, err - } - - return renderedManifest, nil -} - -// generateESAPIKey generates an API key for the given Elasticsearch. -func generateESAPIKey(esClient *elasticsearch.Client, keyName string) (estools.APIKeyResponse, error) { - return estools.CreateAPIKey(context.Background(), esClient, estools.APIKeyRequest{Name: keyName, Expiration: "1d"}) -} - -// k8sDeleteOpts contains options for deleting k8s objects -type k8sDeleteOpts struct { - // wait for the objects to be deleted - wait bool - // timeout for waiting for the objects to be deleted - waitTimeout time.Duration -} - -// k8sDeleteObjects deletes the given k8s objects and waits for them to be deleted if wait is true. -func k8sDeleteObjects(ctx context.Context, client klient.Client, opts k8sDeleteOpts, objects ...k8s.Object) error { - if len(objects) == 0 { - return nil - } - - // Delete the objects - for _, obj := range objects { - _ = client.Resources(obj.GetNamespace()).Delete(ctx, obj) - } - - if !opts.wait { - // no need to wait - return nil - } - - if opts.waitTimeout == 0 { - // default to 20 seconds - opts.waitTimeout = 20 * time.Second - } - - timeoutCtx, timeoutCancel := context.WithTimeout(ctx, opts.waitTimeout) - defer timeoutCancel() - for _, obj := range objects { - for { - if timeoutCtx.Err() != nil { - return errors.New("timeout waiting for k8s objects to be deleted") - } - - err := client.Resources().Get(timeoutCtx, obj.GetName(), obj.GetNamespace(), obj) - if err != nil { - // object has been deleted - break - } - - time.Sleep(100 * time.Millisecond) - } - } - - return nil -} - -// int64Ptr returns a pointer to the given int64 -func int64Ptr(val int64) *int64 { - valPtr := val - return &valPtr -} - -// k8sCreateOpts contains options for k8sCreateObjects -type k8sCreateOpts struct { - // namespace is the namespace to create the objects in - namespace string - // wait specifies whether to wait for the objects to be ready - wait bool - // waitTimeout is the timeout for waiting for the objects to be ready if wait is true - waitTimeout time.Duration -} - -// k8sCreateObjects creates k8s objects and waits for them to be ready if specified in opts. -// Note that if opts.namespace is not empty, all objects will be created and updated to reference -// the given namespace. -func k8sCreateObjects(ctx context.Context, client klient.Client, opts k8sCreateOpts, objects ...k8s.Object) error { - // Create the objects - for _, obj := range objects { - if opts.namespace != "" { - // update the namespace - obj.SetNamespace(opts.namespace) - - // special case for ClusterRoleBinding and RoleBinding - // update the subjects to reference the given namespace - switch objWithType := obj.(type) { - case *rbacv1.ClusterRoleBinding: - for idx := range objWithType.Subjects { - objWithType.Subjects[idx].Namespace = opts.namespace - } - case *rbacv1.RoleBinding: - for idx := range objWithType.Subjects { - objWithType.Subjects[idx].Namespace = opts.namespace - } - } - } - if err := client.Resources().Create(ctx, obj); err != nil { - return fmt.Errorf("failed to create object %s: %w", obj.GetName(), err) - } - } - - if !opts.wait { - // no need to wait - return nil - } - - if opts.waitTimeout == 0 { - // default to 120 seconds - opts.waitTimeout = 120 * time.Second - } - - return k8sWaitForReady(ctx, client, opts.waitTimeout, objects...) -} - -// k8sWaitForReady waits for the given k8s objects to be ready -func k8sWaitForReady(ctx context.Context, client klient.Client, waitDuration time.Duration, objects ...k8s.Object) error { - // use ready checker from helm kube - clientSet, err := kubernetes.NewForConfig(client.RESTConfig()) - if err != nil { - return fmt.Errorf("error creating clientset: %w", err) - } - readyChecker := helmKube.NewReadyChecker(clientSet, func(s string, i ...interface{}) {}) - - ctxTimeout, cancel := context.WithTimeout(ctx, waitDuration) - defer cancel() - - waitFn := func(ri *cliResource.Info) error { - // here we wait for the k8s object (e.g. deployment, daemonset, pod) to be ready - for { - ready, readyErr := readyChecker.IsReady(ctxTimeout, ri) - if ready { - // k8s object is ready - return nil - } - // k8s object is not ready yet - readyErr = errors.Join(fmt.Errorf("k8s object %s is not ready", ri.Name), readyErr) - - if ctxTimeout.Err() != nil { - // timeout - return errors.Join(fmt.Errorf("timeout waiting for k8s object %s to be ready", ri.Name), readyErr) - } - time.Sleep(100 * time.Millisecond) - } - } - - for _, o := range objects { - // convert k8s.Object to resource.Info for ready checker - runtimeObj, ok := o.(runtime.Object) - if !ok { - return fmt.Errorf("unable to convert k8s.Object %s to runtime.Object", o.GetName()) - } - - if err := waitFn(&cliResource.Info{ - Object: runtimeObj, - Name: o.GetName(), - Namespace: o.GetNamespace(), - }); err != nil { - return err - } - // extract pod label selector for all k8s objects that have underlying pods - oPodsLabelSelector, err := helmKube.SelectorsForObject(runtimeObj) - if err != nil { - // k8s object does not have pods - continue - } - - podList, err := clientSet.CoreV1().Pods(o.GetNamespace()).List(ctx, metav1.ListOptions{ - LabelSelector: oPodsLabelSelector.String(), - }) - if err != nil { - return fmt.Errorf("error listing pods: %w", err) - } - - // here we wait for the all pods to be ready - for _, pod := range podList.Items { - if err := waitFn(&cliResource.Info{ - Object: &pod, - Name: pod.Name, - Namespace: pod.Namespace, - }); err != nil { - return err - } - } - } - - return nil -} - -// k8sContext contains all the information needed to run a k8s test -type k8sContext struct { - client klient.Client - clientSet *kubernetes.Clientset - // logsBasePath is the path that will be used to store the pod logs in a case a test fails - logsBasePath string - // agentImage is the full image of elastic-agent to use in the test - agentImage string - // agentImageRepo is the repository of elastic-agent image to use in the test - agentImageRepo string - // agentImageTag is the tag of elastic-agent image to use in the test - agentImageTag string - // esHost is the host of the elasticsearch to use in the test - esHost string - // esAPIKey is the API key of the elasticsearch to use in the test - esAPIKey string - // esEncodedAPIKey is the encoded API key of the elasticsearch to use in the test - esEncodedAPIKey string - // enrollParams contains the information needed to enroll an agent with Fleet in the test - enrollParams *fleettools.EnrollParams - // createdAt is the time when the k8sContext was created - createdAt time.Time -} - -// getNamespace returns a unique namespace for the current test -func (k k8sContext) getNamespace(t *testing.T) string { - nsUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error generating namespace UUID: %v", err) - } - hasher := sha256.New() - hasher.Write([]byte(nsUUID.String())) - testNamespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil))) - return noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") -} - -func k8sSchedulableNodeCount(ctx context.Context, kCtx k8sContext) (int, error) { - nodeList := corev1.NodeList{} - err := kCtx.client.Resources().List(ctx, &nodeList) - if err != nil { - return 0, err - } - - totalSchedulableNodes := 0 - - for _, node := range nodeList.Items { - if node.Spec.Unschedulable { - continue - } - - hasNoScheduleTaint := false - for _, taint := range node.Spec.Taints { - if taint.Effect == corev1.TaintEffectNoSchedule { - hasNoScheduleTaint = true - break - } - } - - if hasNoScheduleTaint { - continue - } - - totalSchedulableNodes++ - } - - return totalSchedulableNodes, err -} - -// k8sGetContext performs all the necessary checks to get a k8sContext for the current test -func k8sGetContext(t *testing.T, info *define.Info) k8sContext { - agentImage := os.Getenv("AGENT_IMAGE") - require.NotEmpty(t, agentImage, "AGENT_IMAGE must be set") - - agentImageParts := strings.SplitN(agentImage, ":", 2) - require.Len(t, agentImageParts, 2, "AGENT_IMAGE must be in the form ':'") - agentImageRepo := agentImageParts[0] - agentImageTag := agentImageParts[1] - - client, err := info.KubeClient() - require.NoError(t, err) - require.NotNil(t, client) - - clientSet, err := kubernetes.NewForConfig(client.RESTConfig()) - require.NoError(t, err) - require.NotNil(t, clientSet) - - testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") - require.NotEmpty(t, testLogsBasePath, "K8S_TESTS_POD_LOGS_BASE must be set") - - err = os.MkdirAll(testLogsBasePath, 0o755) - require.NoError(t, err, "failed to create test logs directory") - - esHost := os.Getenv("ELASTICSEARCH_HOST") - require.NotEmpty(t, esHost, "ELASTICSEARCH_HOST must be set") - - esAPIKey, err := generateESAPIKey(info.ESClient, info.Namespace) - require.NoError(t, err, "failed to generate ES API key") - require.NotEmpty(t, esAPIKey, "failed to generate ES API key") - - enrollParams, err := fleettools.NewEnrollParams(context.Background(), info.KibanaClient) - require.NoError(t, err, "failed to create fleet enroll params") - - return k8sContext{ - client: client, - clientSet: clientSet, - agentImage: agentImage, - agentImageRepo: agentImageRepo, - agentImageTag: agentImageTag, - logsBasePath: testLogsBasePath, - esHost: esHost, - esAPIKey: esAPIKey.APIKey, - esEncodedAPIKey: esAPIKey.Encoded, - enrollParams: enrollParams, - createdAt: time.Now(), - } -} - // k8sTestStep is a function that performs a single step in a k8s integration test type k8sTestStep func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) diff --git a/testing/integration/k8s/main_test.go b/testing/integration/k8s/main_test.go new file mode 100644 index 00000000000..9fc1e04fd71 --- /dev/null +++ b/testing/integration/k8s/main_test.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package k8s + +import ( + "flag" + "log" + "os" + "testing" + + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +var flagSet = flag.CommandLine + +func init() { + define.RegisterFlags("integration.", flagSet) +} + +func TestMain(m *testing.M) { + define.SetKubernetesSupported() + flag.Parse() + + if define.AutoDiscover { + define.InitAutodiscovery(nil) + } + + runExitCode := m.Run() + + if define.AutoDiscover { + discoveredTests, err := define.DumpAutodiscoveryYAML() + if err != nil { + log.Fatalf("Error dumping autodiscovery YAML: %v\n", err) + } + + err = os.WriteFile(define.AutoDiscoveryOutput, discoveredTests, 0644) + if err != nil { + log.Fatalf("Error writing autodiscovery data in %q: %v\n", define.AutoDiscoveryOutput, err) + } + } + + os.Exit(runExitCode) +} diff --git a/testing/integration/otel_helm_test.go b/testing/integration/k8s/otel_helm_test.go similarity index 90% rename from testing/integration/otel_helm_test.go rename to testing/integration/k8s/otel_helm_test.go index 89d4853eacf..ac15f525e71 100644 --- a/testing/integration/otel_helm_test.go +++ b/testing/integration/k8s/otel_helm_test.go @@ -4,7 +4,7 @@ //go:build integration -package integration +package k8s import ( "bufio" @@ -72,7 +72,7 @@ func TestOtelKubeStackHelm(t *testing.T) { k8sStepCreateNamespace(), k8sStepHelmDeployWithValueOptions(chartLocation, "kube-stack-otel", values.Options{ - ValueFiles: []string{"../../deploy/helm/edot-collector/kube-stack/values.yaml"}, + ValueFiles: []string{"../../../deploy/helm/edot-collector/kube-stack/values.yaml"}, Values: []string{ fmt.Sprintf("defaultCRConfig.image.repository=%s", kCtx.agentImageRepo), fmt.Sprintf("defaultCRConfig.image.tag=%s", kCtx.agentImageTag), @@ -114,7 +114,7 @@ func TestOtelKubeStackHelm(t *testing.T) { k8sStepCreateNamespace(), k8sStepHelmDeployWithValueOptions(chartLocation, "kube-stack-otel", values.Options{ - ValueFiles: []string{"../../deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml"}, + ValueFiles: []string{"../../../deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml"}, Values: []string{fmt.Sprintf("defaultCRConfig.image.repository=%s", kCtx.agentImageRepo), fmt.Sprintf("defaultCRConfig.image.tag=%s", kCtx.agentImageTag)}, // override secrets reference with env variables @@ -224,35 +224,3 @@ func k8sStepCheckDatastreamsHits(info *define.Info, dsType, dataset, datastreamN }, 5*time.Minute, 10*time.Second, fmt.Sprintf("at least one document should be available for %s datastream", fmt.Sprintf("%s-%s-%s", dsType, dataset, datastreamNamespace))) } } - -func queryK8sNamespaceDataStream(dsType, dataset, datastreamNamespace, k8snamespace string) map[string]any { - return map[string]any{ - "_source": []string{"message"}, - "query": map[string]any{ - "bool": map[string]any{ - "filter": []any{ - map[string]any{ - "term": map[string]any{ - "data_stream.dataset": dataset, - }, - }, - map[string]any{ - "term": map[string]any{ - "data_stream.namespace": datastreamNamespace, - }, - }, - map[string]any{ - "term": map[string]any{ - "data_stream.type": dsType, - }, - }, - map[string]any{ - "term": map[string]any{ - "resource.attributes.k8s.namespace.name": k8snamespace, - }, - }, - }, - }, - }, - } -} diff --git a/testing/integration/testdata/connectors.agent.yml b/testing/integration/k8s/testdata/connectors.agent.yml similarity index 100% rename from testing/integration/testdata/connectors.agent.yml rename to testing/integration/k8s/testdata/connectors.agent.yml diff --git a/testing/integration/testdata/java_app.yaml b/testing/integration/k8s/testdata/java_app.yaml similarity index 100% rename from testing/integration/testdata/java_app.yaml rename to testing/integration/k8s/testdata/java_app.yaml diff --git a/testing/integration/testdata/k8s.hints.redis.yaml b/testing/integration/k8s/testdata/k8s.hints.redis.yaml similarity index 100% rename from testing/integration/testdata/k8s.hints.redis.yaml rename to testing/integration/k8s/testdata/k8s.hints.redis.yaml diff --git a/testing/integration/logs_ingestion.go b/testing/integration/logs_ingestion.go index 89046e70092..06ffadec9a1 100644 --- a/testing/integration/logs_ingestion.go +++ b/testing/integration/logs_ingestion.go @@ -170,7 +170,7 @@ func testMonitoringLogsAreShipped( ) { // Stage 1: Make sure metricbeat logs are populated t.Log("Making sure metricbeat logs are populated") - docs := findESDocs(t, func() (estools.Documents, error) { + docs := FindESDocs(t, func() (estools.Documents, error) { return estools.GetLogsForDataset(ctx, info.ESClient, "elastic_agent.metricbeat") }) t.Logf("metricbeat: Got %d documents", len(docs.Hits.Hits)) @@ -228,7 +228,7 @@ func testMonitoringLogsAreShipped( // Stage 3: Make sure we have message confirming central management is running t.Log("Making sure we have message confirming central management is running") - docs = findESDocs(t, func() (estools.Documents, error) { + docs = FindESDocs(t, func() (estools.Documents, error) { return estools.FindMatchingLogLines(ctx, info.ESClient, info.Namespace, "Parsed configuration and determined agent is managed by Fleet") }) @@ -249,7 +249,7 @@ func testMonitoringLogsAreShipped( // this field is not mapped. There is an issue for that: // https://github.com/elastic/integrations/issues/6545 // TODO: use runtime fields while the above issue is not resolved. - docs = findESDocs(t, func() (estools.Documents, error) { + docs = FindESDocs(t, func() (estools.Documents, error) { return estools.GetLogsForAgentID(ctx, info.ESClient, agentID) }) require.NoError(t, err, "could not get logs from Agent ID: %q, err: %s", @@ -299,28 +299,6 @@ func queryESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools return docs } -// findESDocs runs `findFn` until at least one document is returned and there is no error -func findESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents { - var docs estools.Documents - require.Eventually( - t, - func() bool { - var err error - docs, err = findFn() - if err != nil { - t.Logf("got an error querying ES, retrying. Error: %s", err) - return false - } - - return docs.Hits.Total.Value != 0 - }, - 3*time.Minute, - 15*time.Second, - ) - - return docs -} - func testFlattenedDatastreamFleetPolicy( t *testing.T, ctx context.Context, diff --git a/testing/integration/otel_test.go b/testing/integration/otel_test.go index 0ca1c70d1c5..e5b00234e70 100644 --- a/testing/integration/otel_test.go +++ b/testing/integration/otel_test.go @@ -11,7 +11,6 @@ import ( "context" "errors" "fmt" - "net/url" "os" "path/filepath" "strings" @@ -314,7 +313,7 @@ func TestOtelLogsIngestion(t *testing.T) { tempDir := t.TempDir() inputFilePath := filepath.Join(tempDir, "input.log") - esHost, err := getESHost() + esHost, err := GetESHost() require.NoError(t, err, "failed to get ES host") require.True(t, len(esHost) > 0) @@ -433,7 +432,7 @@ func TestOtelAPMIngestion(t *testing.T) { require.NoError(t, err) // start apm default config just configure ES output - esHost, err := getESHost() + esHost, err := GetESHost() require.NoError(t, err, "failed to get ES host") require.True(t, len(esHost) > 0) @@ -549,18 +548,6 @@ func TestOtelAPMIngestion(t *testing.T) { apmFixtureWg.Wait() } -func getESHost() (string, error) { - fixedESHost := os.Getenv("ELASTICSEARCH_HOST") - parsedES, err := url.Parse(fixedESHost) - if err != nil { - return "", err - } - if parsedES.Port() == "" { - fixedESHost = fmt.Sprintf("%s:443", fixedESHost) - } - return fixedESHost, nil -} - func createESApiKey(esClient *elasticsearch.Client) (estools.APIKeyResponse, error) { return estools.CreateAPIKey(context.Background(), esClient, estools.APIKeyRequest{Name: "test-api-key", Expiration: "1d"}) } @@ -741,7 +728,7 @@ func TestOtelFBReceiverE2E(t *testing.T) { Index string MinItems int } - esEndpoint, err := getESHost() + esEndpoint, err := GetESHost() require.NoError(t, err, "error getting elasticsearch endpoint") esApiKey, err := createESApiKey(info.ESClient) require.NoError(t, err, "error creating API key")