Merge branch 'main' into update-test-matrix

Author: biscout42

.github/workflows/test.yml

@@ -18,8 +18,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version-file: 'go.mod'
           cache: true
@@ -34,8 +34,8 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version-file: 'tools/go.mod'
           cache: true
@@ -125,8 +125,8 @@ jobs:
           - '8.14.3'
           - '8.15.3'
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version-file: 'go.mod'
           cache: true

CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+- Support multiple group by fields in SLOs ([#870](https://github.com/elastic/terraform-provider-elasticstack/pull/878))
+- Use the auto-generated OAS schema from elastic/kibana for the Fleet API. ([#834](https://github.com/elastic/terraform-provider-elasticstack/issues/834))
+- Support description in `elasticstack_elasticsearch_security_role` data sources. ([#884](https://github.com/elastic/terraform-provider-elasticstack/pull/884))
+- Prevent spurious recreation of `elasticstack_fleet_agent_policy` resources due to 'changing' policy ids ([#885](https://github.com/elastic/terraform-provider-elasticstack/pull/885))
+- Support `elasticstack_kibana_alerting_rule` resources with only one of `kql` or `timeframe` attributes set ([#886](https://github.com/elastic/terraform-provider-elasticstack/pull/886))
+
+## [0.11.11] - 2024-10-25
+
+- Allow `elasticstack_kibana_alerting_rule` to be used without Elasticsearch being configured. ([#869](https://github.com/elastic/terraform-provider-elasticstack/pull/869))
+- Add resource `elasticstack_elasticsearch_data_stream_lifecycle` ([#838](https://github.com/elastic/terraform-provider-elasticstack/issues/838))
+- Ensure API keys are not replaced when upgrading from 0.11.9 or earlier. ([#875](https://github.com/elastic/terraform-provider-elasticstack/pull/875))
+
 ## [0.11.10] - 2024-10-23
 
 - Fix bug updating alert delay ([#859](https://github.com/elastic/terraform-provider-elasticstack/pull/859))
@@ -376,9 +388,10 @@
 - Initial set of docs
 - CI integration
 
-[Unreleased]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.10...HEAD
-[0.11.9]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.9...v0.11.10
-[0.11.8]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.8...v0.11.9
+[Unreleased]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.11...HEAD
+[0.11.11]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.9...v0.11.11
+[0.11.10]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.9...v0.11.10
+[0.11.9]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.8...v0.11.9
 [0.11.8]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.7...v0.11.8
 [0.11.7]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.6...v0.11.7
 [0.11.6]: https://github.com/elastic/terraform-provider-elasticstack/compare/v0.11.5...v0.11.6

Makefile

@@ -2,7 +2,7 @@
 SHELL := /bin/bash
 
 
-VERSION ?= 0.11.10
+VERSION ?= 0.11.11
 
 NAME = elasticstack
 BINARY = terraform-provider-${NAME}
@@ -35,7 +35,6 @@ FLEET_NAME ?= terraform-elasticstack-fleet
 FLEET_ENDPOINT ?= https://$(FLEET_NAME):8220
 
 SOURCE_LOCATION ?= $(shell pwd)
-, := ,
 
 export GOBIN = $(shell pwd)/bin
 
@@ -73,6 +72,11 @@ retry = until [ $$(if [ -z "$$attempt" ]; then echo -n "0"; else echo -n "$$atte
 		backoff=$$((backoff * 2)); \
 	done
 
+# wait_until_healthy command - first argument is the container name
+wait_until_healthy = $(call retry, 5, [ "$$(docker inspect -f '{{ .State.Health.Status }}' $(1))" == "healthy" ])
+
+CURL_OPTS = -sS --retry 5 --retry-all-errors -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json"
+
 # To run specific test (e.g. TestAccResourceActionConnector) execute `make docker-testacc TESTARGS='-run ^TestAccResourceActionConnector$$'`
 # To enable tracing (or debugging), execute `make docker-testacc TF_LOG=TRACE`
 .PHONY: docker-testacc
@@ -107,8 +111,7 @@ docker-testacc-with-token:
 .PHONY: docker-elasticsearch
 docker-elasticsearch: docker-network ## Start Elasticsearch single node cluster in docker container
 	@ docker rm -f $(ELASTICSEARCH_NAME) &> /dev/null || true
-	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(ELASTICSEARCH_NAME) > /dev/null 2>&1 ; then \
-		docker run -d \
+	@ docker run -d \
 		--memory $(ELASTICSEARCH_MEM) \
 		-p 9200:9200 -p 9300:9300 \
 		-e "discovery.type=single-node" \
@@ -122,14 +125,15 @@ docker-elasticsearch: docker-network ## Start Elasticsearch single node cluster
 		-e ELASTIC_PASSWORD=$(ELASTICSEARCH_PASSWORD) \
 		--name $(ELASTICSEARCH_NAME) \
 		--network $(ELASTICSEARCH_NETWORK) \
-		docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION); \
-		fi)
+		--health-cmd="curl http://localhost:9200/_cluster/health" \
+		--health-interval=10s --health-timeout=5s --health-retries=10 \
+		docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION)
+	@ $(call wait_until_healthy, $(ELASTICSEARCH_NAME))
 
 .PHONY: docker-kibana
 docker-kibana: docker-network docker-elasticsearch set-kibana-password ## Start Kibana node in docker container
 	@ docker rm -f $(KIBANA_NAME)  &> /dev/null || true
-	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(KIBANA_NAME) > /dev/null 2>&1 ; then \
-		docker run -d \
+	@ docker run -d \
 		-p 5601:5601 \
 		-e SERVER_NAME=kibana \
 		-e ELASTICSEARCH_HOSTS=$(ELASTICSEARCH_ENDPOINTS) \
@@ -139,18 +143,18 @@ docker-kibana: docker-network docker-elasticsearch set-kibana-password ## Start
 		-e LOGGING_ROOT_LEVEL=debug \
 		--name $(KIBANA_NAME) \
 		--network $(ELASTICSEARCH_NETWORK) \
-		docker.elastic.co/kibana/kibana:$(STACK_VERSION); \
-		fi)
+		--health-cmd="curl http://localhost:5601/api/status" \
+		--health-interval=10s --health-timeout=5s --health-retries=10 \
+		docker.elastic.co/kibana/kibana:$(STACK_VERSION)
+	@ $(call wait_until_healthy, $(KIBANA_NAME))
 
 .PHONY: docker-kibana-with-tls
 docker-kibana-with-tls: docker-network docker-elasticsearch set-kibana-password
 	@ docker rm -f $(KIBANA_NAME)  &> /dev/null || true
 	@ mkdir -p certs
 	@ CAROOT=certs mkcert localhost $(KIBANA_NAME)
 	@ mv localhost*.pem certs/
-
-	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(KIBANA_NAME) > /dev/null 2>&1 ; then \
-		docker run -d \
+	@ docker run -d \
 		-p 5601:5601 \
 		-v $(shell pwd)/certs:/certs \
 		-e SERVER_NAME=kibana \
@@ -164,14 +168,15 @@ docker-kibana-with-tls: docker-network docker-elasticsearch set-kibana-password
 		-e LOGGING_ROOT_LEVEL=debug \
 		--name $(KIBANA_NAME) \
 		--network $(ELASTICSEARCH_NETWORK) \
-		docker.elastic.co/kibana/kibana:$(STACK_VERSION); \
-		fi)
+		--health-cmd="curl -k https://localhost:5601/api/status" \
+		--health-interval=10s --health-timeout=5s --health-retries=10 \
+		docker.elastic.co/kibana/kibana:$(STACK_VERSION)
+	@ $(call wait_until_healthy, $(KIBANA_NAME))
 
 .PHONY: docker-fleet
 docker-fleet: docker-network docker-elasticsearch docker-kibana setup-kibana-fleet ## Start Fleet node in docker container
 	@ docker rm -f $(FLEET_NAME)  &> /dev/null || true
-	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(FLEET_NAME) > /dev/null 2>&1 ; then \
-		docker run -d \
+	@ docker run -d \
 		-p 8220:8220 \
 		-e SERVER_NAME=fleet \
       	-e FLEET_ENROLL=1 \
@@ -188,33 +193,30 @@ docker-fleet: docker-network docker-elasticsearch docker-kibana setup-kibana-fle
       	-e KIBANA_FLEET_PASSWORD=$(ELASTICSEARCH_PASSWORD) \
 		--name $(FLEET_NAME) \
 		--network $(ELASTICSEARCH_NETWORK) \
-		docker.elastic.co/beats/elastic-agent:$(STACK_VERSION); \
-		fi)
+		docker.elastic.co/beats/elastic-agent:$(STACK_VERSION)
 
 
 .PHONY: docker-network
 docker-network: ## Create a dedicated network for ES and test runs
-	@ if ! docker network ls --format '{{.Name}}' | grep -w $(ELASTICSEARCH_NETWORK) > /dev/null 2>&1 ; then \
-		docker network create $(ELASTICSEARCH_NETWORK); \
-		fi
+	@ docker network inspect $(ELASTICSEARCH_NETWORK) >/dev/null 2>&1 || docker network create $(ELASTICSEARCH_NETWORK)
 
 .PHONY: set-kibana-password
 set-kibana-password: ## Sets the ES KIBANA_SYSTEM_USERNAME's password to KIBANA_SYSTEM_PASSWORD. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d '{"password":"$(KIBANA_SYSTEM_PASSWORD)"}' | grep -q "^{}")
+	@ curl $(CURL_OPTS) http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d '{"password":"$(KIBANA_SYSTEM_PASSWORD)"}'
 
 .PHONY: create-es-api-key
 create-es-api-key: ## Creates and outputs a new API Key. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/api_key -d '{"name":"$(KIBANA_API_KEY_NAME)"}')
+	@ curl $(CURL_OPTS) http://localhost:9200/_security/api_key -d '{"name":"$(KIBANA_API_KEY_NAME)"}'
 
 .PHONY: create-es-bearer-token
 create-es-bearer-token: ## Creates and outputs a new OAuth bearer token. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/oauth2/token -d '{"grant_type":"client_credentials"}')
+	@ curl $(CURL_OPTS) http://localhost:9200/_security/oauth2/token -d '{"grant_type":"client_credentials"}'
 
 .PHONY: setup-kibana-fleet
 setup-kibana-fleet: ## Creates the agent and integration policies required to run Fleet. This expects Kibana to be available at localhost:5601
-	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/fleet_server_hosts -d '{"name":"default"$(,)"host_urls":["$(FLEET_ENDPOINT)"]$(,)"is_default":true}')
-	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/agent_policies -d '{"id":"fleet-server"$(,)"name":"Fleet Server"$(,)"namespace":"default"$(,)"monitoring_enabled":["logs"$(,)"metrics"]}')
-	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/package_policies -d '{"name":"fleet-server"$(,)"namespace":"default"$(,)"policy_id":"fleet-server"$(,)"enabled":true$(,)"inputs":[{"type":"fleet-server"$(,)"enabled":true$(,)"streams":[]$(,)"vars":{}}]$(,)"package":{"name":"fleet_server"$(,)"version":"1.5.0"}}')
+	curl $(CURL_OPTS) -H "kbn-xsrf: true" http://localhost:5601/api/fleet/fleet_server_hosts -d '{"name":"default","host_urls":["$(FLEET_ENDPOINT)"],"is_default":true}'
+	curl $(CURL_OPTS) -H "kbn-xsrf: true" http://localhost:5601/api/fleet/agent_policies -d '{"id":"fleet-server","name":"Fleet Server","namespace":"default","monitoring_enabled":["logs","metrics"]}'
+	curl $(CURL_OPTS) -H "kbn-xsrf: true" http://localhost:5601/api/fleet/package_policies -d '{"name":"fleet-server","namespace":"default","policy_id":"fleet-server","enabled":true,"inputs":[{"type":"fleet-server","enabled":true,"streams":[],"vars":{}}],"package":{"name":"fleet_server","version":"1.5.0"}}'
 
 .PHONY: docker-clean
 docker-clean: ## Try to remove provisioned nodes and assigned network

docs/data-sources/elasticsearch_security_role.md

@@ -42,6 +42,7 @@ output "role" {
 
 - `applications` (Set of Object) A list of application privilege entries. (see [below for nested schema](#nestedatt--applications))
 - `cluster` (Set of String) A list of cluster privileges. These privileges define the cluster level actions that users with this role are able to execute.
+- `description` (String) The description of the role.
 - `global` (String) An object defining global privileges.
 - `id` (String) Internal identifier of the resource
 - `indices` (Set of Object) A list of indices permissions entries. (see [below for nested schema](#nestedatt--indices))

docs/resources/elasticsearch_data_stream_lifecycle.md

@@ -0,0 +1,109 @@
+---
+subcategory: "Index"
+layout: ""
+page_title: "Elasticstack: elasticstack_elasticsearch_data_stream_lifecycle Resource"
+description: |-
+  Manages Lifecycle for Elasticsearch Data Streams
+---
+
+# Resource: elasticstack_elasticsearch_data_stream
+
+Configures the data stream lifecycle for the targeted data streams, see: https://www.elastic.co/guide/en/elasticsearch/reference/current/data-stream-apis.html
+
+## Example Usage
+
+```terraform
+provider "elasticstack" {
+  elasticsearch {}
+}
+
+// First we must have a index template created
+resource "elasticstack_elasticsearch_index_template" "my_data_stream_template" {
+  name = "my_data_stream"
+
+  index_patterns = ["my-stream*"]
+
+  data_stream {}
+}
+
+// and now we can create data stream based on the index template
+resource "elasticstack_elasticsearch_data_stream" "my_data_stream" {
+  name = "my-stream"
+
+  // make sure that template is created before the data stream
+  depends_on = [
+    elasticstack_elasticsearch_index_template.my_data_stream_template
+  ]
+}
+
+// finally we can manage lifecycle of data stream
+resource "elasticstack_elasticsearch_data_stream_lifecycle" "my_data_stream_lifecycle" {
+  name           = "my-stream"
+  data_retention = "3d"
+
+  depends_on = [
+    elasticstack_elasticsearch_data_stream.my_data_stream,
+  ]
+}
+
+// or you can use wildcards to manage multiple lifecycles at once
+resource "elasticstack_elasticsearch_data_stream_lifecycle" "my_data_stream_lifecycle_multiple" {
+  name           = "stream-*"
+  data_retention = "3d"
+}
+```
+
+<!-- schema generated by tfplugindocs -->
+## Schema
+
+### Required
+
+- `name` (String) Name of the data stream. Supports wildcards.
+
+### Optional
+
+- `data_retention` (String) Every document added to this data stream will be stored at least for this time frame. When empty, every document in this data stream will be stored indefinitely
+- `downsampling` (Attributes List) Downsampling configuration objects, each defining an after interval representing when the backing index is meant to be downsampled and a fixed_interval representing the downsampling interval. (see [below for nested schema](#nestedatt--downsampling))
+- `elasticsearch_connection` (Block List, Deprecated) Elasticsearch connection configuration block. (see [below for nested schema](#nestedblock--elasticsearch_connection))
+- `enabled` (Boolean) Data stream lifecycle on/off.
+- `expand_wildcards` (String) Determines how wildcard patterns in the `indices` parameter match data streams and indices. Supports comma-separated values, such as `closed,hidden`.
+
+### Read-Only
+
+- `id` (String) Internal identifier of the resource.
+
+<a id="nestedatt--downsampling"></a>
+### Nested Schema for `downsampling`
+
+Required:
+
+- `after` (String) Interval representing when the backing index is meant to be downsampled
+- `fixed_interval` (String) The interval at which to aggregate the original time series index.
+
+
+<a id="nestedblock--elasticsearch_connection"></a>
+### Nested Schema for `elasticsearch_connection`
+
+Optional:
+
+- `api_key` (String, Sensitive) API Key to use for authentication to Elasticsearch
+- `bearer_token` (String, Sensitive) Bearer Token to use for authentication to Elasticsearch
+- `ca_data` (String) PEM-encoded custom Certificate Authority certificate
+- `ca_file` (String) Path to a custom Certificate Authority certificate
+- `cert_data` (String) PEM encoded certificate for client auth
+- `cert_file` (String) Path to a file containing the PEM encoded certificate for client auth
+- `endpoints` (List of String, Sensitive) A list of endpoints where the terraform provider will point to, this must include the http(s) schema and port number.
+- `es_client_authentication` (String, Sensitive) ES Client Authentication field to be used with the bearer token
+- `insecure` (Boolean) Disable TLS certificate validation
+- `key_data` (String, Sensitive) PEM encoded private key for client auth
+- `key_file` (String) Path to a file containing the PEM encoded private key for client auth
+- `password` (String, Sensitive) Password to use for API authentication to Elasticsearch.
+- `username` (String) Username to use for API authentication to Elasticsearch.
+
+## Import
+
+Import is supported using the following syntax:
+
+```shell
+terraform import elasticstack_elasticsearch_data_stream_lifecycle.my_data_stream_lifecycle <cluster_uuid>/<data_stream_name>
+```

docs/resources/elasticsearch_security_user.md

@@ -99,5 +99,5 @@ Optional:
 Import is supported using the following syntax:
 
 ```shell
-terraform import elasticstack_elasticsearch_security_user.user <cluster_uuid>/elastic
+terraform import elasticstack_elasticsearch_security_user.user <cluster_uuid>/<user-name>
 ```

docs/resources/kibana_slo.md

@@ -208,7 +208,7 @@ resource "elasticstack_kibana_slo" "custom_metric" {
 
 - `apm_availability_indicator` (Block List, Max: 1) (see [below for nested schema](#nestedblock--apm_availability_indicator))
 - `apm_latency_indicator` (Block List, Max: 1) (see [below for nested schema](#nestedblock--apm_latency_indicator))
-- `group_by` (String) Optional group by field to use to generate an SLO per distinct value.
+- `group_by` (List of String) Optional group by fields to use to generate an SLO per distinct value.
 - `histogram_custom_indicator` (Block List, Max: 1) (see [below for nested schema](#nestedblock--histogram_custom_indicator))
 - `kql_custom_indicator` (Block List, Max: 1) (see [below for nested schema](#nestedblock--kql_custom_indicator))
 - `metric_custom_indicator` (Block List, Max: 1) (see [below for nested schema](#nestedblock--metric_custom_indicator))

examples/resources/elasticstack_elasticsearch_data_stream_lifecycle/import.sh

@@ -0,0 +1,2 @@
+terraform import elasticstack_elasticsearch_data_stream_lifecycle.my_data_stream_lifecycle <cluster_uuid>/<data_stream_name>
+

examples/resources/elasticstack_elasticsearch_data_stream_lifecycle/resource.tf

@@ -0,0 +1,38 @@
+provider "elasticstack" {
+  elasticsearch {}
+}
+
+// First we must have a index template created
+resource "elasticstack_elasticsearch_index_template" "my_data_stream_template" {
+  name = "my_data_stream"
+
+  index_patterns = ["my-stream*"]
+
+  data_stream {}
+}
+
+// and now we can create data stream based on the index template
+resource "elasticstack_elasticsearch_data_stream" "my_data_stream" {
+  name = "my-stream"
+
+  // make sure that template is created before the data stream
+  depends_on = [
+    elasticstack_elasticsearch_index_template.my_data_stream_template
+  ]
+}
+
+// finally we can manage lifecycle of data stream
+resource "elasticstack_elasticsearch_data_stream_lifecycle" "my_data_stream_lifecycle" {
+  name           = "my-stream"
+  data_retention = "3d"
+
+  depends_on = [
+    elasticstack_elasticsearch_data_stream.my_data_stream,
+  ]
+}
+
+// or you can use wildcards to manage multiple lifecycles at once
+resource "elasticstack_elasticsearch_data_stream_lifecycle" "my_data_stream_lifecycle_multiple" {
+  name           = "stream-*"
+  data_retention = "3d"
+}

examples/resources/elasticstack_elasticsearch_security_user/import.sh

@@ -1 +1 @@
-terraform import elasticstack_elasticsearch_security_user.user <cluster_uuid>/elastic
+terraform import elasticstack_elasticsearch_security_user.user <cluster_uuid>/<user-name>