Skip to content

Commit 4d612ff

Browse files
Ak-skyAk-sky
authored
feat: Update schema variable to support full Apache Avro specification (#526)
* first commit * feat: Update schemas variable to support full Apache Avro specification * feat: Update schemas variable to support full Apache Avro specification * docs: updated the type file * addressed feedback * addressed feedback * addressed feedback
1 parent d98125d commit 4d612ff

File tree

9 files changed

+153
-110
lines changed

9 files changed

+153
-110
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ You need the following permissions to run this module.
161161
| <a name="input_resource_group_id"></a> [resource\_group\_id](#input\_resource\_group\_id) | The resource group ID where the Event Streams instance is created. | `string` | n/a | yes |
162162
| <a name="input_resource_keys"></a> [resource\_keys](#input\_resource\_keys) | A list of service credential resource keys to be created for the Event Streams instance. | <pre>list(object({<br/> name = string<br/> key_name = optional(string, null)<br/> role = optional(string, "Manager")<br/> endpoint = optional(string, "public")<br/> }))</pre> | `[]` | no |
163163
| <a name="input_schema_global_rule"></a> [schema\_global\_rule](#input\_schema\_global\_rule) | Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL\_TRANSITIVE', 'FORWARD', 'FORWARD\_TRANSITIVE', 'BACKWARD', 'BACKWARD\_TRANSITIVE'. | `string` | `null` | no |
164-
| <a name="input_schemas"></a> [schemas](#input\_schemas) | The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object. | <pre>list(object(<br/> {<br/> schema_id = string<br/> schema = object({<br/> type = string<br/> name = string<br/> fields = optional(list(object({<br/> name = string<br/> type = string<br/> })))<br/> })<br/> }<br/> ))</pre> | `[]` | no |
164+
| <a name="input_schemas"></a> [schemas](#input\_schemas) | List of schema objects. Each schema must include `schema_id` and `schema` definition. Supports full Apache Avro specification with nested structures. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-ES_schema_registry#ES_apache_avro_data_format). | `any` | `[]` | no |
165165
| <a name="input_service_endpoints"></a> [service\_endpoints](#input\_service\_endpoints) | The type of service endpoints. Possible values: 'public', 'private', 'public-and-private'. | `string` | `"public"` | no |
166166
| <a name="input_skip_es_s2s_iam_authorization_policy"></a> [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that will allow all Event Streams instances in the given resource group access to read from the mirror source instance. This policy is required when creating a mirroring instance, and will only be created if a value is passed in the mirroring input. | `bool` | `false` | no |
167167
| <a name="input_skip_kms_iam_authorization_policy"></a> [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `kms_key_crn` variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no |

‎examples/complete/main.tf‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,8 @@ module "event_streams" {
1818
source = "../../"
1919
resource_group_id = module.resource_group.resource_group_id
2020
es_name = "${var.prefix}-es"
21-
schemas = [
22-
{
23-
schema_id = "my-es-schema_1"
24-
schema = {
25-
type = "string"
26-
name = "name_1"
27-
}
28-
},
29-
{
30-
schema_id = "my-es-schema_2"
31-
schema = {
32-
type = "string"
33-
name = "name_2"
34-
}
35-
}
36-
]
37-
tags = var.resource_tags
38-
access_tags = var.access_tags
21+
tags = var.resource_tags
22+
access_tags = var.access_tags
3923
topics = [
4024
{
4125
name = "topic-1"

‎examples/fscloud/main.tf‎

Lines changed: 75 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,81 @@ module "cbr_zone_schematics" {
7070
# #############################################################################
7171

7272
module "event_streams" {
73-
source = "../../modules/fscloud"
74-
resource_group_id = module.resource_group.resource_group_id
75-
es_name = "${var.prefix}-es-fs"
76-
kms_key_crn = var.kms_key_crn
77-
schemas = var.schemas
78-
tags = var.resource_tags
79-
topics = var.topics
80-
create_timeout = "6h"
81-
metrics = ["topic", "partition", "consumers"]
73+
source = "../../modules/fscloud"
74+
resource_group_id = module.resource_group.resource_group_id
75+
es_name = "${var.prefix}-es-fs"
76+
kms_key_crn = var.kms_key_crn
77+
tags = var.resource_tags
78+
create_timeout = "6h"
79+
metrics = ["topic", "partition", "consumers"]
80+
schemas = [
81+
{
82+
schema_id = "job_events_cloud_sync_value_v1"
83+
schema = {
84+
type = "record"
85+
name = "job_events_cloud_sync_value_v1"
86+
namespace = "envelope"
87+
fields = [
88+
{ name = "source", type = "string" },
89+
{ name = "subject", type = "string" },
90+
{ name = "time", type = "string" },
91+
{ name = "datacontenttype", type = "string", default = "application/json" },
92+
{ name = "producer", type = "string" },
93+
{
94+
name = "data"
95+
type = {
96+
type = "record"
97+
name = "payload"
98+
namespace = "payload"
99+
fields = [
100+
{ name = "event_type", type = "string" },
101+
{ name = "job_id", type = "string" },
102+
{ name = "metadata", type = ["null", "string"], default = null },
103+
{ name = "monotonic_ns", type = "long" },
104+
{ name = "source_instance_id", type = "string" },
105+
{ name = "source_type_id", type = "string" },
106+
{ name = "sub_job_id", type = ["null", "string"], default = null }
107+
]
108+
}
109+
}
110+
]
111+
}
112+
},
113+
{
114+
schema_id = "my-es-schema_1"
115+
schema = {
116+
type = "record"
117+
name = "book"
118+
fields = [
119+
{
120+
name = "title"
121+
type = "string"
122+
},
123+
{
124+
name = "author"
125+
type = "string"
126+
}
127+
]
128+
}
129+
},
130+
{
131+
schema_id = "my-es-schema_2"
132+
schema = {
133+
type = "record"
134+
name = "book"
135+
fields = [
136+
{
137+
name = "author"
138+
type = "string"
139+
},
140+
{
141+
name = "title"
142+
type = "string"
143+
}
144+
]
145+
}
146+
}
147+
]
82148
mirroring_topic_patterns = ["topic-1", "topic-2"]
83149
mirroring = {
84150
source_crn = var.event_streams_source_crn # Required for mirroring

‎examples/fscloud/variables.tf‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,32 +28,6 @@ variable "resource_tags" {
2828
default = []
2929
}
3030

31-
variable "schemas" {
32-
type = list(object(
33-
{
34-
schema_id = string
35-
schema = object({
36-
type = string
37-
name = string
38-
})
39-
}
40-
))
41-
description = "The list of schema object which contains schema id and format of the schema"
42-
default = []
43-
}
44-
45-
variable "topics" {
46-
type = list(object(
47-
{
48-
name = string
49-
partitions = number
50-
config = map(string)
51-
}
52-
))
53-
description = "List of topics. For lite plan only one topic is allowed."
54-
default = []
55-
}
56-
5731
variable "kms_key_crn" {
5832
type = string
5933
description = "The root key CRN of a Hyper Protect Crypto Service (HPCS) that you want to use for disk encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs&interface=ui for more information on integrating HPCS with Event Streams instance."

‎modules/fscloud/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ No resources.
4141
| <a name="input_resource_group_id"></a> [resource\_group\_id](#input\_resource\_group\_id) | The resource group ID where the Event Streams instance is created. | `string` | n/a | yes |
4242
| <a name="input_resource_keys"></a> [resource\_keys](#input\_resource\_keys) | A list of service credential resource keys to be created for the Event Streams instance. | <pre>list(object({<br/> name = string<br/> role = optional(string, "Reader")<br/> endpoint = optional(string, "private")<br/> }))</pre> | `[]` | no |
4343
| <a name="input_schema_global_rule"></a> [schema\_global\_rule](#input\_schema\_global\_rule) | Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL\_TRANSITIVE', 'FORWARD', 'FORWARD\_TRANSITIVE', 'BACKWARD', 'BACKWARD\_TRANSITIVE'. | `string` | `null` | no |
44-
| <a name="input_schemas"></a> [schemas](#input\_schemas) | The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object. | <pre>list(object(<br/> {<br/> schema_id = string<br/> schema = object({<br/> type = string<br/> name = string<br/> fields = optional(list(object({<br/> name = string<br/> type = string<br/> })))<br/> })<br/> }<br/> ))</pre> | `[]` | no |
44+
| <a name="input_schemas"></a> [schemas](#input\_schemas) | List of schema objects. Each schema must include `schema_id` and `schema` definition. Supports full Apache Avro specification with nested structures. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-ES_schema_registry#ES_apache_avro_data_format). | `any` | `[]` | no |
4545
| <a name="input_skip_es_s2s_iam_authorization_policy"></a> [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an Event Streams s2s IAM authorization policy to provision an Event Streams mirroring instance. This is required to read from the source cluster. This policy is required when creating mirroring instance. | `bool` | `false` | no |
4646
| <a name="input_skip_kms_iam_authorization_policy"></a> [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the kms\_key\_crn variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no |
4747
| <a name="input_tags"></a> [tags](#input\_tags) | The list of tags associated with the Event Streams instance. | `list(string)` | `[]` | no |

‎modules/fscloud/variables.tf‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,10 @@ variable "region" {
2929
}
3030

3131
variable "schemas" {
32-
type = list(object(
33-
{
34-
schema_id = string
35-
schema = object({
36-
type = string
37-
name = string
38-
fields = optional(list(object({
39-
name = string
40-
type = string
41-
})))
42-
})
43-
}
44-
))
45-
description = "The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object."
32+
type = any
33+
description = "List of schema objects. Each schema must include `schema_id` and `schema` definition. Supports full Apache Avro specification with nested structures. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-ES_schema_registry#ES_apache_avro_data_format)."
4634
default = []
4735
}
48-
4936
variable "schema_global_rule" {
5037
type = string
5138
description = "Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL_TRANSITIVE', 'FORWARD', 'FORWARD_TRANSITIVE', 'BACKWARD', 'BACKWARD_TRANSITIVE'."

‎solutions/security-enforced/DA-schemas-topics-cbr.md‎

Lines changed: 61 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,29 +40,77 @@ The following example includes all the configuration options for topics.
4040
## Options with schemas <a name="options-with-schemas"></a>
4141

4242
- `schema_id` (required): The unique ID to be assigned to schema. If this value is not specified, a generated UUID is assigned.
43-
- `schema` (required): The schema in JSON format. Supported parameters are: `type`, `name` and `fields` (optional).
43+
- `schema` (required): The schema definition as a JSON-compatible Terraform object. Supports all [Apache Avro](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-ES_schema_registry#ES_apache_avro_data_format) types including nested structures.
4444

4545
The following example includes all the configuration options for schemas.
4646

4747
```hcl
4848
[
4949
{
50-
schema_id = "my-es-schema_1"
51-
schema = {
50+
schema_id = "job_events_cloud_sync_value_v1"
51+
schema = {
52+
type = "record"
53+
name = "job_events_cloud_sync_value_v1"
54+
namespace = "envelope"
55+
fields = [
56+
{ name = "source", type = "string" },
57+
{ name = "subject", type = "string" },
58+
{ name = "time", type = "string" },
59+
{ name = "datacontenttype", type = "string", default = "application/json" },
60+
{ name = "producer", type = "string" },
61+
{
62+
name = "data"
63+
type = {
64+
type = "record"
65+
name = "payload"
66+
namespace = "payload"
67+
fields = [
68+
{ name = "event_type", type = "string" },
69+
{ name = "job_id", type = "string" },
70+
{ name = "metadata", type = ["null", "string"], default = null },
71+
{ name = "monotonic_ns", type = "long" },
72+
{ name = "source_instance_id", type = "string" },
73+
{ name = "source_type_id", type = "string" },
74+
{ name = "sub_job_id", type = ["null", "string"], default = null }
75+
]
76+
}
77+
}
78+
]
79+
}
80+
},
81+
{
82+
schema_id = "my-es-schema_1"
83+
schema = {
84+
type = "record"
85+
name = "book"
86+
fields = [
87+
{
88+
name = "title"
5289
type = "string"
53-
name = "name_1"
54-
fields = [{
55-
name = "field_name"
56-
type = "string"
57-
}]
58-
}
90+
},
91+
{
92+
name = "author"
93+
type = "string"
94+
}
95+
]
96+
}
5997
},
6098
{
61-
schema_id = "my-es-schema_2"
62-
schema = {
99+
schema_id = "my-es-schema_2"
100+
schema = {
101+
type = "record"
102+
name = "book"
103+
fields = [
104+
{
105+
name = "author"
63106
type = "string"
64-
name = "name_2"
65-
}
107+
},
108+
{
109+
name = "title"
110+
type = "string"
111+
}
112+
]
113+
}
66114
}
67115
]
68116
```

‎solutions/security-enforced/variables.tf‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -63,22 +63,11 @@ variable "access_tags" {
6363
}
6464

6565
variable "schemas" {
66-
type = list(object({
67-
schema_id = string
68-
schema = object({
69-
type = string
70-
name = string
71-
fields = optional(list(object({
72-
name = string
73-
type = string
74-
})))
75-
})
76-
}))
77-
description = "The list of schema objects. Include the `schema_id`, `type` and `name` of the schema in the `schema` object. Learn more: https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/security-enforced/DA-schemas-topics-cbr.md#options-with-schemas."
66+
type = any
67+
description = "List of schema objects. Each schema must include `schema_id` and `schema` definition. Supports full Apache Avro specification with nested structures. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/security-enforced/DA-schemas-topics-cbr.md#options-with-schemas)."
7868
default = []
7969
}
8070

81-
8271
variable "schema_global_rule" {
8372
type = string
8473
description = "Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL_TRANSITIVE', 'FORWARD', 'FORWARD_TRANSITIVE', 'BACKWARD', 'BACKWARD_TRANSITIVE'."

‎variables.tf‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -128,21 +128,16 @@ variable "skip_es_s2s_iam_authorization_policy" {
128128
}
129129

130130
variable "schemas" {
131-
type = list(object(
132-
{
133-
schema_id = string
134-
schema = object({
135-
type = string
136-
name = string
137-
fields = optional(list(object({
138-
name = string
139-
type = string
140-
})))
141-
})
142-
}
143-
))
144-
description = "The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object."
131+
type = any
132+
description = "List of schema objects. Each schema must include `schema_id` and `schema` definition. Supports full Apache Avro specification with nested structures. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-ES_schema_registry#ES_apache_avro_data_format)."
145133
default = []
134+
135+
validation {
136+
condition = alltrue([
137+
for s in var.schemas : s.schema_id != "" && s.schema != null
138+
])
139+
error_message = "Each schema must have a 'schema_id' and a 'schema' definition."
140+
}
146141
}
147142

148143
variable "schema_global_rule" {

0 commit comments

Comments
 (0)