Support for Kinesis pipe (#285)

Author: jkaflik

examples/clickpipe/kinesis_iam_role/README.md

@@ -0,0 +1,9 @@
+## ClickPipe Kinesis with IAM role authentication
+
+This example demonstrates how to deploy a Kinesis ClickPipe using IAM role authentication. This method allows ClickHouse to securely access your Kinesis streams without needing to store access keys.
+
+## How to run
+
+- Rename `variables.tfvars.sample` to `variables.tfvars` and fill in all needed data.
+- Run `terraform init`
+- Run `terraform <plan|apply> -var-file=variables.tfvars`

examples/clickpipe/kinesis_iam_role/main.tf

@@ -0,0 +1,53 @@
+resource "clickhouse_clickpipe" "kinesis_iam_role" {
+  service_id  = var.service_id
+  name        = var.pipe_name
+  description = var.pipe_description
+
+  source = {
+    kinesis = {
+      format        = "JSONEachRow"
+      stream_name   = var.kinesis_stream_name
+      region        = var.aws_region
+      iterator_type = "LATEST"
+
+      # Set to true to use enhanced fan-out consumer (optional)
+      use_enhanced_fan_out = true
+
+      # Using IAM role authentication
+      authentication = "IAM_ROLE"
+      iam_role       = var.iam_role_arn
+    }
+  }
+
+  destination = {
+    table         = "my_table"
+    managed_table = true
+
+    table_definition = {
+      engine = {
+        type = "MergeTree"
+      }
+    }
+
+    columns = [
+      {
+        name = "my_field1"
+        type = "String"
+        }, {
+        name = "my_field2"
+        type = "UInt64"
+      }
+    ]
+  }
+
+  field_mappings = [
+    {
+      source_field      = "my_field"
+      destination_field = "my_field1"
+    }
+  ]
+}
+
+output "clickpipe_id" {
+  value = clickhouse_clickpipe.kinesis_iam_user.id
+}

examples/clickpipe/kinesis_iam_role/provider.tf

@@ -0,0 +1,15 @@
+# This file is generated automatically please do not edit
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "3.0.0-alpha2"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}

examples/clickpipe/kinesis_iam_role/provider.tf.template.alpha

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "${CLICKHOUSE_TERRAFORM_PROVIDER_VERSION}"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}

examples/clickpipe/kinesis_iam_role/variables.tf

@@ -0,0 +1,60 @@
+variable "token_key" {
+  description = "ClickHouse Cloud API Key ID"
+  type        = string
+}
+
+variable "token_secret" {
+  description = "ClickHouse Cloud API Key Secret"
+  type        = string
+  sensitive   = true
+}
+
+variable "organization_id" {
+  description = "ClickHouse Cloud Organization ID"
+  type        = string
+}
+
+variable "service_id" {
+  description = "The ID of the ClickHouse Cloud service"
+  type        = string
+}
+
+variable "pipe_name" {
+  description = "The name of the ClickPipe"
+  type        = string
+  default     = "kinesis-iam-user-example"
+}
+
+variable "pipe_description" {
+  description = "The description of the ClickPipe"
+  type        = string
+  default     = "Example of a Kinesis ClickPipe using IAM User authentication"
+}
+
+variable "database" {
+  description = "The database to store the data"
+  type        = string
+  default     = "default"
+}
+
+variable "table" {
+  description = "The table to store the data"
+  type        = string
+  default     = "kinesis_data"
+}
+
+variable "kinesis_stream_name" {
+  description = "The name of the Kinesis stream"
+  type        = string
+}
+
+variable "aws_region" {
+  description = "The AWS region where the Kinesis stream is located"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "iam_role_arn" {
+  description = "The ARN of the IAM role that ClickHouse will assume to access Kinesis"
+  type        = string
+}

examples/clickpipe/kinesis_iam_role/variables.tfvars.sample

@@ -0,0 +1,10 @@
+api_key_id     = "<YOUR_CLICKHOUSE_CLOUD_API_KEY_ID>"
+api_key_secret = "<YOUR_CLICKHOUSE_CLOUD_API_KEY_SECRET>"
+organization   = "<YOUR_CLICKHOUSE_CLOUD_ORGANIZATION_ID>"
+region         = "us-east-1"
+
+service_id = "<YOUR_CLICKHOUSE_CLOUD_SERVICE_ID>"
+
+kinesis_stream_name = "<YOUR_AWS_KINESIS_STREAM_NAME>"
+aws_region          = "<YOUR_AWS_REGION>"
+iam_role_arn      = "<IAM_ROLE_ARN>"

examples/clickpipe/kinesis_iam_user/README.md

@@ -0,0 +1,9 @@
+## ClickPipe Kinesis with IAM user authentication
+
+This example demonstrates how to deploy a Kinesis ClickPipe using IAM user credentials for authentication.
+
+## How to run
+
+- Rename `variables.tfvars.sample` to `variables.tfvars` and fill in all needed data.
+- Run `terraform init`
+- Run `terraform <plan|apply> -var-file=variables.tfvars`

examples/clickpipe/kinesis_iam_user/main.tf

@@ -0,0 +1,52 @@
+resource "clickhouse_clickpipe" "kinesis_iam_user" {
+  service_id  = var.service_id
+  name        = var.pipe_name
+  description = var.pipe_description
+
+  source = {
+    kinesis = {
+      format        = "JSONEachRow"
+      stream_name   = var.kinesis_stream_name
+      region        = var.aws_region
+      iterator_type = "TRIM_HORIZON"
+
+      authentication = "IAM_USER"
+      access_key = {
+        access_key_id = var.aws_access_key
+        secret_key    = var.aws_secret_key
+      }
+    }
+  }
+
+  destination = {
+    table         = "my_table"
+    managed_table = true
+
+    table_definition = {
+      engine = {
+        type = "MergeTree"
+      }
+    }
+
+    columns = [
+      {
+        name = "my_field1"
+        type = "String"
+        }, {
+        name = "my_field2"
+        type = "UInt64"
+      }
+    ]
+  }
+
+  field_mappings = [
+    {
+      source_field      = "my_field"
+      destination_field = "my_field1"
+    }
+  ]
+}
+
+output "clickpipe_id" {
+  value = clickhouse_clickpipe.kinesis_iam_user.id
+}

examples/clickpipe/kinesis_iam_user/provider.tf

@@ -0,0 +1,15 @@
+# This file is generated automatically please do not edit
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "3.0.0-alpha2"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}

examples/clickpipe/kinesis_iam_user/provider.tf.template.alpha

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "${CLICKHOUSE_TERRAFORM_PROVIDER_VERSION}"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}