Skip to content

Commit e35f083

Browse files
jose-pablo-camachoharesh-suresh
jose-pablo-camacho
and
haresh-suresh
authored
feat(modular): add modular support for cdr/ciem (#35)
* feat(modular): add modular support for cdr/ciem * feat(modular): add modular support for cdr/ciem * feat(modular): add modular support for cdr/ciem * feat(modular): add modular support for cdr/ciem * feat(modular): add modular support for cdr/ciem * feat(modular): address feedback for modular support for cdr/ciem * adding modular onboarding module * fix var refns * adding modular onboarding example * adding config posture module for modular onboarding * updating README * fix role naming & version metadata * updating examples for onboarding & cspm org * cleanup foundational READMEs * use external_id datasource * update README * remove mgmt_group_ids in cspm module * bump sysdig provider version to be consistent & have latest datasources * updating examples * update example * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * add explicit dependency * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * feat(modular): address feedback for modular support for cdr/ciem, rebase and cleanup * enhance(modular): use org domain instead of org ID for gcp api enablement (#40) * rebase and merge fix --------- Co-authored-by: Haresh Suresh <[email protected]>
1 parent 15672c7 commit e35f083

File tree

8 files changed

+657
-0
lines changed

8 files changed

+657
-0
lines changed

‎modules/integrations/pub-sub/README.md‎

Lines changed: 108 additions & 0 deletions
Large diffs are not rendered by default.

‎modules/integrations/pub-sub/main.tf‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
#-----------------------------------------------------------------------------------------------------------------------------------------
2+
# For Organizational installs, see organizational.tf.
3+
# This module takes care of provisioning the necessary resources to make Sysdig's backend able to ingest data from a
4+
# single GCP project.
5+
#
6+
# Note: The alternative definitions for the organizational variant of this module are contained
7+
# in organizational.tf. The only differences w.r.t. the standalone template is in using an
8+
# organizational sink instead of a project-specific one, as well as enabling AuditLogs for
9+
# all the projects that fall within the organization.
10+
#-----------------------------------------------------------------------------------------------------------------------------------------
11+
12+
#-----------------------------------------------------------------------------------------
13+
# Fetch the data sources
14+
#-----------------------------------------------------------------------------------------
15+
data "sysdig_secure_trusted_cloud_identity" "trusted_identity" {
16+
cloud_provider = "gcp"
17+
}
18+
19+
data "google_project" "project" {
20+
project_id = var.project_id
21+
}
22+
23+
data "sysdig_secure_tenant_external_id" "external_id" {}
24+
25+
data "sysdig_secure_cloud_ingestion_assets" "assets" {}
26+
27+
#-----------------------------------------------------------------------------------------
28+
# These locals indicate the suffix to create unique name for resources
29+
#-----------------------------------------------------------------------------------------
30+
locals {
31+
suffix = var.suffix == null ? random_id.suffix[0].hex : var.suffix
32+
role_name = "SysdigIngestionAuthRole"
33+
}
34+
35+
36+
#-----------------------------------------------------------------------------------------------------------------------
37+
# A random resource is used to generate unique Pub Sub name suffix for resources.
38+
# This prevents conflicts when recreating a Pub Sub resources with the same name.
39+
#-----------------------------------------------------------------------------------------------------------------------
40+
resource "random_id" "suffix" {
41+
count = var.suffix == null ? 1 : 0
42+
byte_length = 3
43+
}
44+
45+
#-----------------------------------------------------------------------------------------
46+
# Audit Logs
47+
#-----------------------------------------------------------------------------------------
48+
locals {
49+
# Data structure will be a map for each service, that can have multiple audit_log_config
50+
audit_log_config = { for audit in var.audit_log_config :
51+
audit["service"] => {
52+
log_config = audit["log_config"]
53+
}
54+
}
55+
}
56+
57+
resource "google_project_iam_audit_config" "audit_config" {
58+
for_each = var.is_organizational ? {} : local.audit_log_config
59+
project = var.project_id
60+
service = each.key
61+
62+
dynamic "audit_log_config" {
63+
for_each = each.value.log_config
64+
iterator = log_config
65+
content {
66+
log_type = log_config.value.log_type
67+
exempted_members = log_config.value.exempted_members
68+
}
69+
}
70+
}
71+
72+
#-----------------------------------------------------------------------------------------
73+
# Ingestion Topic
74+
#-----------------------------------------------------------------------------------------
75+
resource "google_pubsub_topic" "ingestion_topic" {
76+
name = "ingestion_topic"
77+
labels = var.labels
78+
project = var.project_id
79+
message_retention_duration = var.message_retention_duration
80+
}
81+
82+
resource "google_pubsub_topic" "deadletter_topic" {
83+
name = "dl-${google_pubsub_topic.ingestion_topic.name}"
84+
project = var.project_id
85+
message_retention_duration = var.message_retention_duration
86+
}
87+
88+
#-----------------------------------------------------------------------------------------
89+
# Sink
90+
#-----------------------------------------------------------------------------------------
91+
resource "google_logging_project_sink" "ingestion_sink" {
92+
count = var.is_organizational ? 0 : 1
93+
name = "${google_pubsub_topic.ingestion_topic.name}_sink"
94+
description = "Sysdig sink to direct the AuditLogs to the PubSub topic used for data gathering"
95+
96+
# NOTE: The target destination is a PubSub topic
97+
destination = "pubsub.googleapis.com/projects/${var.project_id}/topics/${google_pubsub_topic.ingestion_topic.name}"
98+
filter = var.ingestion_sink_filter
99+
100+
# Dynamic block to exclude logs from ingestion
101+
dynamic "exclusions" {
102+
for_each = var.exclude_logs_filter
103+
content {
104+
name = exclusions.value.name
105+
description = exclusions.value.description
106+
filter = exclusions.value.filter
107+
disabled = exclusions.value.disabled
108+
}
109+
}
110+
111+
# NOTE: Used to create a dedicated writer identity and not using the default one
112+
unique_writer_identity = true
113+
}
114+
115+
resource "google_pubsub_topic_iam_member" "publisher_iam_member" {
116+
project = google_pubsub_topic.ingestion_topic.project
117+
topic = google_pubsub_topic.ingestion_topic.name
118+
role = "roles/pubsub.publisher"
119+
member = var.is_organizational ? google_logging_organization_sink.ingestion_sink[0].writer_identity : google_logging_project_sink.ingestion_sink[0].writer_identity
120+
}
121+
122+
#-----------------------------------------------------------------------------------------
123+
# Push Subscription
124+
#-----------------------------------------------------------------------------------------
125+
resource "google_service_account" "push_auth" {
126+
account_id = "sysdig-ingestion-${local.suffix}"
127+
display_name = "Sysdig Ingestion Push Auth Service Account"
128+
project = var.project_id
129+
}
130+
131+
resource "google_service_account_iam_binding" "push_auth_binding" {
132+
service_account_id = google_service_account.push_auth.name
133+
role = "roles/iam.workloadIdentityUser"
134+
135+
members = [
136+
"serviceAccount:${google_service_account.push_auth.email}",
137+
]
138+
}
139+
140+
resource "google_pubsub_subscription" "ingestion_topic_push_subscription" {
141+
name = "${google_pubsub_topic.ingestion_topic.name}_push_subscription"
142+
topic = google_pubsub_topic.ingestion_topic.name
143+
labels = var.labels
144+
ack_deadline_seconds = var.ack_deadline_seconds
145+
message_retention_duration = var.message_retention_duration
146+
project = var.project_id
147+
148+
push_config {
149+
push_endpoint = data.sysdig_secure_cloud_ingestion_assets.assets.gcp_metadata.ingestionURL
150+
attributes = {
151+
x-goog-version = "v1"
152+
}
153+
oidc_token {
154+
service_account_email = google_service_account.push_auth.email
155+
audience = "sysdig_secure"
156+
}
157+
}
158+
159+
retry_policy {
160+
minimum_backoff = var.minimum_backoff
161+
maximum_backoff = var.maximum_backoff
162+
}
163+
164+
dead_letter_policy {
165+
dead_letter_topic = google_pubsub_topic.deadletter_topic.id
166+
max_delivery_attempts = var.max_delivery_attempts
167+
}
168+
}
169+
170+
#-----------------------------------------------------------------------------------------
171+
# Configure Workload Identity Federation for auth
172+
# See https://cloud.google.com/iam/docs/access-resources-aws
173+
# -----------------------------------------------------------------------------------------
174+
175+
resource "google_iam_workload_identity_pool" "ingestion_auth_pool" {
176+
project = var.project_id
177+
workload_identity_pool_id = "sysdig-ingestion-${local.suffix}"
178+
}
179+
180+
resource "google_iam_workload_identity_pool_provider" "ingestion_auth_pool_provider" {
181+
project = var.project_id
182+
workload_identity_pool_id = google_iam_workload_identity_pool.ingestion_auth_pool.workload_identity_pool_id
183+
workload_identity_pool_provider_id = "sysdig-ingestion-${local.suffix}"
184+
display_name = "Sysdigcloud ingestion auth"
185+
description = "AWS identity pool provider for Sysdig Secure Data Ingestion resources"
186+
disabled = false
187+
188+
attribute_condition = "attribute.aws_role==\"arn:aws:sts::${data.sysdig_secure_trusted_cloud_identity.trusted_identity.aws_account_id}:assumed-role/${data.sysdig_secure_trusted_cloud_identity.trusted_identity.aws_role_name}/${data.sysdig_secure_tenant_external_id.external_id.external_id}\""
189+
190+
attribute_mapping = {
191+
"google.subject" = "assertion.arn",
192+
"attribute.aws_role" = "assertion.arn"
193+
}
194+
195+
aws {
196+
account_id = data.sysdig_secure_trusted_cloud_identity.trusted_identity.aws_account_id
197+
}
198+
}
199+
200+
# creating custom role with project-level permissions to access data ingestion resources
201+
resource "google_project_iam_custom_role" "custom_ingestion_auth_role" {
202+
count = var.is_organizational ? 0 : 1
203+
204+
project = var.project_id
205+
role_id = "${local.role_name}${local.suffix}"
206+
title = "Sysdigcloud Ingestion Auth Role"
207+
description = "A Role providing the required permissions for Sysdig Backend to read cloud resources created for data ingestion"
208+
permissions = [
209+
"pubsub.topics.get",
210+
"pubsub.topics.list",
211+
"pubsub.subscriptions.get",
212+
"pubsub.subscriptions.list",
213+
"logging.sinks.get",
214+
"logging.sinks.list",
215+
]
216+
}
217+
218+
# adding custom role with project-level permissions to the service account for auth
219+
resource "google_project_iam_member" "custom" {
220+
count = var.is_organizational ? 0 : 1
221+
222+
project = var.project_id
223+
role = google_project_iam_custom_role.custom_ingestion_auth_role[0].id
224+
member = "serviceAccount:${google_service_account.push_auth.email}"
225+
}
226+
227+
# attaching WIF as a member to the service account for auth
228+
resource "google_service_account_iam_member" "custom_auth" {
229+
service_account_id = google_service_account.push_auth.name
230+
role = "roles/iam.workloadIdentityUser"
231+
member = "principalSet://iam.googleapis.com/projects/${data.google_project.project.number}/locations/global/workloadIdentityPools/${google_iam_workload_identity_pool.ingestion_auth_pool.workload_identity_pool_id}/attribute.aws_role/arn:aws:sts::${data.sysdig_secure_trusted_cloud_identity.trusted_identity.aws_account_id}:assumed-role/${data.sysdig_secure_trusted_cloud_identity.trusted_identity.aws_role_name}/${data.sysdig_secure_tenant_external_id.external_id.external_id}"
232+
}
233+
234+
# adding ciem role with permissions to the service account
235+
resource "google_project_iam_member" "identity_mgmt" {
236+
for_each = var.is_organizational ? [] : toset(["roles/recommender.viewer", "roles/iam.serviceAccountViewer", "roles/iam.roleViewer", "roles/container.clusterViewer", "roles/compute.viewer"])
237+
238+
project = var.project_id
239+
role = each.key
240+
member = "serviceAccount:${google_service_account.push_auth.email}"
241+
}
242+
243+
#-----------------------------------------------------------------------------------------------------------------------------------------
244+
# Call Sysdig Backend to add the pub-sub integration to the Sysdig Cloud Account
245+
#
246+
# Note (optional): To ensure this gets called after all cloud resources are created, add
247+
# explicit dependency using depends_on
248+
#-----------------------------------------------------------------------------------------------------------------------------------------
249+
250+
resource "sysdig_secure_cloud_auth_account_component" "gcp_pubsub_datasource" {
251+
account_id = var.sysdig_secure_account_id
252+
type = "COMPONENT_WEBHOOK_DATASOURCE"
253+
instance = "secure-runtime"
254+
version = "v0.1.0"
255+
webhook_datasource_metadata = jsonencode({
256+
gcp = {
257+
webhook_datasource = {
258+
pubsub_topic_name = google_pubsub_topic.ingestion_topic.name
259+
sink_name = var.is_organizational ? google_logging_organization_sink.ingestion_sink[0].name : google_logging_project_sink.ingestion_sink[0].name
260+
push_subscription_name = google_pubsub_subscription.ingestion_topic_push_subscription.name
261+
push_endpoint = google_pubsub_subscription.ingestion_topic_push_subscription.push_config[0].push_endpoint
262+
routing_key = data.sysdig_secure_cloud_ingestion_assets.assets.gcp_routing_key
263+
}
264+
service_principal = {
265+
workload_identity_federation = {
266+
pool_id = google_iam_workload_identity_pool.ingestion_auth_pool.workload_identity_pool_id
267+
pool_provider_id = google_iam_workload_identity_pool_provider.ingestion_auth_pool_provider.workload_identity_pool_provider_id
268+
project_number = data.google_project.project.number
269+
}
270+
email = google_service_account.push_auth.email
271+
}
272+
}
273+
})
274+
}

‎modules/integrations/pub-sub/organizational.tf‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#-----------------------------------------------------------------------------------------
2+
# Fetch the data sources
3+
#-----------------------------------------------------------------------------------------
4+
5+
data "google_organization" "org" {
6+
count = var.is_organizational ? 1 : 0
7+
domain = var.organization_domain
8+
}
9+
10+
#-----------------------------------------------------------------------------------------
11+
# Audit Logs
12+
#-----------------------------------------------------------------------------------------
13+
14+
resource "google_organization_iam_audit_config" "audit_config" {
15+
for_each = var.is_organizational ? local.audit_log_config : {}
16+
17+
org_id = data.google_organization.org[0].org_id
18+
service = each.key
19+
20+
dynamic "audit_log_config" {
21+
for_each = each.value.log_config
22+
iterator = log_config
23+
content {
24+
log_type = log_config.value.log_type
25+
exempted_members = log_config.value.exempted_members
26+
}
27+
}
28+
}
29+
30+
#-----------------------------------------------------------------------------------------
31+
# Sink
32+
#-----------------------------------------------------------------------------------------
33+
34+
resource "google_logging_organization_sink" "ingestion_sink" {
35+
count = var.is_organizational ? 1 : 0
36+
37+
name = "${google_pubsub_topic.ingestion_topic.name}_sink"
38+
description = "Sysdig sink to direct the AuditLogs to the PubSub topic used for data gathering"
39+
org_id = data.google_organization.org[0].org_id
40+
41+
# NOTE: The target destination is a PubSub topic
42+
destination = "pubsub.googleapis.com/projects/${var.project_id}/topics/${google_pubsub_topic.ingestion_topic.name}"
43+
filter = var.ingestion_sink_filter
44+
45+
# Dynamic block to exclude logs from ingestion
46+
dynamic "exclusions" {
47+
for_each = var.exclude_logs_filter
48+
content {
49+
name = exclusions.value.name
50+
description = exclusions.value.description
51+
filter = exclusions.value.filter
52+
disabled = exclusions.value.disabled
53+
}
54+
}
55+
56+
# NOTE: The include_children attribute is set to true in order to ingest data
57+
# even from potential sub-organizations
58+
include_children = true
59+
}
60+
61+
# creating custom role with organization-level permissions to access data ingestion resources
62+
resource "google_organization_iam_custom_role" "custom_ingestion_auth_role" {
63+
count = var.is_organizational ? 1 : 0
64+
65+
org_id = data.google_organization.org[0].org_id
66+
role_id = "${local.role_name}Org${local.suffix}"
67+
title = "Sysdigcloud Ingestion Org Auth Role"
68+
description = "A Role providing the required permissions for Sysdig Backend to read cloud resources created for data ingestion"
69+
permissions = [
70+
"pubsub.topics.get",
71+
"pubsub.topics.list",
72+
"pubsub.subscriptions.get",
73+
"pubsub.subscriptions.list",
74+
"logging.sinks.get",
75+
"logging.sinks.list",
76+
]
77+
}
78+
79+
# adding custom role with organization-level permissions to the service account for auth
80+
resource "google_organization_iam_member" "custom" {
81+
count = var.is_organizational ? 1 : 0
82+
83+
org_id = data.google_organization.org[0].org_id
84+
role = google_organization_iam_custom_role.custom_ingestion_auth_role[0].id
85+
member = "serviceAccount:${google_service_account.push_auth.email}"
86+
}
87+
88+
# adding ciem role with permissions to the service account for org
89+
resource "google_organization_iam_member" "identity_mgmt" {
90+
for_each = var.is_organizational ? toset(["roles/recommender.viewer", "roles/iam.serviceAccountViewer", "roles/iam.organizationRoleViewer", "roles/container.clusterViewer", "roles/compute.viewer"]) : []
91+
92+
org_id = data.google_organization.org[0].org_id
93+
role = each.key
94+
member = "serviceAccount:${google_service_account.push_auth.email}"
95+
}

‎modules/integrations/pub-sub/outputs.tf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
output "pubsub_datasource_component_id" {
2+
value = "${sysdig_secure_cloud_auth_account_component.gcp_pubsub_datasource.type}/${sysdig_secure_cloud_auth_account_component.gcp_pubsub_datasource.instance}"
3+
description = "Component identifier of Webhook Datasource integration created in Sysdig Backend for Log Ingestion"
4+
depends_on = [sysdig_secure_cloud_auth_account_component.gcp_pubsub_datasource]
5+
}

0 commit comments

Comments
 (0)