Skip to content

Commit 1e77e34

Browse files
NoamstraussNoamstrauss
authored
Merge pull request #8 from aquasecurity/nested_projects_support
Adding support for nested projects discovery
2 parents 722ac7e + 15bd4b4 commit 1e77e34

File tree

31 files changed

+284
-90
lines changed

31 files changed

+284
-90
lines changed

‎README.md‎

Lines changed: 53 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ to enable seamless integration with Aqua’s platform.
1919
- [Pre-requisites](#Pre-requisites)
2020
- [Usage](#usage)
2121
- [Examples](#examples)
22+
- [Providing Project ID List](#providing-project-id-list)
23+
- [Excluding Projects Using Regex](#excluding-projects-using-regex)
2224
- [Using Dedicated Project](#using-an-existing-dedicated-project)
2325
- [Using Existing Network](#using-existing-network-and-firewall)
2426

@@ -52,18 +54,18 @@ locals {
5254
type = "single" # Type of deployment (single project or organization)
5355
org_name = "my-org-name" # Google Cloud Organization name
5456
aqua_tenant_id = "12345" # Aqua tenant ID
55-
project_id = "project_id" # Google Cloud project ID (existing project to be onboarded)
56-
aqua_aws_account_id = "123456789101" # Aqua AWS account ID
57+
project_id = "project_id" # Google Cloud project ID (existing project to be onboarded)
58+
aqua_aws_account_id = "123456789101" # Aqua AWS account ID
5759
aqua_bucket_name = "generic-bucket-name" # Aqua bucket name
5860
aqua_configuration_id = "234e3cea-d84a-4b9e-bb36-92518e6a5772" # Aqua configuration ID
5961
aqua_cspm_group_id = 123456 # Aqua CSPM group ID
6062
aqua_custom_labels = { label = "true" } # Additional custom labels to apply to Aqua resources
61-
aqua_api_key = "REPLACE_ME" # Replace with generated aqua API key
62-
aqua_api_secret = "REPLACE_ME" # Replace with generated aqua API secret
63+
aqua_api_key = "REPLACE_ME" # Replace with generated aqua API key
64+
aqua_api_secret = "REPLACE_ME" # Replace with generated aqua API secret
6365
aqua_autoconnect_url = "https://example-aqua-autoconnect-url.com" # Aqua Autoconnect API URL
64-
aqua_volscan_api_token = "REPLACE_ME" # Replace with Aqua Volume Scanning API token
66+
aqua_volscan_api_token = "REPLACE_ME" # Replace with Aqua Volume Scanning API token
6567
aqua_volscan_api_url = "https://example-aqua-volscan-api-url.com" # Aqua Volume Scanning API URL
66-
dedicated_project_id = "aqua-agentless-${local.tenant_id}-${local.org_hash}"
68+
dedicated_project_id = "aqua-agentless-${local.tenant_id}-${local.org_hash}"
6769
labels = merge(local.aqua_custom_labels, { "aqua-agentless-scanner" = "true" }) # Combined labels for Aqua resources
6870
org_hash = substr(sha1(local.org_name), 0, 6) # Hashed organization name (first 6 characters)
6971
}
@@ -175,8 +177,10 @@ locals {
175177
aqua_volscan_api_token = "<REPLACE_ME>" # Replace with Aqua Volume Scanning API token
176178
aqua_volscan_api_url = "https://example-aqua-volscan-api-url.com" # Aqua Volume Scanning API URL
177179
dedicated_project_id = "aqua-agentless-${local.aqua_tenant_id}-${local.org_hash}"
180+
project_id = "my-project-id" # Google Cloud project ID used to run the Cloud Asset query to fetch all project IDs and create CSPM IAM resources (Cloud Asset API must be enabled)
181+
projects_list = module.aqua_gcp_org_projects.filtered_projects
178182
labels = merge(local.aqua_custom_labels, { "aqua-agentless-scanner" = "true" }) # Combined labels for Aqua resources
179-
org_hash = substr(sha1(local.org_name), 0, 6) # Hashed organization name (first 6 characters)
183+
org_hash = substr(sha1(local.org_name), 0, 6) # Hashed organization name (first 6 characters)
180184
}
181185
182186
################################
@@ -194,18 +198,23 @@ data "google_organization" "org" {
194198
195199
################################
196200
197-
# Getting all projects ID's that are active under the organization ID
198-
data "google_projects" "projects" {
199-
filter = "parent.id=${data.google_organization.org.org_id} AND parent.type=organization AND lifecycleState:ACTIVE"
201+
# Defining the org_projects google provider to fetch all projects ids
202+
provider "google" {
203+
alias = "org_projects"
204+
region = local.region
205+
default_labels = local.labels
206+
user_project_override = true
207+
billing_project = local.project_id
208+
project = local.project_id
200209
}
201210
202-
# Filter out projects containing "aqua-agentless" in their names
203-
locals {
204-
projects_list = [
205-
for project in data.google_projects.projects.projects :
206-
project.project_id
207-
if !can(regex("^.*aqua-agentless.*$", project.project_id))
208-
]
211+
# Fetching all active projects ids
212+
module "aqua_gcp_org_projects" {
213+
source = "../../modules/org_projects"
214+
providers = {
215+
google = google.org_projects
216+
}
217+
org_name = local.org_name
209218
}
210219
211220
################################
@@ -285,23 +294,43 @@ output "onboarding_status" {
285294
}
286295
}
287296
```
297+
For more examples and use cases, please refer to the examples folder in the repository.
288298

289299
## Providing Project ID List
290300

291-
By default we fetch all projects and use that project list, but you can also provide your own list of project IDs by populating the `projects_list` local. To accommodate this, ensure to remove the `data "google_projects"` and then replace the local `projects_list` with your list.
301+
By default we fetch all active projects and use that project list, but you can also provide your own list of project IDs by populating the `projects_list` local. To accommodate this, ensure to remove the `module.aqua_gcp_org_projects` and then replace the local `projects_list` with your list.
292302

293303
```hcl
294304
locals {
295305
projects_list = [
296-
"my-project-id-1",
297-
"my-project-id-2",
298-
// Add more project IDs as needed
306+
"my-project-id-1",
307+
"my-project-id-2",
308+
// Add more project IDs as needed
299309
]
300310
}
301311
```
302312

303-
For more examples and use cases, please refer to the examples folder in the repository.
313+
## Excluding Projects Using Regex
314+
315+
You can exclude specific projects from getting onboarded by using regular expressions.
316+
317+
To exclude projects by id, add the variable `projects_ids_exclude="regex1, regex2, regex3"` to the module `aqua_gcp_org_projects`.
318+
319+
To exclude projects by name, add the variable `projects_names_exclude="regex1, regex2, regex3"` to the module `aqua_gcp_org_projects`.
320+
321+
Here are some examples of traditional exclusions following the instructions above:
322+
323+
1. Exclude Projects Starting with `test-`:
324+
- Regex: `^test-.*$`
325+
- Description: This regex pattern matches GCP project names that start with `test-`.
326+
327+
2. Exclude Projects Ending with `-test`:
328+
- Regex: `^.*-test$`
329+
- Description: This regex pattern matches GCP project names that end with `-test`.
304330

331+
3. Exclude Projects which include test anywhere:
332+
- Regex: `.*test.*`
333+
- Description: This regex pattern matches GCP project names containing the word `test` anywhere in the name.
305334

306335
## Using an Existing Dedicated Project
307336

@@ -365,15 +394,15 @@ When using a dedicated project, the `<project_id>` should follow the format `"aq
365394
|------|---------|
366395
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.6.4 |
367396
| <a name="requirement_external"></a> [external](#requirement\_external) | ~> 2.3.3 |
368-
| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.20.0 |
397+
| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.30.0 |
369398
| <a name="requirement_http"></a> [http](#requirement\_http) | ~> 3.4.2 |
370399
| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.6.0 |
371400

372401
## Providers
373402

374403
| Name | Version |
375404
|------|---------|
376-
| <a name="provider_google"></a> [google](#provider\_google) | ~> 5.20.0 |
405+
| <a name="provider_google"></a> [google](#provider\_google) | ~> 5.30.0 |
377406

378407
## Modules
379408

‎examples/organization-dedicated-project/README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ Before running this example, ensure that you have the following:
2424

2525
## Providing Project ID List
2626

27-
You can provide your own list of project IDs by populating the `projects_list` local. To accommodate this, ensure to remove the `data "google_projects"` and then replace the local `projects_list` with your list.
27+
You can provide your own list of project IDs by populating the `projects_list` local. To accommodate this, ensure to remove the `module.aqua_gcp_org_projects` and then replace the local `projects_list` with your list.
2828

2929
```hcl
3030
locals {
@@ -38,9 +38,10 @@ locals {
3838

3939
## What's Happening
4040

41-
1. The `aqua_gcp_dedicated_project` module is called to create a dedicated GCP project with the name `aqua-agentless-<tenant_id>-<org_hash>`, where `org_hash` is the first six characters of the SHA1 hash of your organization name.
42-
2. The `aqua_gcp_onboarding` module is called to provision the necessary resources (service accounts, roles, networking, etc.) in the dedicated GCP project.
43-
3. The `aqua_gcp_project_attachment` module is called for each GCP project in the organization to create the required IAM resources and trigger the Aqua API to onboard the project.
41+
1. The `aqua_gcp_org_projects` module is called to fetch all GCP active projects ids.
42+
2. The `aqua_gcp_dedicated_project` module is called to create a dedicated GCP project with the name `aqua-agentless-<tenant_id>-<org_hash>`, where `org_hash` is the first six characters of the SHA1 hash of your organization name.
43+
3. The `aqua_gcp_onboarding` module is called to provision the necessary resources (service accounts, roles, networking, etc.) in the dedicated GCP project.
44+
4. The `aqua_gcp_project_attachment` module is called for each GCP project in the organization to create the required IAM resources and trigger the Aqua API to onboard the project.
4445

4546
## Outputs
4647

@@ -49,4 +50,3 @@ locals {
4950
## Cleanup
5051

5152
To remove the resources created by this example, including the organization-level resources, dedicated project, and attached projects, run `terraform destroy`.
52-

‎examples/organization-dedicated-project/main.tf‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ locals {
1818
aqua_volscan_api_token = "<REPLACE_ME>"
1919
aqua_volscan_api_url = "https://example-aqua-volscan-api-url.com"
2020
dedicated_project_id = "aqua-agentless-${local.aqua_tenant_id}-${local.org_hash}"
21+
project_id = "my-project-id" # This project ID is used to run the Cloud Asset query to fetch all project IDs and create CSPM IAM resources
22+
projects_list = module.aqua_gcp_org_projects.filtered_projects
2123
labels = merge(local.aqua_custom_labels, { "aqua-agentless-scanner" = "true" })
2224
org_hash = substr(sha1(local.org_name), 0, 6)
2325
}
@@ -30,25 +32,25 @@ provider "google" {
3032
default_labels = local.labels
3133
}
3234

33-
# Getting google organization ID
34-
data "google_organization" "org" {
35-
domain = local.org_name
36-
}
37-
3835
################################
3936

40-
# Getting all projects ID's that are active under the organization ID
41-
data "google_projects" "projects" {
42-
filter = "parent.id=${data.google_organization.org.org_id} AND parent.type=organization AND lifecycleState:ACTIVE"
37+
# Defining the org_projects google provider to fetch all projects ids
38+
provider "google" {
39+
alias = "org_projects"
40+
region = local.region
41+
default_labels = local.labels
42+
user_project_override = true
43+
billing_project = local.project_id
44+
project = local.project_id
4345
}
4446

45-
# Filter out projects containing "aqua-agentless" in their names
46-
locals {
47-
projects_list = [
48-
for project in data.google_projects.projects.projects :
49-
project.project_id
50-
if !can(regex("^.*aqua-agentless.*$", project.project_id))
51-
]
47+
# Fetching all active projects ids
48+
module "aqua_gcp_org_projects" {
49+
source = "../../modules/org_projects"
50+
providers = {
51+
google = google.org_projects
52+
}
53+
org_name = local.org_name
5254
}
5355

5456
################################
@@ -94,7 +96,7 @@ module "aqua_gcp_onboarding" {
9496

9597
################################
9698

97-
## Iterating over all project and attaching them to the dedicated project
99+
# Iterating over all project and attaching them to the dedicated project
98100
module "aqua_gcp_projects_attachment" {
99101
source = "../../modules/project_attachment"
100102
providers = {

‎examples/organization-same-project-list/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ Before running this example, ensure that you have the following:
2525

2626
## What's Happening
2727

28-
1. The `aqua_gcp_onboarding` module is called for each GCP project specified in the `projects_list` variable to provision the necessary resources (service accounts, roles, networking, etc.).
29-
2. The `aqua_gcp_project_attachment` module is called for each specified GCP project to create the required IAM resources and trigger the Aqua API to onboard the project.
28+
1. The `aqua_gcp_cspm_iam` module is called to create the CSPM IAM resources once per organization.
29+
2. The `aqua_gcp_onboarding` module is called for each GCP project specified in the `projects_list` variable to provision the necessary resources (service accounts, roles, networking, etc.).
30+
3. The `aqua_gcp_project_attachment` module is called for each specified GCP project to create the required IAM resources and trigger the Aqua API to onboard the project.
3031

3132
## Outputs
3233

‎examples/organization-same-project-list/main.tf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ locals {
1616
aqua_autoconnect_url = "https://example-aqua-autoconnect-url.com"
1717
aqua_volscan_api_token = "<REPLACE_ME>"
1818
aqua_volscan_api_url = "https://example-aqua-volscan-api-url.com"
19-
cspm_project_id = "" # project id where CSPM iam resources will be provisioned. If not set, it will be set by default to the first project in the organization
19+
project_id = "my-project-id" # This project ID is used to create CSPM IAM resources
2020
labels = merge(local.aqua_custom_labels, { "aqua-agentless-scanner" = "true" })
2121
projects_list = ["my-project-id-1", "my-project-id-2"]
2222
}
@@ -42,7 +42,7 @@ module "aqua_gcp_cspm_iam" {
4242
providers = {
4343
google = google
4444
}
45-
project_id = local.cspm_project_id == "" ? local.projects_list[0] : local.cspm_project_id
45+
project_id = local.project_id
4646
aqua_bucket_name = local.aqua_bucket_name
4747
aqua_tenant_id = local.aqua_tenant_id
4848
org_id = data.google_organization.org.org_id
@@ -71,7 +71,7 @@ module "aqua_gcp_onboarding" {
7171

7272
################################
7373

74-
## Iterating over all project and creating attachment resources
74+
# Iterating over all project and creating attachment resources
7575
module "aqua_gcp_projects_attachment" {
7676
source = "../../modules/project_attachment"
7777
providers = {

‎examples/organization-same-project/README.md‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,26 @@ Before running this example, ensure that you have the following:
2222
3. Run `terraform init` to initialize the Terraform working directory.
2323
4. Run `terraform apply` to create the resources.
2424

25+
## Providing Project ID List
26+
27+
You can provide your own list of project IDs by populating the `projects_list` local. To accommodate this, ensure to remove the `module.aqua_gcp_org_projects` and then replace the local `projects_list` with your list.
28+
29+
```hcl
30+
locals {
31+
projects_list = [
32+
"my-project-id-1",
33+
"my-project-id-2",
34+
// Add more project IDs as needed
35+
]
36+
}
37+
```
38+
2539
## What's Happening
2640

27-
1. The `aqua_gcp_onboarding` module is called for each GCP project to provision the necessary resources (service accounts, roles, networking, etc.).
28-
2. The `aqua_gcp_project_attachment` module is called for each GCP project to create the required IAM resources and trigger the Aqua API to onboard the project.
41+
1. The `aqua_gcp_org_projects` module is called to fetch all GCP active projects ids.
42+
2. The `aqua_gcp_cspm_iam` module is called to create the CSPM IAM resources once per organization.
43+
3. The `aqua_gcp_onboarding` module is called for each GCP project to provision the necessary resources (service accounts, roles, networking, etc.).
44+
4. The `aqua_gcp_project_attachment` module is called for each GCP project to create the required IAM resources and trigger the Aqua API to onboard the project.
2945

3046
## Outputs
3147

‎examples/organization-same-project/main.tf‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ locals {
1616
aqua_autoconnect_url = "https://example-aqua-autoconnect-url.com"
1717
aqua_volscan_api_token = "<REPLACE_ME>"
1818
aqua_volscan_api_url = "https://example-aqua-volscan-api-url.com"
19-
cspm_project_id = "" # project id where CSPM iam resources will be provisioned. If not set, it will be set by default to the first project in the organization
19+
project_id = "my-project-id" # This project ID is used to run the Cloud Asset query to fetch all project IDs and create CSPM IAM resources.
20+
projects_list = module.aqua_gcp_org_projects.filtered_projects
2021
labels = merge(local.aqua_custom_labels, { "aqua-agentless-scanner" = "true" })
2122
}
2223

@@ -28,36 +29,39 @@ provider "google" {
2829
default_labels = local.labels
2930
}
3031

31-
# Getting google organization ID
32-
data "google_organization" "org" {
33-
domain = local.org_name
34-
}
32+
################################
3533

36-
# Getting all projects ID's that are active under the organization ID
37-
data "google_projects" "projects" {
38-
filter = "parent.id=${data.google_organization.org.org_id} AND parent.type=organization AND lifecycleState:ACTIVE"
34+
# Defining the org_projects google provider to use in the org_projects module to fetch all projects ids
35+
provider "google" {
36+
alias = "org_projects"
37+
region = local.region
38+
default_labels = local.labels
39+
user_project_override = true
40+
billing_project = local.project_id
41+
project = local.project_id
3942
}
4043

41-
# Filter out projects containing "aqua-agentless" in their names
42-
locals {
43-
projects_list = [
44-
for project in data.google_projects.projects.projects :
45-
project.project_id
46-
if !can(regex("^.*aqua-agentless.*$", project.project_id))
47-
]
44+
# Fetching all active projects ids
45+
module "aqua_gcp_org_projects" {
46+
source = "../../modules/org_projects"
47+
providers = {
48+
google = google.org_projects
49+
}
50+
org_name = local.org_name
4851
}
4952

5053
################################
54+
5155
# Creating CSPM IAM resources
5256
module "aqua_gcp_cspm_iam" {
5357
source = "../../modules/cspm_iam"
5458
providers = {
5559
google = google
5660
}
57-
project_id = local.cspm_project_id == "" ? local.projects_list[0] : local.cspm_project_id
61+
project_id = local.project_id
5862
aqua_bucket_name = local.aqua_bucket_name
5963
aqua_tenant_id = local.aqua_tenant_id
60-
org_id = data.google_organization.org.org_id
64+
org_id = module.aqua_gcp_org_projects.org_id
6165
}
6266

6367
################################
@@ -83,7 +87,7 @@ module "aqua_gcp_onboarding" {
8387

8488
################################
8589

86-
## Iterating over all project and creating attachment resources
90+
# Iterating over all project and creating attachment resources
8791
module "aqua_gcp_projects_attachment" {
8892
source = "../../modules/project_attachment"
8993
providers = {

‎examples/single-dedicated-project-addition/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ provider "google" {
111111
default_labels = local.labels
112112
}
113113

114-
## Onboarding an additional project and attaching it to the dedicated project
114+
# Onboarding an additional project and attaching it to the dedicated project
115115
module "aqua_gcp_additional_project_attachment" {
116116
source = "../../modules/project_attachment"
117117
providers = {

‎examples/single-dedicated-project/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ module "aqua_gcp_onboarding" {
7272

7373
################################
7474

75-
## Onboarding a project and attaching it to the dedicated project
75+
# Onboarding a project and attaching it to the dedicated project
7676
module "aqua_gcp_project_attachment" {
7777
source = "../../modules/project_attachment"
7878
providers = {

0 commit comments

Comments
 (0)