Skip to content

Commit f0460ae

Browse files
kharkevichkharkevich
authored
Merge pull request #1 from data-platform-hq/add-module
feat: add module
2 parents 64d7c0d + db4373e commit f0460ae

File tree

6 files changed

+230
-3
lines changed

6 files changed

+230
-3
lines changed

‎README.md‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,59 @@
1-
# Azure <> Terraform module
2-
Terraform module for creation Azure <>
1+
# Azure Data Lake Storage Gen2 Terraform module
2+
Terraform module for creation Azure Data Lake Storage Gen2 file system
33

44
## Usage
55

66
<!-- BEGIN_TF_DOCS -->
7+
## Requirements
78

9+
| Name | Version |
10+
|------|---------|
11+
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
12+
| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | >= 3.23.0 |
13+
| <a name="requirement_null"></a> [null](#requirement\_null) | >=3.1.1 |
14+
15+
## Providers
16+
17+
| Name | Version |
18+
|------|---------|
19+
| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | 3.24.0 |
20+
| <a name="provider_null"></a> [null](#provider\_null) | 3.1.1 |
21+
22+
## Modules
23+
24+
No modules.
25+
26+
## Resources
27+
28+
| Name | Type |
29+
|------|------|
30+
| [azurerm_storage_data_lake_gen2_filesystem.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_data_lake_gen2_filesystem) | resource |
31+
| [null_resource.create_folders](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
32+
| [null_resource.create_root_folder](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
33+
34+
## Inputs
35+
36+
| Name | Description | Type | Default | Required |
37+
|------|-------------|------|---------|:--------:|
38+
| <a name="input_ace_default"></a> [ace\_default](#input\_ace\_default) | Default ACE values | `list(map(string))` | <pre>[<br> {<br> "permissions": "---",<br> "scope": "access",<br> "type": "other"<br> },<br> {<br> "permissions": "---",<br> "scope": "default",<br> "type": "other"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "access",<br> "type": "group"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "access",<br> "type": "mask"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "access",<br> "type": "user"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "default",<br> "type": "group"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "default",<br> "type": "mask"<br> },<br> {<br> "permissions": "rwx",<br> "scope": "default",<br> "type": "user"<br> }<br>]</pre> | no |
39+
| <a name="input_ad_groups"></a> [ad\_groups](#input\_ad\_groups) | Data which is contain mapping AD group name and GUID | `map(string)` | `{}` | no |
40+
| <a name="input_folders"></a> [folders](#input\_folders) | Name of ADLS folders to create in root directory | `list(any)` | `[]` | no |
41+
| <a name="input_name"></a> [name](#input\_name) | Name of ADLS FS to create | `string` | n/a | yes |
42+
| <a name="input_permissions"></a> [permissions](#input\_permissions) | List of ADLS FS permissions | `list(map(string))` | <pre>[<br> {}<br>]</pre> | no |
43+
| <a name="input_root_dir"></a> [root\_dir](#input\_root\_dir) | Name of ADLS root directory | `string` | `"data"` | no |
44+
| <a name="input_storage_account_id"></a> [storage\_account\_id](#input\_storage\_account\_id) | ID of storage account to create ADLS in | `string` | n/a | yes |
45+
| <a name="input_storage_account_name"></a> [storage\_account\_name](#input\_storage\_account\_name) | Name of storage account to create ADLS in | `string` | n/a | yes |
46+
47+
## Outputs
48+
49+
| Name | Description |
50+
|------|-------------|
51+
| <a name="output_id"></a> [id](#output\_id) | The ID of the Data Lake Storage Gen2 Filesystem (container ID). |
52+
| <a name="output_name"></a> [name](#output\_name) | The name of the Data Lake Storage Gen2 Filesystem (container name). |
53+
| <a name="output_root_path"></a> [root\_path](#output\_root\_path) | The name of the root directory. |
54+
| <a name="output_storage_account_id"></a> [storage\_account\_id](#output\_storage\_account\_id) | The ID of the Storage Account where the Data Lake Storage Gen2 Filesystem exists. |
855
<!-- END_TF_DOCS -->
956

1057
## License
1158

12-
Apache 2 Licensed. For more information please see [LICENSE](https://github.com/data-platform-hq/terraform-azurerm<>/tree/master/LICENSE)
59+
Apache 2 Licensed. For more information please see [LICENSE](https://github.com/data-platform-hq/terraform-azurerm-adls-v2/tree/main/LICENSE)

‎az_create_folders.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
ACCOUNT_NAME=$1
3+
FS_NAME=$2
4+
FOLDERS_LIST=$3
5+
EXTRA_ACL=$4
6+
7+
#echo ${ACCOUNT_NAME}
8+
#echo ${FS_NAME}
9+
#echo ${FOLDERS_LIST}
10+
#echo ${EXTRA_ACL}
11+
12+
for dir in ${FOLDERS_LIST//,/ };do
13+
if [ "$(az storage fs directory exists --account-name ${ACCOUNT_NAME} --file-system ${FS_NAME} --name "/${dir}" --only-show-errors | jq -r .exists | tr -d '\n')" = "false" ] ;then
14+
echo "Folder ${dir} already exists"
15+
az storage fs directory create --account-name ${ACCOUNT_NAME} --file-system ${FS_NAME} --name "/${dir}" --only-show-errors
16+
echo "Folder ${dir} created"
17+
else
18+
echo "Folder ${dir} already exists"
19+
fi
20+
default_acl=$(az storage fs access show --only-show-errors -p "/${FOLDER_NAME}" --account-name ${ACCOUNT_NAME} -f ${FS_NAME} | jq -r .acl | egrep -o "(default:)?(group|user|mask|other)::[rwx-]{3}" | uniq | sed -e 's/\(.*\)/\1,/' | tr -d '\n' | sed -e 's/,$//')
21+
echo "Setting ACL='${default_acl}${EXTRA_ACL}' for folder ${dir}"
22+
az storage fs access set-recursive --only-show-errors -p "/${FOLDER_NAME}" --account-name ${ACCOUNT_NAME} -f ${FS_NAME} --acl "${default_acl}${EXTRA_ACL}"
23+
done

‎main.tf‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
locals {
2+
folders = length(var.folders) == 0 ? "" : join(",", [for f in var.folders : "${var.root_dir}/${f}"])
3+
extra_acl = length(var.permissions) == 0 ? "" : format(",%s",
4+
join(
5+
",",
6+
concat(
7+
[for v in [for k in var.permissions : k if(contains(keys(k), "user") && k["scope"] == "access")] : "${v.type}:${v.user}:${v.permissions}"],
8+
[for v in [for k in var.permissions : k if(contains(keys(k), "user") && k["scope"] == "default")] : "default:${v.type}:${v.user}:${v.permissions}"]
9+
)
10+
)
11+
)
12+
}
13+
14+
resource "azurerm_storage_data_lake_gen2_filesystem" "this" {
15+
name = var.name
16+
storage_account_id = var.storage_account_id
17+
18+
lifecycle { prevent_destroy = false }
19+
20+
dynamic "ace" {
21+
for_each = length(var.permissions) == 0 ? [] : [for k in var.permissions : k if contains(keys(k), "group")]
22+
content {
23+
id = lookup(var.ad_groups, ace.value["group"], "default")
24+
permissions = ace.value["permissions"]
25+
scope = ace.value["scope"]
26+
type = ace.value["type"]
27+
}
28+
}
29+
dynamic "ace" {
30+
for_each = length(var.permissions) == 0 ? [] : [for k in var.permissions : k if contains(keys(k), "user")]
31+
content {
32+
id = ace.value["user"]
33+
permissions = ace.value["permissions"]
34+
scope = ace.value["scope"]
35+
type = ace.value["type"]
36+
}
37+
}
38+
dynamic "ace" {
39+
for_each = var.ace_default
40+
content {
41+
permissions = ace.value["permissions"]
42+
scope = ace.value["scope"]
43+
type = ace.value["type"]
44+
}
45+
}
46+
}
47+
48+
resource "null_resource" "create_root_folder" {
49+
triggers = {
50+
build_number = "${timestamp()}${azurerm_storage_data_lake_gen2_filesystem.this.id}"
51+
}
52+
provisioner "local-exec" {
53+
on_failure = continue
54+
command = "bash az_create_folders.sh \"${var.storage_account_name}\" \"${azurerm_storage_data_lake_gen2_filesystem.this.name}\" \"${var.root_dir}\" \"${local.extra_acl}\""
55+
}
56+
}
57+
58+
resource "null_resource" "create_folders" {
59+
triggers = {
60+
build_number = "${timestamp()}${azurerm_storage_data_lake_gen2_filesystem.this.id}"
61+
}
62+
provisioner "local-exec" {
63+
on_failure = continue
64+
command = "bash az_create_folders.sh \"${var.storage_account_name}\" \"${azurerm_storage_data_lake_gen2_filesystem.this.name}\" \"${local.folders}\" \"${local.extra_acl}\""
65+
}
66+
depends_on = [
67+
null_resource.create_root_folder
68+
]
69+
}

‎outputs.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
output "id" {
2+
value = azurerm_storage_data_lake_gen2_filesystem.this.id
3+
description = "The ID of the Data Lake Storage Gen2 Filesystem (container ID)."
4+
}
5+
6+
output "name" {
7+
value = azurerm_storage_data_lake_gen2_filesystem.this.name
8+
description = "The name of the Data Lake Storage Gen2 Filesystem (container name)."
9+
}
10+
11+
output "storage_account_id" {
12+
value = azurerm_storage_data_lake_gen2_filesystem.this.storage_account_id
13+
description = "The ID of the Storage Account where the Data Lake Storage Gen2 Filesystem exists."
14+
}
15+
16+
output "root_path" {
17+
value = var.root_dir
18+
description = "The name of the root directory."
19+
}

‎variables.tf‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Required
2+
variable "name" {
3+
type = string
4+
description = "Name of ADLS FS to create"
5+
}
6+
7+
variable "storage_account_id" {
8+
type = string
9+
description = "ID of storage account to create ADLS in"
10+
}
11+
12+
variable "storage_account_name" {
13+
type = string
14+
description = "Name of storage account to create ADLS in"
15+
}
16+
17+
# Optional
18+
variable "ace_default" {
19+
type = list(map(string))
20+
description = "Default ACE values"
21+
default = [
22+
{ "permissions" = "---", "scope" = "access", "type" = "other" },
23+
{ "permissions" = "---", "scope" = "default", "type" = "other" },
24+
{ "permissions" = "rwx", "scope" = "access", "type" = "group" },
25+
{ "permissions" = "rwx", "scope" = "access", "type" = "mask" },
26+
{ "permissions" = "rwx", "scope" = "access", "type" = "user" },
27+
{ "permissions" = "rwx", "scope" = "default", "type" = "group" },
28+
{ "permissions" = "rwx", "scope" = "default", "type" = "mask" },
29+
{ "permissions" = "rwx", "scope" = "default", "type" = "user" },
30+
]
31+
}
32+
33+
variable "ad_groups" {
34+
type = map(string)
35+
description = "Data which is contain mapping AD group name and GUID"
36+
default = {}
37+
}
38+
39+
variable "permissions" {
40+
type = list(map(string))
41+
description = "List of ADLS FS permissions"
42+
default = [{}]
43+
}
44+
45+
variable "root_dir" {
46+
type = string
47+
description = "Name of ADLS root directory"
48+
default = "data"
49+
}
50+
51+
variable "folders" {
52+
type = list(any)
53+
description = "Name of ADLS folders to create in root directory"
54+
default = []
55+
}

‎versions.tf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
terraform {
2+
required_version = ">= 1.0.0"
3+
4+
required_providers {
5+
azurerm = {
6+
source = "hashicorp/azurerm"
7+
version = ">= 3.23.0"
8+
}
9+
null = {
10+
source = "hashicorp/null"
11+
version = ">=3.1.1"
12+
}
13+
}
14+
}

0 commit comments

Comments
 (0)