Skip to content

Commit 675138b

Browse files
ionelpanaitescuionelpanaitescu
committed
new upgraded datalake house solution and upgraded modules
1 parent ef97bce commit 675138b

File tree

50 files changed

+5018
-3313
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+5018
-3313
lines changed

‎cloud-foundation/modules/cloud-foundation-library/data-flow_private_endpoint/main.tf

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
resource "oci_dataflow_application" "this" {
5+
for_each = var.dataflow_params
6+
compartment_id = each.value.compartment_id
7+
display_name = each.value.application_display_name
8+
driver_shape = each.value.application_driver_shape
9+
executor_shape = each.value.application_executor_shape
10+
language = each.value.application_language
11+
num_executors = each.value.application_num_executors
12+
spark_version = each.value.application_spark_version
13+
class_name = each.value.application_class_name
14+
defined_tags = each.value.defined_tags
15+
file_uri = each.value.application_file_uri
16+
freeform_tags = each.value.freeform_tags
17+
logs_bucket_uri = each.value.logs_bucket_uri
18+
private_endpoint_id = oci_dataflow_private_endpoint.this[each.key].id
19+
lifecycle {
20+
ignore_changes = all
21+
}
22+
}
23+
24+
resource "oci_dataflow_private_endpoint" "this" {
25+
for_each = var.dataflow_params
26+
compartment_id = each.value.compartment_id
27+
dns_zones = each.value.dns_zones
28+
subnet_id = each.value.subnet_id
29+
defined_tags = each.value.defined_tags
30+
description = each.value.description
31+
display_name = each.value.display_name
32+
freeform_tags = each.value.freeform_tags
33+
max_host_count = each.value.max_host_count
34+
nsg_ids = each.value.nsg_ids
35+
lifecycle {
36+
ignore_changes = all
37+
}
38+
}
39+

‎cloud-foundation/modules/cloud-foundation-library/data-flow_private_endpoint/outputs.tf

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
5+
output "dataflow_application" {
6+
description = "Data Flow informations."
7+
value = length(oci_dataflow_application.this) > 0 ? oci_dataflow_application.this[*] : null
8+
}
9+
10+
output "dataflow_private_endpoint" {
11+
description = "Data Flow informations."
12+
value = length(oci_dataflow_private_endpoint.this) > 0 ? oci_dataflow_private_endpoint.this[*] : null
13+
}

‎cloud-foundation/modules/cloud-foundation-library/data-flow_private_endpoint/variables.tf

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
variable "dataflow_params" {
5+
type = map(object({
6+
compartment_id = string
7+
application_display_name = string
8+
application_driver_shape = string
9+
application_executor_shape = string
10+
application_file_uri = string
11+
application_language = string
12+
application_num_executors = number
13+
application_spark_version = string
14+
application_class_name = string
15+
defined_tags = map(string)
16+
freeform_tags = map(string)
17+
logs_bucket_uri = string
18+
dns_zones = list(string)
19+
subnet_id = string
20+
description = string
21+
display_name = string
22+
max_host_count = number
23+
nsg_ids = list(string)
24+
}
25+
))
26+
}
27+

‎cloud-foundation/modules/cloud-foundation-library/data-science_private_endpoint/main.tf

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
resource "oci_datascience_project" "this" {
5+
for_each = var.datascience_params
6+
compartment_id = each.value.compartment_id
7+
description = each.value.project_description
8+
display_name = each.value.project_display_name
9+
defined_tags = each.value.defined_tags
10+
}
11+
12+
resource "oci_datascience_notebook_session" "this" {
13+
for_each = var.notebook_params
14+
compartment_id = each.value.compartment_id
15+
notebook_session_configuration_details {
16+
shape = each.value.notebook_session_notebook_session_configuration_details_shape
17+
subnet_id = each.value.subnet_id
18+
19+
block_storage_size_in_gbs = each.value.notebook_session_notebook_session_configuration_details_block_storage_size_in_gbs
20+
}
21+
22+
project_id = oci_datascience_project.this[each.value.project_name].id
23+
24+
display_name = each.value.notebook_session_display_name
25+
defined_tags = each.value.defined_tags
26+
lifecycle {
27+
ignore_changes = all
28+
}
29+
}
30+
31+

‎cloud-foundation/modules/cloud-foundation-library/data-science_private_endpoint/outputs.tf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
output "datascience" {
5+
value = length(oci_datascience_project.this) > 0 ? oci_datascience_project.this[*] : null
6+
}
7+
8+
output "notebook" {
9+
value = length(oci_datascience_notebook_session.this) > 0 ? oci_datascience_notebook_session.this[*] : null
10+
}

‎cloud-foundation/modules/cloud-foundation-library/data-science_private_endpoint/variables.tf

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
## Copyright © 2023, Oracle and/or its affiliates.
2+
## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
3+
4+
variable "datascience_params" {
5+
type = map(object({
6+
compartment_id = string
7+
project_description = string
8+
project_display_name = string
9+
defined_tags = map(string)
10+
}
11+
))
12+
}
13+
14+
variable "notebook_params" {
15+
type = map(object({
16+
project_name = string
17+
compartment_id = string
18+
notebook_session_notebook_session_configuration_details_shape = string
19+
subnet_id = string
20+
notebook_session_notebook_session_configuration_details_block_storage_size_in_gbs = number
21+
notebook_session_display_name = string
22+
defined_tags = map(string)
23+
}
24+
))
25+
}

‎cloud-foundation/modules/cloud-foundation-library/datacatalog_lakehouse/main.tf

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# Copyright © 2023, Oracle and/or its affiliates.
2+
# All rights reserved. Licensed under the Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
resource "oci_datacatalog_catalog_private_endpoint" "this" {
5+
for_each = var.datacatalog_params
6+
compartment_id = each.value.compartment_id
7+
dns_zones = each.value.private_endpoint_dns_zones
8+
subnet_id = each.value.subnet_id
9+
display_name = each.value.catalog_display_name
10+
defined_tags = each.value.defined_tags
11+
}
12+
13+
resource "oci_datacatalog_catalog" "this" {
14+
for_each = var.datacatalog_params
15+
compartment_id = each.value.compartment_id
16+
display_name = each.value.catalog_display_name
17+
defined_tags = each.value.defined_tags
18+
attached_catalog_private_endpoints = [oci_datacatalog_catalog_private_endpoint.this[each.key].id]
19+
}
20+
21+
resource "oci_datacatalog_data_asset" "this" {
22+
for_each = var.datacatalog_params
23+
catalog_id = oci_datacatalog_catalog.this[each.key].id
24+
display_name = each.value.adw_data_asset_display_name
25+
type_key = data.oci_datacatalog_catalog_types.this[each.key].type_collection[0].items[0].key
26+
properties = {
27+
"default.database" = var.db_name
28+
"default.privateendpoint" = "true"
29+
}
30+
}
31+
32+
resource "oci_datacatalog_data_asset" "that" {
33+
for_each = var.datacatalog_params
34+
catalog_id = oci_datacatalog_catalog.this[each.key].id
35+
display_name = each.value.object_storage_data_asset_display_name
36+
type_key = data.oci_datacatalog_catalog_types.objectstorage_data_asset[each.key].type_collection[0].items[0].key
37+
properties = {
38+
"default.namespace" = data.oci_objectstorage_namespace.os.namespace
39+
"default.url" = "https://swiftobjectstorage.${var.region}.oraclecloud.com"
40+
}
41+
}
42+
43+
# resource "oci_datacatalog_connection" "this" {
44+
# for_each = var.datacatalog_params
45+
# catalog_id = oci_datacatalog_catalog.this[each.key].id
46+
# data_asset_key = oci_datacatalog_data_asset.this[each.key].id
47+
# display_name = each.value.catalog_display_name
48+
# is_default = "false"
49+
# type_key = data.oci_datacatalog_catalog_types.that[each.key].type_collection[0].items[2].key
50+
# properties = {
51+
# "default.alias" = "${var.db_name}_low"
52+
# "default.username" = each.value.dbusername
53+
# "default.walletAndSecrets" = "plainWallet"
54+
# }
55+
# enc_properties = {
56+
# "default.password" = each.value.dbpassword
57+
# "default.wallet" = var.wallet
58+
# }
59+
# }
60+
61+
resource "oci_datacatalog_connection" "that" {
62+
for_each = var.datacatalog_params
63+
catalog_id = oci_datacatalog_catalog.this[each.key].id
64+
data_asset_key = oci_datacatalog_data_asset.that[each.key].id
65+
display_name = each.value.object_storage_data_asset_display_name
66+
is_default = "true"
67+
properties = {
68+
"default.ociCompartment" = each.value.compartment_id
69+
"default.ociRegion" = var.region
70+
}
71+
type_key = data.oci_datacatalog_catalog_types.resource_principal_connection[each.key].type_collection[0].items[0].key
72+
}
73+
74+
data "oci_datacatalog_catalog_types" "this" {
75+
for_each = var.datacatalog_params
76+
catalog_id = oci_datacatalog_catalog.this[each.key].id
77+
type_category = "dataAsset"
78+
name = "Autonomous Data Warehouse"
79+
state = "ACTIVE"
80+
}
81+
82+
83+
data "oci_datacatalog_catalog_types" "that" {
84+
for_each = var.datacatalog_params
85+
catalog_id = oci_datacatalog_catalog.this[each.key].id
86+
type_category = "connection"
87+
name = "Generic"
88+
state = "ACTIVE"
89+
}
90+
91+
data "oci_datacatalog_catalog_types" "objectstorage_data_asset" {
92+
for_each = var.datacatalog_params
93+
catalog_id = oci_datacatalog_catalog.this[each.key].id
94+
95+
filter {
96+
name = "name"
97+
values = ["Oracle Object Storage"]
98+
}
99+
filter {
100+
name = "type_category"
101+
values = ["dataAsset"]
102+
}
103+
}
104+
105+
data "oci_datacatalog_catalog_types" "resource_principal_connection" {
106+
for_each = var.datacatalog_params
107+
catalog_id = oci_datacatalog_catalog.this[each.key].id
108+
109+
filter {
110+
name = "name"
111+
values = ["Resource Principal"]
112+
}
113+
filter {
114+
name = "type_category"
115+
values = ["connection"]
116+
}
117+
}
118+
119+
data "oci_datacatalog_catalog_private_endpoint" "this" {
120+
for_each = var.datacatalog_params
121+
catalog_private_endpoint_id = oci_datacatalog_catalog_private_endpoint.this[each.key].id
122+
}
123+
124+
data "oci_objectstorage_namespace" "os" {
125+
compartment_id = var.tenancy_ocid
126+
}
127+

‎cloud-foundation/modules/cloud-foundation-library/datacatalog_lakehouse/outputs.tf

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright © 2023, Oracle and/or its affiliates.
2+
# All rights reserved. Licensed under the Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
output "datacatalog" {
5+
value = {
6+
for datacatalog in oci_datacatalog_catalog.this:
7+
datacatalog.display_name => datacatalog.display_name
8+
}
9+
}
10+
11+
output "datacatalog_data_asset_adw" {
12+
value = {
13+
for datacatalog_data_asset in oci_datacatalog_data_asset.this:
14+
datacatalog_data_asset.display_name => datacatalog_data_asset.display_name
15+
}
16+
}
17+
18+
output "datacatalog_data_asset_object_storage" {
19+
value = {
20+
for datacatalog_data_asset in oci_datacatalog_data_asset.that:
21+
datacatalog_data_asset.display_name => datacatalog_data_asset.display_name
22+
}
23+
}

‎cloud-foundation/modules/cloud-foundation-library/datacatalog_lakehouse/variables.tf

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright © 2023, Oracle and/or its affiliates.
2+
# All rights reserved. Licensed under the Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
variable "tenancy_ocid" {
5+
type = string
6+
}
7+
8+
variable "region" {
9+
type = string
10+
}
11+
12+
variable "datacatalog_params" {
13+
type = map(object({
14+
compartment_id = string
15+
catalog_display_name = string
16+
adw_data_asset_display_name = string
17+
object_storage_data_asset_display_name = string
18+
defined_tags = map(string)
19+
private_endpoint_dns_zones = list(string)
20+
subnet_id = string
21+
# dbusername = string
22+
# dbpassword = string
23+
}))
24+
}
25+
26+
variable "db_name" {
27+
type = string
28+
}
29+
30+
# variable "wallet" {}

0 commit comments

Comments
 (0)