Skip to content

Commit c385243

Browse files
gurinder-s-brargurinder-s-brar
authored
feat(opensearch): FTRS-856 Index creation and population (#673)
* feat(opensearch): FTRS-856 Index creation and population
1 parent 4b2a024 commit c385243

25 files changed

+3315
-2
lines changed

‎.github/actions/create-open-search-index/action.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: 'Create open search index action'
2+
description: 'Creates the required index in open search'
3+
4+
inputs:
5+
workspace:
6+
description: 'The name of the workspace'
7+
required: false
8+
default: ''
9+
environment:
10+
description: 'Environment to authenticate against (e.g., dev, test, prod)'
11+
required: true
12+
domain:
13+
description: 'The name of the collection to target (defaults to ftrs-dos-dev-osc)'
14+
required: true
15+
default: 'ftrs-dos-dev-osc'
16+
index:
17+
description: 'The OpenSearch index name to create'
18+
required: true
19+
default: 'triage_code'
20+
aws_region:
21+
description: 'AWS region to use for AWS CLI and boto3 calls (optional)'
22+
required: false
23+
default: ''
24+
outputs:
25+
endpoint:
26+
description: 'Resolved OpenSearch collection endpoint'
27+
value: '${{ steps.create_index.outputs.endpoint }}'
28+
final_index:
29+
description: 'Final index name created'
30+
value: '${{ steps.create_index.outputs.final_index }}'
31+
runs:
32+
using: 'composite'
33+
steps:
34+
- name: Create OpenSearch index
35+
id: create_index
36+
shell: bash
37+
env:
38+
INDEX: ${{ inputs.index }}
39+
OPEN_SEARCH_DOMAIN: ${{ inputs.domain }}
40+
WORKSPACE: ${{ inputs.workspace }}
41+
ENVIRONMENT: ${{ inputs.environment }}
42+
AWS_REGION: ${{ inputs.aws_region }}
43+
run: |
44+
set -euo pipefail
45+
46+
echo "Using OPEN_SEARCH_DOMAIN=${OPEN_SEARCH_DOMAIN} INDEX=${INDEX} WORKSPACE=${WORKSPACE} AWS_REGION=${AWS_REGION:-<unset>}"
47+
48+
python3 -m pip install --upgrade pip setuptools wheel
49+
50+
python3 -m pip install boto3 requests
51+
52+
python3 ./scripts/workflow/create_open_search_index.py

‎.github/actions/populate-open-search-index/action.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: 'Populate open search index action'
2+
description: 'Populates the required index in open search'
3+
4+
inputs:
5+
workspace:
6+
description: 'The name of the workspace'
7+
required: false
8+
default: ''
9+
environment:
10+
description: 'Environment to authenticate against (e.g., dev, test, prod)'
11+
required: true
12+
aws_region:
13+
description: 'AWS region to use for AWS CLI and boto3 calls (optional)'
14+
required: false
15+
default: ''
16+
table:
17+
description: 'The DynamoDB table name to use'
18+
required: true
19+
default: 'ftrs-dos-dev-database-healthcare-service'
20+
endpoint:
21+
description: 'Resolved OpenSearch endpoint (from build action)'
22+
required: true
23+
default: ''
24+
final_index:
25+
description: 'Final index name created (from build action)'
26+
required: true
27+
default: ''
28+
runs:
29+
using: 'composite'
30+
steps:
31+
- name: Populate OpenSearch index
32+
id: populate_index
33+
shell: bash
34+
env:
35+
WORKSPACE: ${{ inputs.workspace }}
36+
ENVIRONMENT: ${{ inputs.environment }}
37+
AWS_REGION: ${{ inputs.aws_region }}
38+
DYNAMODB_TABLE: ${{ inputs.table }}
39+
OS_ENDPOINT: ${{ inputs.endpoint }}
40+
OS_FINAL_INDEX: ${{ inputs.final_index }}
41+
run: |
42+
set -euo pipefail
43+
44+
echo "Using WORKSPACE=${WORKSPACE} AWS_REGION=${AWS_REGION:-<unset>} OS_ENDPOINT=${OS_ENDPOINT:-<unset>} OS_FINAL_INDEX=${OS_FINAL_INDEX:-<unset>}"
45+
46+
python3 -m pip install --upgrade pip setuptools wheel
47+
48+
python3 -m pip install boto3 requests
49+
50+
python3 ./scripts/workflow/populate_open_search_index.py

‎.github/workflows/deploy-open-search-indexes.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Deploy Open Search Indexes workflow
2+
3+
permissions:
4+
id-token: write
5+
contents: read
6+
7+
on:
8+
workflow_call:
9+
inputs:
10+
environment:
11+
description: "Environment to authenticate against (for example: dev, test, prod)"
12+
required: true
13+
type: string
14+
workspace:
15+
description: "Workspace name to deploy into"
16+
required: true
17+
type: string
18+
aws_region:
19+
description: "AWS region to use for AWS CLI calls"
20+
required: false
21+
type: string
22+
default: ''
23+
24+
jobs:
25+
build-index:
26+
name: Build index
27+
runs-on: ubuntu-latest
28+
environment: ${{ inputs.environment }}
29+
steps:
30+
- name: Checkout repository
31+
uses: actions/checkout@v6
32+
33+
- name: Configure AWS credentials
34+
uses: ./.github/actions/configure-credentials
35+
with:
36+
aws_account_id: ${{ secrets.ACCOUNT_ID }}
37+
aws_region: ${{ inputs.aws_region }}
38+
type: app
39+
environment: ${{ inputs.environment }}
40+
41+
- name: Create OpenSearch index
42+
id: create-index
43+
uses: ./.github/actions/create-open-search-index
44+
with:
45+
environment: ${{ inputs.environment }}
46+
workspace: ${{ inputs.workspace }}
47+
aws_region: ${{ inputs.aws_region }}
48+
49+
- name: Populate OpenSearch index
50+
id: populate-index
51+
uses: ./.github/actions/populate-open-search-index
52+
with:
53+
environment: ${{ inputs.environment }}
54+
workspace: ${{ inputs.workspace }}
55+
aws_region: ${{ inputs.aws_region }}
56+
endpoint: ${{ steps.create-index.outputs.endpoint }}
57+
final_index: ${{ steps.create-index.outputs.final_index }}

‎.github/workflows/pipeline-deploy-application.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,20 @@ jobs:
202202
type: app
203203
secrets: inherit
204204

205+
deploy-open-search-indexes:
206+
name: "Deploy OpenSearch indexes to ${{ needs.metadata.outputs.environment }}"
207+
if: false
208+
needs:
209+
- metadata
210+
- deploy-application-infrastructure
211+
- restore-dynamodb-from-s3
212+
uses: ./.github/workflows/deploy-open-search-indexes.yaml
213+
with:
214+
environment: ${{ needs.metadata.outputs.environment }}
215+
workspace: ${{ needs.metadata.outputs.workspace }}
216+
aws_region: ${{ vars.AWS_REGION }}
217+
secrets: inherit
218+
205219
deploy-frontend-services:
206220
name: "Deploy ${{ matrix.name }} to ${{ needs.metadata.outputs.environment }}"
207221
concurrency:

‎infrastructure/common/common-variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,3 +177,9 @@ variable "included_cloudfront_log_fields" {
177177
"sc-range-end"
178178
]
179179
}
180+
181+
variable "index_base" {
182+
description = "Base name of the OpenSearch index (workspace suffix will be appended at runtime)"
183+
type = string
184+
default = "triage_code"
185+
}

‎infrastructure/common/locals.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,7 @@ locals {
4444
rds = "alias/${local.project_prefix}-rds-kms"
4545
opensearch = "alias/${local.project_prefix}-opensearch-kms"
4646
}
47+
48+
# Will be used by dos-search stack
49+
opensearch_index_name = "${var.index_base}${local.workspace_suffix}"
4750
}

‎infrastructure/stacks/opensearch/opensearch_collection_policy.tf‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,83 @@ resource "aws_opensearchserverless_access_policy" "opensearch_serverless_data_ac
6464
}
6565
])
6666
}
67+
68+
// NOTE: Collection and dashboards are currently public (AllowFromPublic = true) for ease of testing.
69+
// To make the collection private later, create VPCE(s) in the account-wide stack and supply their IDs
70+
// to this stack, then set AllowFromPublic = false and set SourceVPCEs.
71+
// This should be done once a Lambda (or any other compute running inside a VPC) needs to read or write
72+
// indexes in the collection, so that access occurs over PrivateLink instead of the public endpoint.
73+
74+
resource "aws_opensearchserverless_security_policy" "opensearch_serverless_workspace_network_access_policy" {
75+
count = local.workspace_suffix == "" ? 0 : 1
76+
name = "${var.environment}-${var.stack_name}${local.workspace_suffix}-nap"
77+
description = "Workspace-level network access for collection dashboards and collection endpoint"
78+
type = "network"
79+
80+
policy = jsonencode([
81+
{
82+
Description = "Workspace dashboard access"
83+
AllowFromPublic = true
84+
Rules = [
85+
{
86+
Resource = ["collection/${data.aws_opensearchserverless_collection.opensearch_serverless_collection.name}"]
87+
ResourceType = "dashboard"
88+
}
89+
]
90+
},
91+
{
92+
Description = "Workspace network access for collection"
93+
AllowFromPublic = true
94+
Rules = [
95+
{
96+
Resource = ["collection/${data.aws_opensearchserverless_collection.opensearch_serverless_collection.name}"]
97+
ResourceType = "collection"
98+
}
99+
]
100+
}
101+
])
102+
}
103+
104+
resource "aws_opensearchserverless_access_policy" "opensearch_serverless_workspace_data_access_policy" {
105+
count = local.workspace_suffix == "" ? 0 : 1
106+
107+
name = "${var.environment}-${var.stack_name}${local.workspace_suffix}-dap"
108+
type = "data"
109+
description = "Collection-level data access policy for OpenSearch collection ${data.aws_opensearchserverless_collection.opensearch_serverless_collection.name} (grants collection & index ops)"
110+
111+
policy = jsonencode([
112+
{
113+
Rules = [
114+
{
115+
ResourceType = "collection"
116+
Resource = ["collection/${data.aws_opensearchserverless_collection.opensearch_serverless_collection.name}"]
117+
Permission = [
118+
"aoss:CreateCollectionItems",
119+
"aoss:UpdateCollectionItems",
120+
"aoss:DescribeCollectionItems",
121+
"aoss:DeleteCollectionItems"
122+
]
123+
},
124+
{
125+
ResourceType = "index"
126+
Resource = ["index/${data.aws_opensearchserverless_collection.opensearch_serverless_collection.name}/${local.opensearch_index_name}"]
127+
Permission = [
128+
"aoss:CreateIndex",
129+
"aoss:UpdateIndex",
130+
"aoss:DescribeIndex",
131+
"aoss:DeleteIndex",
132+
"aoss:ReadDocument",
133+
"aoss:WriteDocument"
134+
]
135+
}
136+
],
137+
Principal = concat(
138+
[
139+
data.aws_caller_identity.current.arn,
140+
aws_iam_role.osis_pipelines_role.arn
141+
],
142+
local.env_sso_roles
143+
)
144+
}
145+
])
146+
}

‎scripts/config/sonar-scanner.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ sonar.host.url=https://sonarcloud.io
44
sonar.qualitygate.wait=true
55
sonar.sourceEncoding=UTF-8
66
sonar.sources=.
7-
sonar.coverage.exclusions= infrastructure/**, tests/**, **/*.test.ts, **/*.spec.ts, tests/ui/src/**
8-
sonar.exclusions=tests/**, /**/__tests__/**, */*.test.tsx, **/*.test.ts, **/*test*.py, **/test_*.py, **/tests/unit/**
7+
sonar.coverage.exclusions= infrastructure/**, tests/**, scripts/workflow/tests/**, **/*.test.ts, **/*.spec.ts, tests/ui/src/**, scripts/workflow/create_open_search_index.py, scripts/workflow/populate_open_search_index.py
8+
sonar.exclusions=tests/**, scripts/workflow/tests/**, /**/__tests__/**, */*.test.tsx, **/*.test.ts, **/*test*.py, **/test_*.py, **/tests/unit/**
99

1010
#sonar.python.coverage.reportPaths=.coverage/coverage.xml
1111
#sonar.[javascript|typescript].lcov.reportPaths=.coverage/lcov.info

0 commit comments

Comments
 (0)