Skip to content

Commit 2fdd722

Browse files
mattdean3-nhsmattdean3-nhs
authored
Merge pull request #670 from NHSDigital/feature/made14-NRL-762-rollback-pipeline
[NRL-762] Create rollback pipeline for blue/green
2 parents c0a21c6 + 7938833 commit 2fdd722

File tree

7 files changed

+156
-14
lines changed

7 files changed

+156
-14
lines changed

‎.github/workflows/activate-stack.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Switch Active Stack
2+
run-name: Switch active stack to ${{ inputs.stack_name }} in ${{ inputs.environment }} by ${{ github.actor }}
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
environment:
8+
description: "Environment to activate the stack in"
9+
required: true
10+
default: "dev"
11+
type: environment
12+
13+
stack_name:
14+
description: Name of stack to activate
15+
required: true
16+
type: string
17+
18+
permissions:
19+
id-token: write
20+
contents: read
21+
actions: write
22+
23+
jobs:
24+
activate-stack:
25+
name: Activate ${{ inputs.stack_name }} for ${{ inputs.environment }}
26+
runs-on: [self-hosted, ci]
27+
environment: ${{ inputs.environment }}
28+
29+
steps:
30+
- name: Git clone - ${{ github.ref }}
31+
uses: actions/checkout@v4
32+
with:
33+
ref: ${{ github.ref }}
34+
35+
- name: Setup asdf cache
36+
uses: actions/cache@v4
37+
with:
38+
path: ~/.asdf
39+
key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }}
40+
restore-keys: |
41+
${{ runner.os }}-asdf-
42+
43+
- name: Install asdf
44+
uses: asdf-vm/actions/[email protected]
45+
46+
- name: Configure Management Credentials
47+
uses: aws-actions/configure-aws-credentials@v4
48+
with:
49+
aws-region: eu-west-2
50+
role-to-assume: ${{ secrets.MGMT_ROLE_ARN }}
51+
role-session-name: github-actions-ci-${{ inputs.environment }}-${{ github.run_id}}
52+
53+
- name: Install zip
54+
run: sudo apt-get install zip
55+
56+
- name: Setup Python environment
57+
run: |
58+
poetry install --no-root
59+
source $(poetry env info --path)/bin/activate
60+
61+
- name: Get current environment config
62+
run: |
63+
poetry run python ./scripts/get_env_config.py all ${{ inputs.environment }}
64+
65+
- name: Activate Stack
66+
run: |
67+
poetry run python ./scripts/activate-stack.py ${{ inputs.stack_name }} ${{ inputs.environment }}

‎.github/workflows/persistent-environment.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,4 +339,4 @@ jobs:
339339
- name: Deactivate Stack
340340
run: |
341341
inactive_stack_name=$(poetry run python ./scripts/get_env_config.py inactive-stack ${{ inputs.environment }})
342-
poetry run python ./scripts/activate-stack.py ${inactive_stack_name}
342+
poetry run python ./scripts/activate-stack.py ${inactive_stack_name} ${{ inputs.environment }}

‎.github/workflows/rollback-stack.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Rollback Stack
2+
run-name: Rollback to inactive stack in ${{ inputs.environment }} by ${{ github.actor }}
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
environment:
8+
description: "Environment to rollback the stack in"
9+
required: true
10+
default: "dev"
11+
type: environment
12+
13+
permissions:
14+
id-token: write
15+
contents: read
16+
actions: write
17+
18+
jobs:
19+
rollback-stack:
20+
name: Rollback to inactive stack for ${{ inputs.environment }}
21+
runs-on: [self-hosted, ci]
22+
environment: ${{ inputs.environment }}
23+
24+
steps:
25+
- name: Git clone - ${{ github.ref }}
26+
uses: actions/checkout@v4
27+
with:
28+
ref: ${{ github.ref }}
29+
30+
- name: Setup asdf cache
31+
uses: actions/cache@v4
32+
with:
33+
path: ~/.asdf
34+
key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }}
35+
restore-keys: |
36+
${{ runner.os }}-asdf-
37+
38+
- name: Install asdf
39+
uses: asdf-vm/actions/[email protected]
40+
41+
- name: Configure Management Credentials
42+
uses: aws-actions/configure-aws-credentials@v4
43+
with:
44+
aws-region: eu-west-2
45+
role-to-assume: ${{ secrets.MGMT_ROLE_ARN }}
46+
role-session-name: github-actions-ci-${{ inputs.environment }}-${{ github.run_id}}
47+
48+
- name: Install zip
49+
run: sudo apt-get install zip
50+
51+
- name: Setup Python environment
52+
run: |
53+
poetry install --no-root
54+
source $(poetry env info --path)/bin/activate
55+
56+
- name: Get current environment config
57+
run: |
58+
poetry run python ./scripts/get_env_config.py all ${{ inputs.environment }}
59+
60+
- name: Rollback
61+
run: |
62+
inactive_stack_name=$(poetry run python ./scripts/get_env_config.py inactive-stack ${{ inputs.environment }})
63+
poetry run python ./scripts/activate-stack.py ${inactive_stack_name} ${{ inputs.environment }}
64+
65+
- name: "Smoke Test"
66+
run: |
67+
make ENV=${{ inputs.environment }} test-smoke-external

‎scripts/activate_stack.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,18 +93,18 @@ def activate_stack(stack_name: str, env: str, session: any):
9393
environment_config = json.loads(response["SecretString"])
9494
print(f"Got environment config for {env}: {environment_config}")
9595

96+
current_active_stack = environment_config[CONFIG_ACTIVE_STACK]
97+
if current_active_stack == stack_name:
98+
print("Cannot activate stack, stack is already active", file=sys.stderr)
99+
sys.exit(1)
100+
96101
lock_state = environment_config[CONFIG_LOCK_STATE]
97102
if lock_state != "open":
98103
print(
99104
f"Unable to activate stack as lock state is not open: {lock_state}",
100105
file=sys.stderr,
101106
)
102-
return
103-
104-
current_active_stack = environment_config[CONFIG_ACTIVE_STACK]
105-
if current_active_stack == stack_name:
106-
print("Cannot activate stack, stack is already active", file=sys.stderr)
107-
return
107+
sys.exit(1)
108108

109109
_set_lock_state(
110110
STATE_LOCKED,
@@ -131,7 +131,7 @@ def activate_stack(stack_name: str, env: str, session: any):
131131
)
132132
print(f"Failed to activate stack: {err}", file=sys.stderr)
133133
print(f"Stack trace: {traceback.format_exc()}", file=sys.stderr)
134-
return
134+
sys.exit(1)
135135

136136
print("Updating environment config and unlocking....")
137137
environment_config[CONFIG_INACTIVE_STACK] = current_active_stack

‎scripts/get_env_config.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,16 @@ def main(parameter_name: str, env: str):
1414
response = sm.get_secret_value(SecretId=secret_key)
1515
parameters = json.loads(response["SecretString"])
1616

17-
if parameter_name in parameters:
17+
if parameter_name == "all":
18+
print(parameters)
19+
elif parameter_name in parameters:
1820
print(parameters[parameter_name])
1921
else:
2022
print(
2123
f"Parameter {parameter_name} not found in environment config",
2224
file=sys.stderr,
2325
)
26+
sys.exit(1)
2427

2528

2629
if __name__ == "__main__":

‎scripts/tests/test_activate_stack.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,10 @@ def test_lock_state_not_open(mock_boto_session, mock_secretsmanager):
103103
Name="nhsd-nrlf--locked--env-config", SecretString=json.dumps(inital_env_config)
104104
)
105105

106-
activate_stack("test-stack-1", "locked", session=mock_boto_session)
106+
with pytest.raises(SystemExit) as excinfo:
107+
activate_stack("test-stack-1", "locked", session=mock_boto_session)
107108

109+
assert excinfo.value.code == 1
108110
result = mock_secretsmanager.get_secret_value(
109111
SecretId="nhsd-nrlf--locked--env-config" # pragma: allowlist secret
110112
)
@@ -122,8 +124,10 @@ def test_stack_already_active(mock_boto_session, mock_secretsmanager):
122124
SecretString=json.dumps(intial_env_config),
123125
)
124126

125-
activate_stack("test-stack-2", "already-active", session=mock_boto_session)
127+
with pytest.raises(SystemExit) as excinfo:
128+
activate_stack("test-stack-2", "already-active", session=mock_boto_session)
126129

130+
assert excinfo.value.code == 1
127131
result = mock_secretsmanager.get_secret_value(
128132
SecretId="nhsd-nrlf--already-active--env-config" # pragma: allowlist secret
129133
)

‎terraform/infrastructure/ephemeral-resources.tf‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
module "ephemeral-s3-permission-store" {
2-
count = var.use_shared_resources ? 0 : 1
3-
source = "./modules/permissions-store-bucket"
4-
name_prefix = local.prefix
2+
count = var.use_shared_resources ? 0 : 1
3+
source = "./modules/permissions-store-bucket"
4+
name_prefix = local.prefix
5+
enable_bucket_force_destroy = true
56
}
67

78
module "ephemeral-pointers-table" {

0 commit comments

Comments
 (0)