Skip to content

Commit ce7acc0

Browse files
alhendricksonalhendrickson
committed
ops: Create Examples and modules for deploying medcat to AWS, Azure and Openstack
1 parent 9ffc3ec commit ce7acc0

File tree

94 files changed

+3535
-2
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+3535
-2
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,13 @@
55
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
66
"image": "mcr.microsoft.com/devcontainers/base:jammy",
77
"features": {
8-
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
9-
"ghcr.io/devcontainers/features/python:1": {}
8+
"ghcr.io/devcontainers/features/python:1": {},
9+
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
10+
"ghcr.io/devcontainers/features/terraform:1": {},
11+
"ghcr.io/devcontainers-extra/features/ansible:2": {},
12+
"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {},
13+
"ghcr.io/devcontainers/features/aws-cli:1": {},
14+
"ghcr.io/devcontainers/features/azure-cli:1": {}
1015
},
1116

1217
// Features to add to the dev container. More info: https://containers.dev/features.

‎.gitignore‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,46 @@
22
observability/examples/simple/observability-simple
33
observability/examples/full/cogstack-observability
44
_build
5+
**/.build/
6+
7+
8+
### Terraform Git Ignore
9+
# https://github.com/github/gitignore/blob/main/Terraform.gitignore
10+
11+
# Local .terraform directories
12+
**/.terraform/
13+
14+
# .tfstate files
15+
**/*.tfstate
16+
**/*.tfstate.*
17+
18+
# Crash log files
19+
**/crash.log
20+
**/crash.*.log
21+
22+
# Exclude all .tfvars files, which are likely to contain sensitive data
23+
**/*.tfvars
24+
**/*.tfvars.json
25+
26+
# Ignore override files
27+
**/override.tf
28+
**/override.tf.json
29+
**/*_override.tf
30+
**/*_override.tf.json
31+
32+
# Ignore transient lock info files created by terraform apply
33+
**/.terraform.tfstate.lock.info
34+
35+
# Include override files you do wish to add to version control using negated pattern
36+
# !example_override.tf
37+
38+
# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
39+
# example: *tfplan*
40+
41+
# Ignore CLI configuration files
42+
**/.terraformrc
43+
**/terraform.rc
44+
545

646
# Python ignores
747
# Byte-compiled / optimized / DLL files

‎deployment/terraform/examples/aws-kubernetes/.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
export AWS_ACCESS_KEY_ID=
2+
export AWS_SECRET_ACCESS_KEY=
3+
export AWS_REGION=eu-west-1

‎deployment/terraform/examples/aws-kubernetes/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
2+
.env

‎deployment/terraform/examples/aws-kubernetes/README.md‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# AWS Deployment
2+
3+
This is an example deployment of CogStack in AWS. It will create publically accessible services, so is not suitable for production deployment.
4+
5+
The recommended deployment in AWS is based on using Kubernetes through AWS EKS.
6+
7+
This example will create a AWS EKS cluster, setup any necessary config, deploy CogStack to the cluster, and test that it is available.
8+
9+
## Usage
10+
Deployment through terraform is carried out through two terraform commands, to handle the sequencing issues between making a k8s cluster and using it in AWS.
11+
12+
### Requirements
13+
- Terraform - [Install Terraform](https://developer.hashicorp.com/terraform/install)
14+
- AWS Credentials for an account that can create and destroy resources.
15+
16+
17+
### Steps
18+
19+
### 1. Add Required Secrets for your env
20+
This readme uses environment variables for access:
21+
22+
1. See the `.env.example` file for the required details.
23+
2. Create a file `.env` with those fields set for your account.
24+
3. Execute `source .env` to set those environment variables
25+
26+
If desired, see the official documentation for other ways to provide AWS credentials https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration
27+
28+
### 2. Run Terraform
29+
Terraform is run on two modules for AWS, so we will run one terraform apply in one folder, then another terraform apply in a second folder.
30+
31+
Initial provisioning takes around 15 minutes.
32+
33+
```bash
34+
# Set AWS credentials
35+
source .env
36+
37+
# Create AWS EKS infra
38+
cd eks-cluster
39+
terraform init
40+
terraform apply --auto-approve
41+
42+
AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file)
43+
44+
# Deploy services to kubernetes
45+
cd ../kubernetes-deployment
46+
export TF_VAR_kubeconfig_file=$AWS_KUBECONFIG
47+
terraform init
48+
terraform apply --auto-approve
49+
```
50+
51+
### 3. Accessing the CogStack Platform
52+
53+
Once the deployment is complete and all services are running, you can access the CogStack platform and its components using the following URLs:
54+
55+
```bash
56+
terraform output service_urls
57+
```
58+
59+
60+
### Optional - Destroy
61+
62+
You can destroy the infra to save costs when it wont be used for a long time.
63+
64+
Do note that there is an initial cost every time the EKS infrastructure is created, looks to be around $0.50 at time of writing.
65+
66+
```bash
67+
cd ../kubernetes-deployment
68+
terraform destroy
69+
70+
cd ../eks-cluster
71+
terraform destroy
72+
```
73+
74+
75+
## Optionally use the K8s cluster as normal with the CLI
76+
After setting up the cluster, it is possible to interact directly with it using the kubectl CLI
77+
78+
The requirement is to get the KUBECONFIG file created by the terraform apply.
79+
80+
```bash
81+
# Get KUBECONFIG
82+
cd eks-cluster
83+
AWS_KUBECONFIG=$(terraform output -raw kubeconfig_file)
84+
85+
# SET KUBECONFIG
86+
export KUBECONFIG=${AWS_KUBECONFIG}
87+
```
88+
89+
Note - alternatively you could use the AWS CLI to set your kubeconfig using `aws eks update-kubeconfig --name $(terraform output -raw cluster_name)`.
90+
91+
You can then interact with kubernetes via the CLI
92+
93+
```bash
94+
# Run Medcat service
95+
helm install my-medcat oci://registry-1.docker.io/cogstacksystems/medcat-service-helm --wait --timeout 10m0s
96+
97+
# Create the ingress
98+
kubectl apply -f resources/ingress-medcat-service.yaml
99+
# Find public url
100+
kubectl get ingress
101+
```

‎deployment/terraform/examples/aws-kubernetes/eks-cluster/.terraform.lock.hcl‎

Lines changed: 125 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deployment/terraform/examples/aws-kubernetes/eks-cluster/eks.tf‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Portions of this code adapted from tha Amazon AWS terraform-aws-eks example module (Apache 2.0):
2+
# https://github.com/terraform-aws-modules/terraform-aws-eks/blob/v21.0.4/examples/eks-auto-mode/README.md#module_eks
3+
4+
data "aws_availability_zones" "available" {
5+
# Exclude local zones
6+
filter {
7+
name = "opt-in-status"
8+
values = ["opt-in-not-required"]
9+
}
10+
}
11+
12+
locals {
13+
name = "ex-${basename(path.cwd)}"
14+
kubernetes_version = "1.33"
15+
region = "eu-west-1"
16+
17+
vpc_cidr = "10.0.0.0/16"
18+
azs = slice(data.aws_availability_zones.available.names, 0, 3)
19+
20+
tags = {
21+
Test = local.name
22+
GithubRepo = "cogstack-devops"
23+
GithubOrg = "CogStack"
24+
}
25+
}
26+
27+
################################################################################
28+
# EKS Module
29+
################################################################################
30+
31+
module "eks" {
32+
source = "terraform-aws-modules/eks/aws"
33+
version = "21.0.4"
34+
name = local.name
35+
kubernetes_version = local.kubernetes_version
36+
endpoint_public_access = true
37+
38+
enable_cluster_creator_admin_permissions = true
39+
40+
compute_config = {
41+
enabled = true
42+
node_pools = ["general-purpose"]
43+
}
44+
45+
vpc_id = module.vpc.vpc_id
46+
subnet_ids = module.vpc.private_subnets
47+
48+
tags = local.tags
49+
}
50+
51+
################################################################################
52+
# Supporting Resources
53+
################################################################################
54+
55+
module "vpc" {
56+
source = "terraform-aws-modules/vpc/aws"
57+
version = "~> 6.0"
58+
59+
name = local.name
60+
cidr = local.vpc_cidr
61+
62+
azs = local.azs
63+
private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
64+
public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]
65+
intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)]
66+
67+
enable_nat_gateway = true
68+
single_nat_gateway = true
69+
70+
public_subnet_tags = {
71+
"kubernetes.io/role/elb" = 1
72+
}
73+
74+
private_subnet_tags = {
75+
"kubernetes.io/role/internal-elb" = 1
76+
}
77+
78+
tags = local.tags
79+
}

‎deployment/terraform/examples/aws-kubernetes/eks-cluster/main.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
2+
resource "null_resource" "copy_kubeconfig" {
3+
depends_on = [module.eks, module.vpc]
4+
5+
provisioner "local-exec" {
6+
# Extract the kubeconfig file using the AWS CLI. Save it as a local file
7+
command = <<EOT
8+
aws eks update-kubeconfig --name ${module.eks.cluster_name} --kubeconfig ${local.kubeconfig_file}
9+
EOT
10+
}
11+
}

0 commit comments

Comments
 (0)