Skip to content

Commit 7069665

Browse files
sspaetisspaeti
committed
init
0 parents  commit 7069665

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1617
-0
lines changed

‎.github/workflows/deploy.yaml‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
name: Deploy with Flux
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'domains/**'
9+
- 'infrastructure/**'
10+
- 'platform/**'
11+
- 'pipelines/**'
12+
- 'clusters/**'
13+
workflow_dispatch:
14+
inputs:
15+
environment:
16+
description: 'Environment to deploy to'
17+
required: true
18+
default: 'dev'
19+
type: choice
20+
options:
21+
- dev
22+
- test
23+
- prod
24+
25+
env:
26+
REGISTRY: ghcr.io
27+
REGISTRY_USERNAME: ${{ github.actor }}
28+
REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
29+
KUBECONFIG: ${{ github.workspace }}/.kube/config
30+
31+
jobs:
32+
validate:
33+
name: Validate Manifests
34+
runs-on: ubuntu-latest
35+
steps:
36+
- name: Checkout
37+
uses: actions/checkout@v3
38+
39+
- name: Setup Kustomize
40+
uses: fluxcd/pkg//actions/kustomize@main
41+
42+
- name: Validate Kustomize overlays
43+
run: |
44+
kustomize build clusters/dev | kubeval --ignore-missing-schemas
45+
kustomize build infrastructure/overlays/dev | kubeval --ignore-missing-schemas
46+
kustomize build platform/overlays/dev | kubeval --ignore-missing-schemas
47+
kustomize build domains/finance/overlays/dev | kubeval --ignore-missing-schemas
48+
49+
build-docker-images:
50+
name: Build and Push Images
51+
runs-on: ubuntu-latest
52+
needs: validate
53+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
54+
strategy:
55+
matrix:
56+
image:
57+
- path: pipelines/ingest/example-pipeline
58+
name: data-platform/ingest-example
59+
- path: pipelines/transform
60+
name: data-platform/transform
61+
62+
steps:
63+
- name: Checkout
64+
uses: actions/checkout@v3
65+
66+
- name: Set up Docker Buildx
67+
uses: docker/setup-buildx-action@v2
68+
69+
- name: Login to Container Registry
70+
uses: docker/login-action@v2
71+
with:
72+
registry: ${{ env.REGISTRY }}
73+
username: ${{ env.REGISTRY_USERNAME }}
74+
password: ${{ env.REGISTRY_PASSWORD }}
75+
76+
- name: Extract metadata
77+
id: meta
78+
uses: docker/metadata-action@v4
79+
with:
80+
images: ${{ env.REGISTRY }}/${{ matrix.image.name }}
81+
tags: |
82+
type=sha,format=short
83+
type=ref,event=branch
84+
85+
- name: Build and push
86+
uses: docker/build-push-action@v4
87+
with:
88+
context: ${{ matrix.image.path }}
89+
push: true
90+
tags: ${{ steps.meta.outputs.tags }}
91+
labels: ${{ steps.meta.outputs.labels }}
92+
cache-from: type=gha
93+
cache-to: type=gha,mode=max
94+
95+
update-image-tags:
96+
name: Update Image Tags
97+
runs-on: ubuntu-latest
98+
needs: build-docker-images
99+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
100+
steps:
101+
- name: Checkout
102+
uses: actions/checkout@v3
103+
with:
104+
fetch-depth: 0
105+
106+
- name: Setup Flux CLI
107+
uses: fluxcd/flux2/action@main
108+
109+
- name: Configure Git
110+
run: |
111+
git config --global user.name "GitHub Actions"
112+
git config --global user.email "[email protected]"
113+
114+
- name: Update image tags in manifests
115+
run: |
116+
# Example: Use the latest SHA tag to update image references
117+
# This would typically be done by Flux's image automation controllers
118+
# but we're doing it manually here as an example
119+
REPO="${GITHUB_REPOSITORY}"
120+
SHA="${GITHUB_SHA::7}"
121+
122+
# Update transform image
123+
sed -i "s|image: ${REGISTRY}/data-platform/transform:.*|image: ${REGISTRY}/data-platform/transform:sha-${SHA}|g" \
124+
domains/finance/overlays/dev/dbt-jobs.yaml
125+
126+
# Update ingest image
127+
sed -i "s|image: ${REGISTRY}/data-platform/ingest-example:.*|image: ${REGISTRY}/data-platform/ingest-example:sha-${SHA}|g" \
128+
domains/finance/overlays/dev/airflow-dags.yaml
129+
130+
git add domains/finance/overlays/dev/dbt-jobs.yaml domains/finance/overlays/dev/airflow-dags.yaml
131+
git commit -m "chore: update image tags to sha-${SHA}"
132+
git push origin main
133+
134+
deploy:
135+
name: Deploy with Flux
136+
runs-on: ubuntu-latest
137+
needs: [validate, update-image-tags]
138+
if: github.event_name == 'workflow_dispatch'
139+
environment:
140+
name: ${{ github.event.inputs.environment || 'dev' }}
141+
steps:
142+
- name: Checkout
143+
uses: actions/checkout@v3
144+
145+
- name: Setup Flux CLI
146+
uses: fluxcd/flux2/action@main
147+
148+
- name: Setup kubectl
149+
uses: azure/setup-kubectl@v3
150+
with:
151+
version: 'v1.25.0'
152+
153+
- name: Azure login
154+
uses: azure/login@v1
155+
with:
156+
creds: ${{ secrets.AZURE_CREDENTIALS }}
157+
158+
- name: Get AKS credentials
159+
env:
160+
ENVIRONMENT: ${{ github.event.inputs.environment || 'dev' }}
161+
run: |
162+
az aks get-credentials \
163+
--resource-group data-platform-$ENVIRONMENT-rg \
164+
--name data-platform-$ENVIRONMENT \
165+
--admin
166+
167+
- name: Verify cluster connection
168+
run: kubectl cluster-info
169+
170+
- name: Check Flux installation
171+
run: flux check --pre
172+
173+
- name: Bootstrap or update Flux (if needed)
174+
env:
175+
ENVIRONMENT: ${{ github.event.inputs.environment || 'dev' }}
176+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
177+
REPO_URL: ${{ github.server_url }}/${{ github.repository }}
178+
run: |
179+
if ! flux check; then
180+
flux bootstrap github \
181+
--owner=${{ github.repository_owner }} \
182+
--repository=$(echo ${{ github.repository }} | cut -d '/' -f 2) \
183+
--branch=main \
184+
--path=clusters/$ENVIRONMENT \
185+
--personal \
186+
--token-auth
187+
else
188+
flux reconcile source git flux-system
189+
flux reconcile kustomization flux-system
190+
fi
191+
192+
- name: Force reconciliation of all resources
193+
run: |
194+
echo "Reconciling all Flux resources..."
195+
flux reconcile source git flux-system --reconcile
196+
flux reconcile kustomization flux-system --reconcile
197+
flux reconcile kustomization infrastructure --reconcile
198+
flux reconcile kustomization platform --reconcile
199+
flux reconcile kustomization domains --reconcile
200+
201+
- name: Wait for deployment to complete
202+
run: |
203+
echo "Waiting for all kustomizations to be ready..."
204+
flux get kustomizations --watch
205+
206+
- name: Verify deployment
207+
run: |
208+
echo "Verifying platform services..."
209+
kubectl get pods -n platform
210+
211+
echo "Verifying domain services..."
212+
kubectl get pods -n finance

0 commit comments

Comments
 (0)