Skip to content

Commit b86f23b

Browse files
committed
k8s updates
1 parent 7f99c52 commit b86f23b

File tree

3 files changed

+359
-0
lines changed

3 files changed

+359
-0
lines changed

.gitignore

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Environment files
2+
.env
3+
.test.env
4+
*.env
5+
6+
# Kubernetes secrets (NEVER commit these!)
7+
k8s/secret.yaml
8+
k8s/secrets.yaml
9+
10+
# Python
11+
__pycache__/
12+
*.py[cod]
13+
*$py.class
14+
*.so
15+
.Python
16+
*.egg
17+
*.egg-info/
18+
dist/
19+
build/
20+
.eggs/
21+
22+
# Virtual environments
23+
.venv/
24+
venv/
25+
ENV/
26+
env/
27+
28+
# IDE
29+
.vscode/
30+
.idea/
31+
*.swp
32+
*.swo
33+
*~
34+
.DS_Store
35+
36+
# Testing
37+
.pytest_cache/
38+
.coverage
39+
.coverage.*
40+
htmlcov/
41+
.tox/
42+
*.cover
43+
.hypothesis/
44+
45+
# Notebooks
46+
.ipynb_checkpoints/
47+
48+
# Logs
49+
*.log
50+
/tmp/
51+
52+
# UV/pip cache
53+
.uv/
54+
uv.lock
55+
56+
# Data directories (local development)
57+
data/*.csv
58+
data/*.parquet
59+
data/*.db
60+
data/*.lmdb
61+
62+
# Build artifacts
63+
*.tar.gz
64+
*.zip

k8s/deploy-to-gke.sh

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# GKE Deployment Script for AMP ERC20 Loader
5+
# This script automates the deployment of the ERC20 loader to Google Kubernetes Engine
6+
7+
# Colors for output
8+
RED='\033[0;31m'
9+
GREEN='\033[0;32m'
10+
YELLOW='\033[1;33m'
11+
NC='\033[0m' # No Color
12+
13+
# Configuration (override with environment variables)
14+
PROJECT_ID="${GCP_PROJECT_ID:-}"
15+
CLUSTER_NAME="${GKE_CLUSTER_NAME:-staging}"
16+
REGION="${GKE_REGION:-us-central1}"
17+
ZONE="${GKE_ZONE:-us-central1-a}"
18+
NODE_MACHINE_TYPE="${GKE_MACHINE_TYPE:-n1-highmem-4}"
19+
NUM_NODES="${GKE_NUM_NODES:-1}"
20+
21+
# Function to print colored output
22+
print_info() {
23+
echo -e "${GREEN}[INFO]${NC} $1"
24+
}
25+
26+
print_warning() {
27+
echo -e "${YELLOW}[WARN]${NC} $1"
28+
}
29+
30+
print_error() {
31+
echo -e "${RED}[ERROR]${NC} $1"
32+
}
33+
34+
# Function to check if command exists
35+
command_exists() {
36+
command -v "$1" >/dev/null 2>&1
37+
}
38+
39+
# Check prerequisites
40+
print_info "Checking prerequisites..."
41+
42+
if ! command_exists gcloud; then
43+
print_error "gcloud CLI is not installed. Please install it from https://cloud.google.com/sdk/docs/install"
44+
exit 1
45+
fi
46+
47+
if ! command_exists kubectl; then
48+
print_error "kubectl is not installed. Please install it: gcloud components install kubectl"
49+
exit 1
50+
fi
51+
52+
# Get or prompt for project ID
53+
if [ -z "$PROJECT_ID" ]; then
54+
print_warning "GCP_PROJECT_ID not set. Attempting to get from gcloud config..."
55+
PROJECT_ID=$(gcloud config get-value project 2>/dev/null)
56+
57+
if [ -z "$PROJECT_ID" ]; then
58+
print_error "Could not determine GCP project ID."
59+
echo "Please set it with: export GCP_PROJECT_ID=your-project-id"
60+
exit 1
61+
fi
62+
fi
63+
64+
print_info "Using GCP Project: $PROJECT_ID"
65+
print_info "Using Cluster: $CLUSTER_NAME in $REGION"
66+
67+
# SAFETY CHECK: Ensure project contains "staging"
68+
if [[ ! "$PROJECT_ID" =~ staging ]]; then
69+
print_error "SAFETY CHECK FAILED!"
70+
print_error "Project ID must contain 'staging' for deployment safety."
71+
print_error "Current project: $PROJECT_ID"
72+
echo ""
73+
print_error "This prevents accidental deployment to production."
74+
print_error "If you need to deploy to a different environment, update this script."
75+
exit 1
76+
fi
77+
78+
print_info "Safety check passed: Project contains 'staging'"
79+
80+
# Set the project
81+
gcloud config set project "$PROJECT_ID"
82+
83+
# Check if cluster exists
84+
print_info "Checking if GKE cluster exists..."
85+
if gcloud container clusters describe "$CLUSTER_NAME" --region="$REGION" >/dev/null 2>&1; then
86+
print_info "Cluster '$CLUSTER_NAME' already exists. Using existing cluster."
87+
else
88+
print_warning "Cluster '$CLUSTER_NAME' does not exist."
89+
read -p "Do you want to create it? (y/n) " -n 1 -r
90+
echo
91+
if [[ $REPLY =~ ^[Yy]$ ]]; then
92+
print_info "Creating GKE cluster '$CLUSTER_NAME'..."
93+
gcloud container clusters create "$CLUSTER_NAME" \
94+
--region="$REGION" \
95+
--machine-type="$NODE_MACHINE_TYPE" \
96+
--num-nodes="$NUM_NODES" \
97+
--enable-autoscaling \
98+
--min-nodes=1 \
99+
--max-nodes=3 \
100+
--enable-autorepair \
101+
--enable-autoupgrade \
102+
--disk-size=50 \
103+
--disk-type=pd-standard
104+
105+
print_info "Cluster created successfully!"
106+
else
107+
print_error "Cannot proceed without a cluster. Exiting."
108+
exit 1
109+
fi
110+
fi
111+
112+
# Get cluster credentials
113+
print_info "Getting cluster credentials..."
114+
gcloud container clusters get-credentials "$CLUSTER_NAME" --region="$REGION"
115+
116+
# Create namespace if it doesn't exist
117+
#print_info "Ensuring namespace exists..."
118+
#kubectl create namespace amp-loader --dry-run=client -o yaml | kubectl apply -f -
119+
120+
# Set current context to namespace
121+
kubectl config set-context --current --namespace=nozzle
122+
123+
# Create secrets from .env file if it exists
124+
if [ -f "../.test.env" ]; then
125+
print_info "Creating Kubernetes secrets from .env file..."
126+
127+
# Source the .env file
128+
export $(cat ../.env | grep -v '^#' | xargs)
129+
130+
# Create the amp-secrets secret
131+
kubectl create secret generic amp-secrets \
132+
--from-literal=amp-server-url="$AMP_SERVER_GOOGLE_CLOUD_URL" \
133+
--from-literal=snowflake-account="$SNOWFLAKE_ACCOUNT" \
134+
--from-literal=snowflake-user="$SNOWFLAKE_USER" \
135+
--from-literal=snowflake-warehouse="$SNOWFLAKE_WAREHOUSE" \
136+
--from-literal=snowflake-database="$SNOWFLAKE_DATABASE" \
137+
--from-literal=snowflake-private-key="$SNOWFLAKE_PRIVATE_KEY" \
138+
--dry-run=client -o yaml | kubectl apply -f -
139+
140+
print_info "Secrets created successfully!"
141+
else
142+
print_warning ".env file not found. You'll need to create secrets manually."
143+
print_info "You can use: kubectl apply -f k8s/secret.yaml"
144+
fi
145+
146+
# Create GitHub Container Registry secret if credentials are available
147+
#if [ -n "$GITHUB_USERNAME" ] && [ -n "$GITHUB_PAT" ]; then
148+
# print_info "Creating GitHub Container Registry secret..."
149+
# kubectl create secret docker-registry ghcr-secret \
150+
# --docker-server=ghcr.io \
151+
# --docker-username="$GITHUB_USERNAME" \
152+
# --docker-password="$GITHUB_PAT" \
153+
# --docker-email="${GITHUB_EMAIL:[email protected]}" \
154+
# --dry-run=client -o yaml | kubectl apply -f -
155+
#
156+
# print_info "GHCR secret created successfully!"
157+
#else
158+
# print_warning "GitHub credentials not found (GITHUB_USERNAME, GITHUB_PAT)."
159+
# print_info "If using a private registry, create the secret manually:"
160+
# print_info " kubectl create secret docker-registry ghcr-secret \\"
161+
# print_info " --docker-server=ghcr.io \\"
162+
# print_info " --docker-username=YOUR_USERNAME \\"
163+
# print_info " --docker-password=YOUR_PAT"
164+
#fi
165+
166+
# Apply Kubernetes manifests
167+
print_info "Applying Kubernetes deployment..."
168+
kubectl apply -f deployment.yaml
169+
170+
# Wait for deployment to be ready
171+
print_info "Waiting for deployment to be ready..."
172+
kubectl wait --for=condition=available --timeout=300s deployment/amp-erc20-loader
173+
174+
# Get deployment status
175+
print_info "Deployment status:"
176+
kubectl get deployments
177+
kubectl get pods
178+
179+
# Show logs from the first pod
180+
print_info "Fetching logs from the loader..."
181+
POD_NAME=$(kubectl get pods -l app=amp-erc20-loader -o jsonpath='{.items[0].metadata.name}')
182+
echo ""
183+
print_info "Pod name: $POD_NAME"
184+
echo ""
185+
print_info "Recent logs (last 50 lines):"
186+
kubectl logs "$POD_NAME" --tail=50
187+
188+
echo ""
189+
print_info "Deployment completed successfully!"
190+
echo ""
191+
print_info "Useful commands:"
192+
echo " - View logs: kubectl logs -f deployment/amp-erc20-loader"
193+
echo " - Get pod status: kubectl get pods -l app=amp-erc20-loader"
194+
echo " - Describe pod: kubectl describe pod $POD_NAME"
195+
echo " - Delete deployment: kubectl delete -f k8s/deployment.yaml"
196+
echo " - Scale deployment: kubectl scale deployment/amp-erc20-loader --replicas=2"
197+
echo ""

k8s/deployment.yaml

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: amp-erc20-loader
5+
labels:
6+
app: amp-erc20-loader
7+
version: v1
8+
spec:
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: amp-erc20-loader
13+
template:
14+
metadata:
15+
labels:
16+
app: amp-erc20-loader
17+
version: v1
18+
spec:
19+
containers:
20+
- name: loader
21+
image: ghcr.io/edgeandnode/amp-python:latest
22+
imagePullPolicy: Always
23+
24+
# Command line arguments for the loader
25+
args:
26+
- "--blocks"
27+
- "10000000"
28+
- "--workers"
29+
- "8"
30+
- "--flush-interval"
31+
- "0.5"
32+
33+
# Environment variables from secrets
34+
env:
35+
- name: AMP_SERVER_URL
36+
valueFrom:
37+
secretKeyRef:
38+
name: amp-secrets
39+
key: amp-server-url
40+
- name: SNOWFLAKE_ACCOUNT
41+
valueFrom:
42+
secretKeyRef:
43+
name: amp-secrets
44+
key: snowflake-account
45+
- name: SNOWFLAKE_USER
46+
valueFrom:
47+
secretKeyRef:
48+
name: amp-secrets
49+
key: snowflake-user
50+
- name: SNOWFLAKE_WAREHOUSE
51+
valueFrom:
52+
secretKeyRef:
53+
name: amp-secrets
54+
key: snowflake-warehouse
55+
- name: SNOWFLAKE_DATABASE
56+
valueFrom:
57+
secretKeyRef:
58+
name: amp-secrets
59+
key: snowflake-database
60+
- name: SNOWFLAKE_PRIVATE_KEY
61+
valueFrom:
62+
secretKeyRef:
63+
name: amp-secrets
64+
key: snowflake-private-key
65+
- name: PYTHONUNBUFFERED
66+
value: "1"
67+
- name: PYTHONPATH
68+
value: "/app"
69+
70+
# Resource allocation
71+
resources:
72+
requests:
73+
memory: "2Gi"
74+
cpu: "4"
75+
limits:
76+
memory: "4Gi"
77+
cpu: "12"
78+
79+
# Security context
80+
securityContext:
81+
runAsNonRoot: true
82+
runAsUser: 1000
83+
allowPrivilegeEscalation: false
84+
readOnlyRootFilesystem: false
85+
86+
# Image pull secrets for private GitHub Container Registry
87+
imagePullSecrets:
88+
- name: docker-registry
89+
90+
# Tolerations to allow scheduling on tainted nodes
91+
tolerations:
92+
- key: "app"
93+
operator: "Equal"
94+
value: "nozzle"
95+
effect: "NoSchedule"
96+
97+
# Restart policy
98+
restartPolicy: Always

0 commit comments

Comments
 (0)