diff --git a/.gitignore b/.gitignore index 35f9301cb11..2ad5842222a 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,10 @@ # MacOS ignores .DS_Store + +# Docker build artifacts +docker/accumulo/dist/ + +# Helm chart build artifacts +charts/accumulo/charts/ +values-generated.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..dfd597dab5d --- /dev/null +++ b/Makefile @@ -0,0 +1,230 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Variables +REGISTRY ?= accumulo +TAG ?= 4.0.0-SNAPSHOT +RELEASE_NAME ?= accumulo-dev +NAMESPACE ?= default +VALUES_FILE ?= charts/accumulo/values-dev.yaml + +# Colors +BLUE = \033[0;34m +GREEN = \033[0;32m +YELLOW = \033[1;33m +RED = \033[0;31m +NC = \033[0m # No Color + +# Helper function to print colored output +define log_info + @echo -e "$(BLUE)[INFO]$(NC) $(1)" +endef + +define log_success + @echo -e "$(GREEN)[SUCCESS]$(NC) $(1)" +endef + +define log_warning + @echo -e "$(YELLOW)[WARNING]$(NC) $(1)" +endef + +define log_error + @echo -e "$(RED)[ERROR]$(NC) $(1)" +endef + +.PHONY: help +help: ## Show this help message + @echo "Apache Accumulo with Alluxio on Kubernetes" + @echo "" + @echo "Available targets:" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @echo "" + @echo "Variables:" + @echo " REGISTRY=$(REGISTRY) - Docker registry" + @echo " TAG=$(TAG) - Docker tag" + @echo " RELEASE_NAME=$(RELEASE_NAME) - Helm release name" + @echo " NAMESPACE=$(NAMESPACE) - Kubernetes namespace" + @echo " VALUES_FILE=$(VALUES_FILE) - Helm values file" + +.PHONY: build +build: ## Build Accumulo distribution + $(call log_info,"Building Accumulo distribution...") + mvn clean package -DskipTests -pl assemble -am + $(call log_success,"Accumulo distribution built successfully") + +.PHONY: docker-build +docker-build: build ## Build Docker image + $(call log_info,"Building Docker image: $(REGISTRY)/accumulo:$(TAG)") + ./scripts/build-docker.sh -r $(REGISTRY) -t $(TAG) + $(call log_success,"Docker image built successfully") + +.PHONY: docker-push +docker-push: build ## Build and push Docker image + $(call log_info,"Building and pushing Docker image: $(REGISTRY)/accumulo:$(TAG)") + ./scripts/build-docker.sh -r $(REGISTRY) -t $(TAG) -p + $(call log_success,"Docker image built and pushed successfully") + +.PHONY: generate-config +generate-config: ## Generate configuration with secrets + $(call log_info,"Generating configuration...") + ./scripts/generate-secrets.sh -o values-generated.yaml --non-interactive -i $(RELEASE_NAME) + $(call log_success,"Configuration generated: values-generated.yaml") + +.PHONY: generate-config-interactive +generate-config-interactive: ## Generate configuration interactively + $(call log_info,"Generating configuration interactively...") + ./scripts/generate-secrets.sh -o values-generated.yaml -i $(RELEASE_NAME) + $(call log_success,"Configuration generated: values-generated.yaml") + +.PHONY: deploy-dev +deploy-dev: ## Deploy development environment + $(call log_info,"Deploying development environment...") + ./scripts/helm-deploy.sh install -r $(RELEASE_NAME) -f $(VALUES_FILE) --create-namespace -n $(NAMESPACE) + $(call log_success,"Development environment deployed successfully") + +.PHONY: deploy +deploy: generate-config ## Deploy with generated configuration + $(call log_info,"Deploying with generated configuration...") + ./scripts/helm-deploy.sh install -r $(RELEASE_NAME) -f values-generated.yaml --create-namespace -n $(NAMESPACE) + $(call log_success,"Deployment completed successfully") + +.PHONY: upgrade +upgrade: ## Upgrade existing deployment + $(call log_info,"Upgrading deployment...") + ./scripts/helm-deploy.sh upgrade -r $(RELEASE_NAME) -f $(VALUES_FILE) -n $(NAMESPACE) + $(call log_success,"Upgrade completed successfully") + +.PHONY: test +test: ## Run smoke tests + $(call log_info,"Running smoke tests...") + ./scripts/helm-deploy.sh test -r $(RELEASE_NAME) -n $(NAMESPACE) + $(call log_success,"Tests completed successfully") + +.PHONY: status +status: ## Show deployment status + ./scripts/helm-deploy.sh status -r $(RELEASE_NAME) -n $(NAMESPACE) + +.PHONY: uninstall +uninstall: ## Uninstall deployment + $(call log_warning,"Uninstalling deployment...") + ./scripts/helm-deploy.sh uninstall -r $(RELEASE_NAME) -n $(NAMESPACE) + $(call log_success,"Deployment uninstalled successfully") + +.PHONY: logs +logs: ## Show logs from all Accumulo components + $(call log_info,"Showing logs from Accumulo components...") + kubectl logs -l app.kubernetes.io/name=accumulo -n $(NAMESPACE) --tail=100 + +.PHONY: shell +shell: ## Access Accumulo shell + $(call log_info,"Connecting to Accumulo shell...") + kubectl exec -it deployment/$(RELEASE_NAME)-manager -n $(NAMESPACE) -- /opt/accumulo/bin/accumulo shell -u root + +.PHONY: port-forward +port-forward: ## Forward ports for local access + $(call log_info,"Setting up port forwarding...") + @echo "Accumulo Monitor will be available at: http://localhost:9995" + @echo "Alluxio Master will be available at: http://localhost:19999" + @echo "Press Ctrl+C to stop port forwarding" + kubectl port-forward svc/$(RELEASE_NAME)-monitor 9995:9995 -n $(NAMESPACE) & + kubectl port-forward svc/$(RELEASE_NAME)-alluxio-master 19999:19999 -n $(NAMESPACE) & + wait + +.PHONY: clean-docker +clean-docker: ## Clean up Docker images and containers + $(call log_info,"Cleaning up Docker images...") + docker images | grep $(REGISTRY)/accumulo | awk '{print $$3}' | xargs -r docker rmi -f + $(call log_success,"Docker cleanup completed") + +.PHONY: validate +validate: ## Validate Helm chart + $(call log_info,"Validating Helm chart...") + helm lint charts/accumulo + $(call log_success,"Helm chart validation passed") + +.PHONY: template +template: ## Generate Kubernetes templates + $(call log_info,"Generating Kubernetes templates...") + helm template $(RELEASE_NAME) charts/accumulo -f $(VALUES_FILE) --namespace $(NAMESPACE) > accumulo-templates.yaml + $(call log_success,"Templates generated: accumulo-templates.yaml") + +.PHONY: debug +debug: ## Debug deployment issues + $(call log_info,"Gathering debug information...") + @echo "=== Helm Status ===" + -helm status $(RELEASE_NAME) -n $(NAMESPACE) + @echo "" + @echo "=== Pod Status ===" + -kubectl get pods -l app.kubernetes.io/name=accumulo -n $(NAMESPACE) + @echo "" + @echo "=== Service Status ===" + -kubectl get services -l app.kubernetes.io/name=accumulo -n $(NAMESPACE) + @echo "" + @echo "=== Recent Events ===" + -kubectl get events -n $(NAMESPACE) --sort-by='.lastTimestamp' | tail -10 + @echo "" + @echo "=== Pod Descriptions ===" + -kubectl describe pods -l app.kubernetes.io/name=accumulo -n $(NAMESPACE) + +.PHONY: kind-create +kind-create: ## Create KinD cluster for local development + $(call log_info,"Creating KinD cluster...") + kind create cluster --name accumulo-dev --config - < + +# Helm Charts for Apache Accumulo + +This directory contains Helm charts for deploying Apache Accumulo in Kubernetes with Alluxio as the storage layer. + +## Charts + +- `accumulo/` - Main Helm chart for deploying Apache Accumulo with Alluxio \ No newline at end of file diff --git a/charts/SUMMARY.md b/charts/SUMMARY.md new file mode 100644 index 00000000000..c1d30c3065a --- /dev/null +++ b/charts/SUMMARY.md @@ -0,0 +1,169 @@ + + +# Helm Chart Implementation Summary + +## Overview + +Successfully implemented a comprehensive Helm chart for deploying Apache Accumulo on Kubernetes with Alluxio as the storage layer, replacing HDFS with cloud-native object storage. + +## What Was Delivered + +### Core Requirements Met + +[x] **Production Helm Charts**: Complete umbrella chart with all Accumulo and Alluxio components +[x] **Alluxio Integration**: Configured to persist to object storage (S3/GCS/Azure/MinIO) +[x] **Cloud Storage Support**: Replaces HDFS with cloud object stores via Alluxio +[x] **Accumulo 2.x Components**: Manager, TabletServers, GC, Monitor, Compactors +[x] **ZooKeeper Options**: Embedded or external ZooKeeper support +[x] **Per-path Write Modes**: WAL=THROUGH, tables=CACHE_THROUGH, tmp=ASYNC_THROUGH +[x] **Cloud Authentication**: AWS/GCP/Azure credentials and identity options +[x] **Resiliency**: Anti-affinity, probes, resources, PVCs +[x] **Local Dev Mode**: MinIO integration for KinD/local testing +[x] **Documentation**: Comprehensive docs and smoke tests + +### File Structure + +``` +charts/accumulo/ +|- Chart.yaml # Helm chart metadata with dependencies +|- values.yaml # Default production values +|- values-dev.yaml # Development/local testing values +|- values-production-aws.yaml # AWS production example +|- README.md # Comprehensive usage guide +|- DEPLOYMENT.md # Step-by-step deployment guide +\- templates/ + |- _helpers.tpl # Template helpers and functions + |- configmap.yaml # Accumulo and Alluxio configuration + |- secret.yaml # Credentials management + |- serviceaccount.yaml # Kubernetes RBAC + |- alluxio-master-deployment.yaml # Alluxio master deployment + |- alluxio-master-service.yaml # Alluxio master service + |- alluxio-worker-daemonset.yaml # Alluxio workers on all nodes + |- accumulo-manager-deployment.yaml # Accumulo cluster manager + |- accumulo-manager-service.yaml # Manager service + |- accumulo-tserver-deployment.yaml # Tablet servers + |- accumulo-tserver-service.yaml # TabletServer service + |- accumulo-monitor-deployment.yaml # Web UI and monitoring + |- accumulo-monitor-service.yaml # Monitor service + |- accumulo-gc-deployment.yaml # Garbage collection + |- accumulo-compactor-deployment.yaml # Background compaction + \- tests/ + \- smoke-test.yaml # End-to-end validation tests +``` + +### Architecture Implemented + +``` ++------------------+ +------------------+ +------------------+ +| Accumulo | | Alluxio | | Cloud Storage | +| Components |--->| (Cache Layer) |--->| (S3/GCS/...) | ++------------------+ +------------------+ +------------------+ +``` + +**Accumulo Layer**: Manager, TabletServers, Monitor, GC, Compactors +**Alluxio Layer**: Distributed caching with memory/disk tiers +**Storage Layer**: Cloud object stores (S3, GCS, Azure Blob, MinIO) + +### Key Features + +#### Production Readiness +- **High Availability**: Multi-replica deployments with anti-affinity +- **Resource Management**: CPU/memory requests and limits for all components +- **Health Monitoring**: Liveness and readiness probes +- **Persistent Storage**: PVCs for Alluxio journal and cache +- **Security**: Cloud authentication with IRSA/Workload Identity/Managed Identity + +#### Development Experience +- **Local Testing**: Complete setup with MinIO in KinD +- **Smoke Tests**: Automated validation of all functionality +- **Documentation**: Step-by-step guides for all scenarios +- **Flexibility**: Multiple configuration examples + +#### Cloud Integration +- **AWS S3**: Native S3 support with IRSA authentication +- **Google Cloud**: GCS integration with Workload Identity +- **Azure Blob**: Azure Blob Storage with Managed Identity +- **Multi-cloud**: Alluxio enables seamless multi-cloud deployments + +### Usage Examples + +#### Quick Local Development +```bash +# Deploy locally with MinIO +helm install accumulo-dev ./charts/accumulo -f ./charts/accumulo/values-dev.yaml + +# Run tests +helm test accumulo-dev + +# Access services +kubectl port-forward svc/accumulo-dev-monitor 9995:9995 +``` + +#### Production AWS Deployment +```bash +# Deploy on EKS with S3 +helm install accumulo-prod ./charts/accumulo -f values-production-aws.yaml +``` + +#### Validation +```bash +# Run comprehensive smoke tests +helm test accumulo-prod + +# Manual verification +kubectl exec -it deployment/accumulo-prod-manager -- /opt/accumulo/bin/accumulo shell -u root +``` + +### Benefits Achieved + +#### Operational Excellence +- **Reduced Complexity**: No HDFS cluster to manage +- **Cloud Native**: Leverages managed object storage +- **Auto-scaling**: Kubernetes-native scaling capabilities +- **Monitoring**: Built-in web interfaces and metrics + +#### Cost Optimization +- **Storage Efficiency**: Pay-per-use object storage +- **Resource Elasticity**: Scale components independently +- **Multi-tenancy**: Shared Alluxio cache across workloads + +#### Performance +- **Intelligent Caching**: Hot data in memory/SSD tiers +- **Optimized Writes**: Per-path write policies for different data types +- **Network Efficiency**: Distributed caching reduces cloud API calls + +## Next Steps + +### Immediate +1. **Deploy and Test**: Use the development setup for validation +2. **Customize**: Adapt production values for your specific environment +3. **Monitor**: Set up metrics collection and alerting + +### Future Enhancements (Beyond Scope) +- Horizontal Pod Autoscaler configurations +- Advanced compaction strategies and tuning +- Migration tools from HDFS-based deployments +- Helm operator for GitOps workflows + +## Conclusion + +This implementation provides a complete, production-ready solution for running Apache Accumulo on Kubernetes with cloud storage. The focus on operational simplicity aligns with the goal of minimizing ops overhead while maintaining the power and flexibility of Accumulo for big data workloads. + +The chart successfully abstracts the complexity of distributed storage through Alluxio, enabling teams to focus on their core applications rather than infrastructure management. \ No newline at end of file diff --git a/charts/accumulo/Chart.yaml b/charts/accumulo/Chart.yaml new file mode 100644 index 00000000000..043c09b0529 --- /dev/null +++ b/charts/accumulo/Chart.yaml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v2 +name: accumulo +description: Apache Accumulo with Alluxio storage layer for Kubernetes +type: application +version: 1.0.0 +appVersion: "4.0.0-SNAPSHOT" +home: https://accumulo.apache.org +sources: + - https://github.com/apache/accumulo + - https://github.com/SentriusLLC/veculo +maintainers: + - name: Sentrius LLC +keywords: + - accumulo + - alluxio + - big-data + - hadoop + - database +annotations: + category: Database +dependencies: + - name: zookeeper + version: "1.0.0" + repository: "file://charts/zookeeper" + condition: zookeeper.enabled + - name: minio + version: "1.0.0" + repository: "file://charts/minio" + condition: minio.enabled \ No newline at end of file diff --git a/charts/accumulo/DEPLOYMENT.md b/charts/accumulo/DEPLOYMENT.md new file mode 100644 index 00000000000..b80f62b0437 --- /dev/null +++ b/charts/accumulo/DEPLOYMENT.md @@ -0,0 +1,522 @@ + + +# Deployment Guide + +This guide provides step-by-step instructions for deploying Apache Accumulo with Alluxio on Kubernetes. + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Local Development Deployment](#local-development-deployment) +3. [Production Deployment](#production-deployment) +4. [Post-Deployment Validation](#post-deployment-validation) +5. [Common Configuration Scenarios](#common-configuration-scenarios) +6. [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Software Requirements + +- **Kubernetes**: 1.19+ (tested on 1.24+) +- **Helm**: 3.2.0+ +- **kubectl**: Compatible with your cluster version + +### Infrastructure Requirements + +#### Development +- **CPU**: 4+ cores available to Kubernetes +- **Memory**: 8GB+ RAM available to Kubernetes +- **Storage**: 20GB+ available storage + +#### Production +- **CPU**: 20+ cores across multiple nodes +- **Memory**: 64GB+ RAM across multiple nodes +- **Storage**: Persistent volumes with high IOPS for Alluxio journal and cache +- **Network**: High bandwidth between nodes (10Gbps+ recommended) + +### Cloud Prerequisites + +#### AWS +- S3 bucket for data storage +- IAM role with S3 permissions (for IRSA) +- EKS cluster with CSI driver for EBS volumes + +#### Google Cloud +- GCS bucket for data storage +- Service account with Storage permissions +- GKE cluster with Workload Identity enabled + +#### Azure +- Azure Blob Storage container +- Managed Identity or Service Principal +- AKS cluster with Azure Disk CSI driver + +## Local Development Deployment + +Perfect for development, testing, and CI/CD pipelines. + +### 1. Create Local Kubernetes Cluster + +Using KinD (Kubernetes in Docker): + +```bash +# Install KinD +go install sigs.k8s.io/kind@latest + +# Create cluster with extra ports for services +cat < trust-policy.json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::ACCOUNT:oidc-provider/oidc.eks.REGION.amazonaws.com/id/OIDC_ID" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.REGION.amazonaws.com/id/OIDC_ID:sub": "system:serviceaccount:default:accumulo-prod" + } + } + } + ] +} +EOF + +aws iam create-role --role-name AccumuloProdRole --assume-role-policy-document file://trust-policy.json + +# Attach S3 permissions +aws iam put-role-policy --role-name AccumuloProdRole --policy-name S3Access --policy-document '{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::your-company-accumulo-prod", + "arn:aws:s3:::your-company-accumulo-prod/*" + ] + } + ] +}' +``` + +#### GCP Setup + +```bash +# Create GCS bucket +gsutil mb gs://your-company-accumulo-prod + +# Create service account +gcloud iam service-accounts create accumulo-prod + +# Grant storage permissions +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member="serviceAccount:accumulo-prod@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/storage.admin" + +# Enable Workload Identity +gcloud iam service-accounts add-iam-policy-binding \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:PROJECT_ID.svc.id.goog[default/accumulo-prod]" \ + accumulo-prod@PROJECT_ID.iam.gserviceaccount.com +``` + +### 2. Prepare Production Values + +Create your production values file based on the examples: + +```bash +# Copy and modify production values +cp ./charts/accumulo/values-production-aws.yaml my-production-values.yaml + +# Edit the values file +vim my-production-values.yaml +``` + +Key settings to customize: +- `accumulo.instance.secret`: Use a strong secret +- `storage.s3.bucket`: Your S3 bucket name +- `auth.serviceAccount.annotations`: Your IAM role ARN +- `zookeeper.external.hosts`: Your ZooKeeper cluster +- Resource requests/limits based on your workload + +### 3. Deploy to Production + +```bash +# Create namespace (optional) +kubectl create namespace accumulo-prod + +# Deploy with production values +helm install accumulo-prod ./charts/accumulo \ + -f my-production-values.yaml \ + --namespace accumulo-prod \ + --timeout 20m \ + --wait + +# Verify deployment +kubectl get pods -n accumulo-prod +kubectl get services -n accumulo-prod +``` + +### 4. Configure External Access + +```bash +# Get LoadBalancer external IP (if using LoadBalancer service type) +kubectl get svc accumulo-prod-monitor -n accumulo-prod + +# Or use Ingress for HTTPS termination +cat < + +# Apache Accumulo Helm Chart + +This Helm chart deploys Apache Accumulo on Kubernetes with Alluxio as the distributed storage layer, replacing HDFS for cloud-native deployments. + +## Features + +- **Cloud-native storage**: Uses Alluxio to provide a unified view over cloud object stores (S3, GCS, Azure Blob) +- **Production-ready**: Includes anti-affinity rules, resource limits, probes, and PVCs for resiliency +- **Multiple storage backends**: Supports AWS S3, Google Cloud Storage, Azure Blob Storage, and MinIO +- **Development mode**: Local development setup with MinIO and reduced resource requirements +- **Comprehensive monitoring**: Includes Accumulo Monitor web UI and optional metrics integration +- **Flexible authentication**: Support for cloud provider authentication methods (IRSA, Workload Identity, etc.) + +## Quick Start + +### Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ +- StorageClass for persistent volumes (production) + +### Local Development with MinIO + +For local development and testing, use the development values: + +```bash +# Install with MinIO for local testing +helm install accumulo-dev ./charts/accumulo -f ./charts/accumulo/values-dev.yaml + +# Run smoke tests +helm test accumulo-dev +``` + +### Production Deployment + +1. **Prepare values file for your cloud provider:** + +For AWS S3: +```yaml +storage: + provider: "s3" + s3: + endpoint: "https://s3.amazonaws.com" + bucket: "your-accumulo-bucket" + region: "us-west-2" + accessKey: "your-access-key" + secretKey: "your-secret-key" + +auth: + method: "serviceAccount" + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/accumulo-role" +``` + +For Google Cloud Storage: +```yaml +storage: + provider: "gcs" + gcs: + projectId: "your-project-id" + bucket: "your-accumulo-bucket" + keyFile: | + { + "type": "service_account", + "project_id": "your-project-id", + ... + } + +auth: + method: "workloadIdentity" + serviceAccount: + annotations: + iam.gke.io/gcp-service-account: "accumulo@your-project.iam.gserviceaccount.com" +``` + +2. **Deploy to production:** + +```bash +helm install accumulo ./charts/accumulo -f your-production-values.yaml +``` + +## Configuration + +### Core Settings + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `accumulo.instance.name` | Accumulo instance name | `accumulo` | +| `accumulo.instance.secret` | Instance secret (change in production!) | `DEFAULT_CHANGE_ME` | +| `accumulo.instance.volumes` | Accumulo volumes path | `alluxio://alluxio-master:19998/accumulo` | + +### Component Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `accumulo.manager.enabled` | Enable Accumulo Manager | `true` | +| `accumulo.manager.replicaCount` | Number of Manager replicas | `1` | +| `accumulo.tserver.enabled` | Enable TabletServers | `true` | +| `accumulo.tserver.replicaCount` | Number of TabletServer replicas | `3` | +| `accumulo.monitor.enabled` | Enable Monitor web UI | `true` | +| `accumulo.gc.enabled` | Enable Garbage Collector | `true` | +| `accumulo.compactor.enabled` | Enable Compactors | `true` | + +### Alluxio Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `alluxio.enabled` | Enable Alluxio deployment | `true` | +| `alluxio.master.replicaCount` | Number of Alluxio masters | `1` | +| `alluxio.worker.replicaCount` | Number of Alluxio workers | `3` | +| `alluxio.properties.alluxio.worker.memory.size` | Worker memory size | `1GB` | + +### Storage Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `storage.provider` | Storage provider (s3, gcs, azure, minio) | `minio` | +| `storage.s3.bucket` | S3 bucket name | `accumulo-data` | +| `storage.s3.region` | S3 region | `us-west-2` | +| `storage.gcs.bucket` | GCS bucket name | `accumulo-data` | +| `storage.azure.container` | Azure container name | `accumulo-data` | + +## Architecture + +The chart deploys the following components: + +### Accumulo Components +- **Manager**: Cluster coordination and metadata management +- **TabletServers**: Handle read/write operations and host tablets +- **Monitor**: Web UI for cluster monitoring and management +- **Garbage Collector**: Cleans up unused files +- **Compactors**: Background compaction of tablets + +### Alluxio Components +- **Master**: Metadata management and coordination +- **Workers**: Distributed caching layer with memory and disk tiers + +### Supporting Services +- **ZooKeeper**: Coordination service (embedded or external) +- **MinIO**: Object storage for development (optional) + +## Storage Architecture + +``` ++------------------+ +------------------+ +------------------+ +| Accumulo | | Alluxio | | Cloud Storage | +| Components |--->| (Cache Layer) |--->| (S3/GCS/...) | ++------------------+ +------------------+ +------------------+ +``` + +Alluxio provides: +- **Unified namespace**: Single view across multiple storage systems +- **Intelligent caching**: Hot data cached in memory/SSD for performance +- **Write optimization**: Different write modes per path (WAL, tables, temp) + +## Monitoring + +### Web Interfaces + +- **Accumulo Monitor**: `http://:9995/` +- **Alluxio Master**: `http://:19999/` + +### Prometheus Metrics (Optional) + +Enable Prometheus metrics collection: + +```yaml +monitoring: + prometheus: + enabled: true +``` + +## Security + +### Cloud Authentication + +The chart supports multiple authentication methods: + +- **Service Account**: Use Kubernetes service accounts with cloud IAM +- **Access Keys**: Direct credential configuration +- **Workload Identity**: GKE Workload Identity +- **IRSA**: EKS IAM Roles for Service Accounts +- **Managed Identity**: Azure Managed Identity + +### Network Security + +- All inter-component communication uses Kubernetes services +- Optional Istio service mesh support +- Configurable network policies (not included in this chart) + +## Troubleshooting + +### Common Issues + +1. **Pods stuck in Pending**: Check resource requests and node capacity +2. **Storage connection issues**: Verify cloud credentials and bucket permissions +3. **Alluxio mount failures**: Check storage provider configuration + +### Debugging Commands + +```bash +# Check Accumulo Manager logs +kubectl logs deployment/accumulo-manager + +# Check Alluxio Master status +kubectl port-forward svc/accumulo-alluxio-master 19999:19999 +curl http://localhost:19999/ + +# Run shell commands +kubectl exec -it deployment/accumulo-manager -- /opt/accumulo/bin/accumulo shell -u root +``` + +### Smoke Tests + +Run the built-in smoke tests to validate deployment: + +```bash +helm test +``` + +The smoke test validates: +- All services are accessible +- Accumulo table operations work +- Alluxio integration is functional +- Monitor web interface is available + +## Upgrade Guide + +### From Previous Versions + +1. **Backup your data**: Ensure data is safely stored in cloud object storage +2. **Update values**: Review new configuration options +3. **Perform upgrade**: `helm upgrade ./charts/accumulo` + +### Rolling Updates + +The chart supports rolling updates for most components: +- TabletServers can be updated rolling +- Compactors support rolling updates +- Manager updates may cause brief unavailability + +## Development + +### Local Development Setup + +1. **Install KinD**: For local Kubernetes cluster +2. **Deploy with dev values**: Use `values-dev.yaml` +3. **Access services**: Use port-forwarding for local access + +```bash +# Create local cluster +kind create cluster --name accumulo-dev + +# Install chart +helm install accumulo-dev ./charts/accumulo -f ./charts/accumulo/values-dev.yaml + +# Port forward to access services +kubectl port-forward svc/accumulo-dev-monitor 9995:9995 +kubectl port-forward svc/accumulo-dev-alluxio-master 19999:19999 +``` + +### Contributing + +1. **Test changes**: Always test with smoke tests +2. **Update documentation**: Keep README and values comments current +3. **Validate templates**: Use `helm template` and `helm lint` + +## License + +This chart is provided under the Apache License 2.0, same as Apache Accumulo. + +## Support + +For issues related to: +- **Chart configuration**: Open GitHub issues +- **Accumulo functionality**: Refer to Apache Accumulo documentation +- **Alluxio integration**: Check Alluxio documentation +- **Cloud provider setup**: Consult respective cloud provider documentation \ No newline at end of file diff --git a/charts/accumulo/templates/_helpers.tpl b/charts/accumulo/templates/_helpers.tpl new file mode 100644 index 00000000000..e4a69a844e7 --- /dev/null +++ b/charts/accumulo/templates/_helpers.tpl @@ -0,0 +1,176 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "accumulo.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "accumulo.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "accumulo.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "accumulo.labels" -}} +helm.sh/chart: {{ include "accumulo.chart" . }} +{{ include "accumulo.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.global.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "accumulo.selectorLabels" -}} +app.kubernetes.io/name: {{ include "accumulo.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Component labels +*/}} +{{- define "accumulo.componentLabels" -}} +{{ include "accumulo.labels" . }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "accumulo.serviceAccountName" -}} +{{- if .Values.auth.serviceAccount.create }} +{{- default (include "accumulo.fullname" .) .Values.auth.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.auth.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Accumulo image +*/}} +{{- define "accumulo.image" -}} +{{- $registry := .Values.global.imageRegistry | default .Values.accumulo.image.registry }} +{{- printf "%s/%s:%s" $registry .Values.accumulo.image.repository .Values.accumulo.image.tag }} +{{- end }} + +{{/* +Alluxio image +*/}} +{{- define "alluxio.image" -}} +{{- $registry := .Values.global.imageRegistry | default .Values.alluxio.image.registry }} +{{- printf "%s/%s:%s" $registry .Values.alluxio.image.repository .Values.alluxio.image.tag }} +{{- end }} + +{{/* +ZooKeeper connection string +*/}} +{{- define "accumulo.zookeeperHosts" -}} +{{- if .Values.zookeeper.enabled }} +{{- $fullname := include "accumulo.fullname" . }} +{{- printf "%s-zookeeper:2181" $fullname }} +{{- else }} +{{- .Values.zookeeper.external.hosts }} +{{- end }} +{{- end }} + +{{/* +Storage configuration based on provider +*/}} +{{- define "accumulo.storageConfig" -}} +{{- $provider := .Values.storage.provider }} +{{- if eq $provider "s3" }} +alluxio.master.mount.table.root.ufs=s3://{{ .Values.storage.s3.bucket }}/ +{{- else if eq $provider "gcs" }} +alluxio.master.mount.table.root.ufs=gs://{{ .Values.storage.gcs.bucket }}/ +{{- else if eq $provider "azure" }} +alluxio.master.mount.table.root.ufs=abfs://{{ .Values.storage.azure.container }}@{{ .Values.storage.azure.account }}.dfs.core.windows.net/ +{{- else if eq $provider "minio" }} +alluxio.master.mount.table.root.ufs=s3://{{ .Values.storage.minio.bucket }}/ +{{- end }} +{{- end }} + +{{/* +Pod anti-affinity configuration +*/}} +{{- define "accumulo.podAntiAffinity" -}} +{{- if .podAntiAffinity.enabled }} +podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: {{ .component }} + topologyKey: {{ .podAntiAffinity.topologyKey }} +{{- end }} +{{- end }} + +{{/* +Resource configuration +*/}} +{{- define "accumulo.resources" -}} +{{- if .resources }} +resources: + {{- toYaml .resources | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Common environment variables for Accumulo containers +*/}} +{{- define "accumulo.commonEnv" -}} +- name: ACCUMULO_INSTANCE_NAME + value: {{ .Values.accumulo.instance.name | quote }} +- name: ACCUMULO_INSTANCE_SECRET + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: instance-secret +- name: ZOOKEEPER_HOSTS + value: {{ include "accumulo.zookeeperHosts" . | quote }} +- name: ACCUMULO_LOG_DIR + value: "/opt/accumulo/logs" +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-compactor-deployment.yaml b/charts/accumulo/templates/accumulo-compactor-deployment.yaml new file mode 100644 index 00000000000..a635fb3f314 --- /dev/null +++ b/charts/accumulo/templates/accumulo-compactor-deployment.yaml @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.compactor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-compactor + labels: + {{- $component := "compactor" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.accumulo.compactor.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: compactor + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: compactor + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.accumulo.compactor.podAntiAffinity.enabled }} + affinity: + {{- $component := "compactor" }} + {{- $podAntiAffinity := .Values.accumulo.compactor.podAntiAffinity }} + {{- include "accumulo.podAntiAffinity" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component "podAntiAffinity" $podAntiAffinity) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-manager + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for Accumulo manager to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-manager 9999; do + echo "Waiting for manager..." + sleep 5 + done + echo "Manager is ready" + containers: + - name: compactor + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /opt/accumulo/bin/accumulo + - compactor + - -q + - default + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + - name: ACCUMULO_SERVICE_INSTANCE + value: "compactor" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + - name: logs + mountPath: /opt/accumulo/logs + resources: + {{- toYaml .Values.accumulo.resources.compactor | nindent 10 }} + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 + - name: logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-gc-deployment.yaml b/charts/accumulo/templates/accumulo-gc-deployment.yaml new file mode 100644 index 00000000000..c2ec2c6a2b1 --- /dev/null +++ b/charts/accumulo/templates/accumulo-gc-deployment.yaml @@ -0,0 +1,96 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.gc.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-gc + labels: + {{- $component := "gc" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.accumulo.gc.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: gc + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: gc + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-manager + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for Accumulo manager to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-manager 9999; do + echo "Waiting for manager..." + sleep 5 + done + echo "Manager is ready" + containers: + - name: gc + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /opt/accumulo/bin/accumulo + - gc + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + - name: ACCUMULO_SERVICE_INSTANCE + value: "gc" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + - name: logs + mountPath: /opt/accumulo/logs + resources: + {{- toYaml .Values.accumulo.resources.gc | nindent 10 }} + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 + - name: logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-manager-deployment.yaml b/charts/accumulo/templates/accumulo-manager-deployment.yaml new file mode 100644 index 00000000000..4ccf1b6f996 --- /dev/null +++ b/charts/accumulo/templates/accumulo-manager-deployment.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.manager.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-manager + labels: + {{- $component := "manager" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.accumulo.manager.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: manager + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: manager + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.accumulo.manager.podAntiAffinity.enabled }} + affinity: + {{- $component := "manager" }} + {{- $podAntiAffinity := .Values.accumulo.manager.podAntiAffinity }} + {{- include "accumulo.podAntiAffinity" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component "podAntiAffinity" $podAntiAffinity) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-zookeeper + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for ZooKeeper to be ready..." + until nc -z {{ include "accumulo.zookeeperHosts" . | replace ":2181" "" }} 2181; do + echo "Waiting for ZooKeeper..." + sleep 5 + done + echo "ZooKeeper is ready" + - name: wait-for-alluxio + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for Alluxio master to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-alluxio-master 19998; do + echo "Waiting for Alluxio master..." + sleep 5 + done + echo "Alluxio master is ready" + - name: init-accumulo + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + # Check if instance is already initialized + if /opt/accumulo/bin/accumulo org.apache.accumulo.server.util.ListInstances | grep -q "{{ .Values.accumulo.instance.name }}"; then + echo "Accumulo instance '{{ .Values.accumulo.instance.name }}' already exists" + exit 0 + fi + + echo "Initializing Accumulo instance '{{ .Values.accumulo.instance.name }}'" + /opt/accumulo/bin/accumulo init \ + --instance-name {{ .Values.accumulo.instance.name }} \ + --password {{ .Values.accumulo.instance.secret }} + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + containers: + - name: manager + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /opt/accumulo/bin/accumulo + - manager + ports: + - name: client + containerPort: 9999 + protocol: TCP + - name: replication + containerPort: 10001 + protocol: TCP + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + - name: ACCUMULO_SERVICE_INSTANCE + value: "manager" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + - name: logs + mountPath: /opt/accumulo/logs + resources: + {{- toYaml .Values.accumulo.resources.manager | nindent 10 }} + livenessProbe: + tcpSocket: + port: client + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + tcpSocket: + port: client + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 + - name: logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-manager-service.yaml b/charts/accumulo/templates/accumulo-manager-service.yaml new file mode 100644 index 00000000000..71b7ad7559d --- /dev/null +++ b/charts/accumulo/templates/accumulo-manager-service.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.manager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-manager + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: manager + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: client + port: 9999 + targetPort: client + protocol: TCP + - name: replication + port: 10001 + targetPort: replication + protocol: TCP + selector: + {{- include "accumulo.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: manager +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-monitor-deployment.yaml b/charts/accumulo/templates/accumulo-monitor-deployment.yaml new file mode 100644 index 00000000000..a9c8d9fc9a6 --- /dev/null +++ b/charts/accumulo/templates/accumulo-monitor-deployment.yaml @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.monitor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-monitor + labels: + {{- $component := "monitor" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.accumulo.monitor.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitor + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitor + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-manager + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for Accumulo manager to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-manager 9999; do + echo "Waiting for manager..." + sleep 5 + done + echo "Manager is ready" + containers: + - name: monitor + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /opt/accumulo/bin/accumulo + - monitor + ports: + - name: http + containerPort: 9995 + protocol: TCP + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + - name: ACCUMULO_SERVICE_INSTANCE + value: "monitor" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + - name: logs + mountPath: /opt/accumulo/logs + resources: + {{- toYaml .Values.accumulo.resources.monitor | nindent 10 }} + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 + - name: logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-monitor-service.yaml b/charts/accumulo/templates/accumulo-monitor-service.yaml new file mode 100644 index 00000000000..6e98c3ed559 --- /dev/null +++ b/charts/accumulo/templates/accumulo-monitor-service.yaml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-monitor + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: monitor + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.accumulo.monitor.service.type }} + ports: + - name: http + port: {{ .Values.accumulo.monitor.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "accumulo.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: monitor +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-tserver-deployment.yaml b/charts/accumulo/templates/accumulo-tserver-deployment.yaml new file mode 100644 index 00000000000..376ecee8ee2 --- /dev/null +++ b/charts/accumulo/templates/accumulo-tserver-deployment.yaml @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.tserver.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-tserver + labels: + {{- $component := "tserver" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.accumulo.tserver.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: tserver + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: tserver + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.accumulo.tserver.podAntiAffinity.enabled }} + affinity: + {{- $component := "tserver" }} + {{- $podAntiAffinity := .Values.accumulo.tserver.podAntiAffinity }} + {{- include "accumulo.podAntiAffinity" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component "podAntiAffinity" $podAntiAffinity) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-manager + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for Accumulo manager to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-manager 9999; do + echo "Waiting for manager..." + sleep 5 + done + echo "Manager is ready" + containers: + - name: tserver + image: {{ include "accumulo.image" . }} + imagePullPolicy: {{ .Values.accumulo.image.pullPolicy }} + command: + - /opt/accumulo/bin/accumulo + - tserver + ports: + - name: client + containerPort: 9997 + protocol: TCP + - name: replication + containerPort: 10002 + protocol: TCP + env: + {{- include "accumulo.commonEnv" . | nindent 8 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + - name: ACCUMULO_SERVICE_INSTANCE + value: "tserver" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + - name: accumulo-config + mountPath: /opt/accumulo/conf/log4j2-service.properties + subPath: log4j2-service.properties + - name: logs + mountPath: /opt/accumulo/logs + resources: + {{- toYaml .Values.accumulo.resources.tserver | nindent 10 }} + livenessProbe: + tcpSocket: + port: client + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + tcpSocket: + port: client + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 + - name: logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/accumulo-tserver-service.yaml b/charts/accumulo/templates/accumulo-tserver-service.yaml new file mode 100644 index 00000000000..fc2f19caf1f --- /dev/null +++ b/charts/accumulo/templates/accumulo-tserver-service.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.accumulo.tserver.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-tserver + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: tserver + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: client + port: 9997 + targetPort: client + protocol: TCP + - name: replication + port: 10002 + targetPort: replication + protocol: TCP + selector: + {{- include "accumulo.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: tserver +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/alluxio-master-deployment.yaml b/charts/accumulo/templates/alluxio-master-deployment.yaml new file mode 100644 index 00000000000..aa100ebe263 --- /dev/null +++ b/charts/accumulo/templates/alluxio-master-deployment.yaml @@ -0,0 +1,180 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.alluxio.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-master + labels: + {{- $component := "alluxio-master" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.alluxio.master.replicaCount }} + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: alluxio-master + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: alluxio-master + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + containers: + - name: alluxio-master + image: {{ include "alluxio.image" . }} + imagePullPolicy: {{ .Values.alluxio.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + # Create journal directory + mkdir -p /opt/alluxio/journal + + # Format journal if it doesn't exist + if [ ! -f /opt/alluxio/journal/.formatted ]; then + /opt/alluxio/bin/alluxio formatJournal + touch /opt/alluxio/journal/.formatted + fi + + # Start master + /opt/alluxio/bin/alluxio-start.sh master + + # Keep container running and monitor process + while true; do + if ! pgrep -f "alluxio.master.AlluxioMaster" > /dev/null; then + echo "Alluxio master process died, restarting..." + /opt/alluxio/bin/alluxio-start.sh master + fi + sleep 30 + done + ports: + - name: rpc + containerPort: 19998 + protocol: TCP + - name: web + containerPort: 19999 + protocol: TCP + env: + - name: ALLUXIO_MASTER_HOSTNAME + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if eq .Values.storage.provider "s3" }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: s3-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: s3-secret-key + {{- else if eq .Values.storage.provider "minio" }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: minio-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: minio-secret-key + {{- end }} + volumeMounts: + - name: alluxio-config + mountPath: /opt/alluxio/conf/alluxio-site.properties + subPath: alluxio-site.properties + - name: journal + mountPath: /opt/alluxio/journal + {{- if and (eq .Values.storage.provider "gcs") .Values.storage.gcs.keyFile }} + - name: gcs-secret + mountPath: /opt/alluxio/secrets + readOnly: true + {{- end }} + resources: + {{- toYaml .Values.alluxio.master.resources | nindent 10 }} + livenessProbe: + httpGet: + path: / + port: web + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: / + port: web + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: alluxio-config + configMap: + name: {{ include "accumulo.fullname" . }}-alluxio-config + - name: journal + {{- if .Values.alluxio.master.journal.storageClass }} + persistentVolumeClaim: + claimName: {{ include "accumulo.fullname" . }}-alluxio-master-journal + {{- else }} + emptyDir: {} + {{- end }} + {{- if and (eq .Values.storage.provider "gcs") .Values.storage.gcs.keyFile }} + - name: gcs-secret + secret: + secretName: {{ include "accumulo.fullname" . }}-gcs-secret + {{- end }} +--- +{{- if .Values.alluxio.master.journal.storageClass }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-master-journal + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-master + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - ReadWriteOnce + {{- if .Values.alluxio.master.journal.storageClass }} + storageClassName: {{ .Values.alluxio.master.journal.storageClass }} + {{- else if .Values.global.storageClass }} + storageClassName: {{ .Values.global.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.alluxio.master.journal.size }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/alluxio-master-service.yaml b/charts/accumulo/templates/alluxio-master-service.yaml new file mode 100644 index 00000000000..24d78d874a5 --- /dev/null +++ b/charts/accumulo/templates/alluxio-master-service.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.alluxio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-master + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-master + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: rpc + port: 19998 + targetPort: rpc + protocol: TCP + - name: web + port: 19999 + targetPort: web + protocol: TCP + selector: + {{- include "accumulo.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-master +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/alluxio-worker-daemonset.yaml b/charts/accumulo/templates/alluxio-worker-daemonset.yaml new file mode 100644 index 00000000000..86484d1c8a6 --- /dev/null +++ b/charts/accumulo/templates/alluxio-worker-daemonset.yaml @@ -0,0 +1,197 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.alluxio.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-worker + labels: + {{- $component := "alluxio-worker" }} + {{- include "accumulo.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "accumulo.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: alluxio-worker + template: + metadata: + labels: + {{- include "accumulo.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: alluxio-worker + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + hostNetwork: false + containers: + - name: alluxio-worker + image: {{ include "alluxio.image" . }} + imagePullPolicy: {{ .Values.alluxio.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + # Wait for master to be ready + echo "Waiting for Alluxio master to be ready..." + until nc -z {{ include "accumulo.fullname" . }}-alluxio-master 19998; do + echo "Waiting for master..." + sleep 5 + done + + # Create directories + mkdir -p /opt/ramdisk + mkdir -p /opt/alluxio/logs + + # Mount ramdisk for memory tier + mount -t tmpfs -o size={{ .Values.alluxio.properties.alluxio.worker.memory.size }} tmpfs /opt/ramdisk + + # Start worker + /opt/alluxio/bin/alluxio-start.sh worker + + # Keep container running and monitor process + while true; do + if ! pgrep -f "alluxio.worker.AlluxioWorker" > /dev/null; then + echo "Alluxio worker process died, restarting..." + /opt/alluxio/bin/alluxio-start.sh worker + fi + sleep 30 + done + ports: + - name: rpc + containerPort: 29999 + protocol: TCP + - name: data + containerPort: 29999 + protocol: TCP + - name: web + containerPort: 30000 + protocol: TCP + env: + - name: ALLUXIO_WORKER_HOSTNAME + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: ALLUXIO_MASTER_HOSTNAME + value: {{ include "accumulo.fullname" . }}-alluxio-master + {{- if eq .Values.storage.provider "s3" }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: s3-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: s3-secret-key + {{- else if eq .Values.storage.provider "minio" }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: minio-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "accumulo.fullname" . }}-secret + key: minio-secret-key + {{- end }} + volumeMounts: + - name: alluxio-config + mountPath: /opt/alluxio/conf/alluxio-site.properties + subPath: alluxio-site.properties + - name: storage + mountPath: /opt/alluxio/underFSStorage + - name: ramdisk + mountPath: /opt/ramdisk + {{- if and (eq .Values.storage.provider "gcs") .Values.storage.gcs.keyFile }} + - name: gcs-secret + mountPath: /opt/alluxio/secrets + readOnly: true + {{- end }} + resources: + {{- toYaml .Values.alluxio.worker.resources | nindent 10 }} + securityContext: + privileged: true # Required for mounting tmpfs + livenessProbe: + httpGet: + path: / + port: web + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: / + port: web + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: alluxio-config + configMap: + name: {{ include "accumulo.fullname" . }}-alluxio-config + - name: storage + {{- if .Values.alluxio.worker.storage.storageClass }} + persistentVolumeClaim: + claimName: {{ include "accumulo.fullname" . }}-alluxio-worker-storage + {{- else }} + emptyDir: {} + {{- end }} + - name: ramdisk + emptyDir: + medium: Memory + {{- if and (eq .Values.storage.provider "gcs") .Values.storage.gcs.keyFile }} + - name: gcs-secret + secret: + secretName: {{ include "accumulo.fullname" . }}-gcs-secret + {{- end }} +--- +{{- if .Values.alluxio.worker.storage.storageClass }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-worker-storage + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-worker + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - ReadWriteOnce + {{- if .Values.alluxio.worker.storage.storageClass }} + storageClassName: {{ .Values.alluxio.worker.storage.storageClass }} + {{- else if .Values.global.storageClass }} + storageClassName: {{ .Values.global.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.alluxio.worker.storage.size }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/configmap.yaml b/charts/accumulo/templates/configmap.yaml new file mode 100644 index 00000000000..f0ce12652bd --- /dev/null +++ b/charts/accumulo/templates/configmap.yaml @@ -0,0 +1,268 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "accumulo.fullname" . }}-config + labels: + {{- include "accumulo.labels" . | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + accumulo.properties: | + # Apache Accumulo Configuration for Kubernetes + + ## Instance configuration + instance.volumes={{ .Values.accumulo.instance.volumes }} + instance.zookeeper.host={{ include "accumulo.zookeeperHosts" . }} + instance.secret={{ .Values.accumulo.instance.secret }} + + ## Enable native maps for better performance + tserver.memory.maps.native.enabled=true + + ## Manager configuration + manager.recovery.delay=10s + manager.lease.recovery.waiting.period=5s + + ## Tablet server configuration + tserver.port.search=true + tserver.hold.time.max=5m + tserver.memory.maps.max=1G + + ## Monitor configuration + monitor.port.client=9995 + monitor.ssl.port=9995 + + ## GC configuration + gc.cycle.start=30s + gc.cycle.delay=5m + + ## Compactor configuration + compactor.max.open.files=100 + + ## Performance tuning for Kubernetes + general.rpc.timeout=120s + tserver.scan.timeout.enable=true + tserver.scan.timeout.max=5m + + ## Alluxio-specific configuration + general.vfs.context.class.name=org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment + general.vfs.cache.dir=/tmp/accumulo-vfs-cache + + accumulo-env.sh: | + #!/usr/bin/env bash + + ## Accumulo environment for Kubernetes deployment + + ## Required environment variables + export ACCUMULO_LOG_DIR="${ACCUMULO_LOG_DIR:-/opt/accumulo/logs}" + export HADOOP_HOME="${HADOOP_HOME:-/opt/hadoop}" + export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-/opt/hadoop/etc/hadoop}" + export ZOOKEEPER_HOME="${ZOOKEEPER_HOME:-/opt/zookeeper}" + + ## Build classpath + if [[ -n $CLASSPATH ]]; then + CLASSPATH="${CLASSPATH}:${ACCUMULO_CONF_DIR}" + else + CLASSPATH="${ACCUMULO_CONF_DIR}" + fi + + # Add Accumulo libraries + CLASSPATH="${CLASSPATH}:${ACCUMULO_HOME}/lib/*" + + # Add Hadoop libraries + CLASSPATH="${CLASSPATH}:${HADOOP_CONF_DIR}:${HADOOP_HOME}/share/hadoop/client/*" + + # Add ZooKeeper libraries + ZK_JARS=$(find "${ZOOKEEPER_HOME}/lib/" -maxdepth 1 -name '*.jar' -not -name '*slf4j*' -not -name '*log4j*' | paste -sd: -) + CLASSPATH="${CLASSPATH}:${ZOOKEEPER_HOME}/*:${ZK_JARS}" + + export CLASSPATH + + ## JVM options for all processes + JAVA_OPTS=( + '-XX:OnOutOfMemoryError=kill -9 %p' + '-XX:-OmitStackTraceInFastThrow' + '-Djava.net.preferIPv4Stack=true' + "-Daccumulo.native.lib.path=${ACCUMULO_HOME}/lib/native" + ) + + ## Component-specific JVM options + case "${ACCUMULO_SERVICE_INSTANCE}" in + manager) + JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") + ;; + monitor) + JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") + ;; + gc) + JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") + ;; + tserver) + JAVA_OPTS=('-Xmx1024m' '-Xms1024m' "${JAVA_OPTS[@]}") + ;; + compactor) + JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") + ;; + *) + JAVA_OPTS=('-Xmx256m' '-Xms64m' "${JAVA_OPTS[@]}") + ;; + esac + + ## Logging configuration + JAVA_OPTS=( + "-Daccumulo.log.dir=${ACCUMULO_LOG_DIR}" + "-Daccumulo.application=${ACCUMULO_SERVICE_INSTANCE}_$(hostname)" + "-Daccumulo.metrics.service.instance=${ACCUMULO_SERVICE_INSTANCE}" + "-Dlog4j2.statusLoggerLevel=ERROR" + "-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" + "${JAVA_OPTS[@]}" + ) + + ## Service-specific log configuration + case "${ACCUMULO_SERVICE_INSTANCE}" in + monitor | gc | manager | tserver | compactor) + JAVA_OPTS=('-Dlog4j.configurationFile=log4j2-service.properties' "${JAVA_OPTS[@]}") + ;; + esac + + export JAVA_OPTS + export MALLOC_ARENA_MAX=1 + + log4j2-service.properties: | + # Log4j2 configuration for Accumulo services in Kubernetes + + status = ERROR + name = AccumuloServiceConfig + + # Console appender for container logs + appender.console.type = Console + appender.console.name = STDOUT + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{ISO8601} [%c{2}] %-5p: %m%n + + # File appender for service logs + appender.file.type = File + appender.file.name = FILE + appender.file.fileName = ${sys:accumulo.log.dir}/accumulo-${sys:accumulo.application}.log + appender.file.layout.type = PatternLayout + appender.file.layout.pattern = %d{ISO8601} [%c{2}] %-5p: %m%n + + # Root logger + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = STDOUT + rootLogger.appenderRef.file.ref = FILE + + # Accumulo-specific loggers + logger.accumulo.name = org.apache.accumulo + logger.accumulo.level = INFO + logger.accumulo.additivity = false + logger.accumulo.appenderRef.console.ref = STDOUT + logger.accumulo.appenderRef.file.ref = FILE + + # Hadoop/Alluxio loggers (reduce verbosity) + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = WARN + + logger.alluxio.name = alluxio + logger.alluxio.level = INFO +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-config + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: alluxio + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + alluxio-site.properties: | + # Alluxio configuration for Accumulo storage + + ## Master configuration + alluxio.master.hostname={{ include "accumulo.fullname" . }}-alluxio-master + alluxio.master.port=19998 + alluxio.master.web.port=19999 + alluxio.master.journal.type=UFS + alluxio.master.journal.folder=/opt/alluxio/journal + + ## Worker configuration + alluxio.worker.hostname=${ALLUXIO_WORKER_HOSTNAME} + alluxio.worker.port=29999 + alluxio.worker.web.port=30000 + alluxio.worker.data.port=29999 + alluxio.worker.rpc.port=29999 + + ## Memory and storage configuration + alluxio.worker.memory.size={{ .Values.alluxio.properties.alluxio.worker.memory.size }} + alluxio.worker.tieredstore.levels=1 + alluxio.worker.tieredstore.level0.alias=MEM + alluxio.worker.tieredstore.level0.dirs.path=/opt/ramdisk + alluxio.worker.tieredstore.level0.dirs.quota={{ .Values.alluxio.properties.alluxio.worker.memory.size }} + alluxio.worker.tieredstore.level0.watermark.high.ratio=0.9 + alluxio.worker.tieredstore.level0.watermark.low.ratio=0.7 + + ## Under storage system configuration + {{- include "accumulo.storageConfig" . }} + + {{- if eq .Values.storage.provider "s3" }} + # S3 configuration + alluxio.underfs.s3.endpoint={{ .Values.storage.s3.endpoint }} + alluxio.underfs.s3.region={{ .Values.storage.s3.region }} + s3a.access.key={{ .Values.storage.s3.accessKey }} + s3a.secret.key={{ .Values.storage.s3.secretKey }} + {{- else if eq .Values.storage.provider "minio" }} + # MinIO configuration + alluxio.underfs.s3.endpoint={{ .Values.storage.minio.endpoint }} + alluxio.underfs.s3.disable.dns.buckets=true + alluxio.underfs.s3.inherit.acl=false + s3a.access.key={{ .Values.storage.minio.accessKey }} + s3a.secret.key={{ .Values.storage.minio.secretKey }} + {{- else if eq .Values.storage.provider "gcs" }} + # GCS configuration + fs.gcs.project.id={{ .Values.storage.gcs.projectId }} + fs.gcs.auth.service.account.json.keyfile=/opt/alluxio/secrets/gcs-key.json + {{- else if eq .Values.storage.provider "azure" }} + # Azure configuration + fs.azure.account.key.{{ .Values.storage.azure.account }}.dfs.core.windows.net={{ .Values.storage.azure.key }} + {{- end }} + + ## Performance and cache settings + alluxio.user.file.write.location.policy.class={{ .Values.alluxio.properties.alluxio.user.file.write.location.policy.class }} + alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes={{ .Values.alluxio.properties.alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes }} + + ## Path-specific write modes for Accumulo data + {{- range $path, $mode := .Values.alluxio.pathWriteModes }} + alluxio.user.file.write.type.{{ $path }}={{ $mode }} + {{- end }} + + ## Network and RPC settings + alluxio.network.host.resolution.timeout=5s + alluxio.user.rpc.retry.max.duration=10s + alluxio.user.rpc.retry.base.sleep=1s + + ## Security configuration + alluxio.security.authentication.type=NOSASL + alluxio.security.authorization.permission.enabled=false \ No newline at end of file diff --git a/charts/accumulo/templates/secret.yaml b/charts/accumulo/templates/secret.yaml new file mode 100644 index 00000000000..17472d0f89a --- /dev/null +++ b/charts/accumulo/templates/secret.yaml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "accumulo.fullname" . }}-secret + labels: + {{- include "accumulo.labels" . | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + instance-secret: {{ .Values.accumulo.instance.secret | b64enc | quote }} +{{- if eq .Values.storage.provider "s3" }} + s3-access-key: {{ .Values.storage.s3.accessKey | b64enc | quote }} + s3-secret-key: {{ .Values.storage.s3.secretKey | b64enc | quote }} +{{- else if eq .Values.storage.provider "minio" }} + minio-access-key: {{ .Values.storage.minio.accessKey | b64enc | quote }} + minio-secret-key: {{ .Values.storage.minio.secretKey | b64enc | quote }} +{{- else if eq .Values.storage.provider "azure" }} + azure-key: {{ .Values.storage.azure.key | b64enc | quote }} +{{- end }} +{{- if and (eq .Values.storage.provider "gcs") .Values.storage.gcs.keyFile }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "accumulo.fullname" . }}-gcs-secret + labels: + {{- include "accumulo.labels" . | nindent 4 }} + {{- with .Values.global.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + gcs-key.json: {{ .Values.storage.gcs.keyFile | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/serviceaccount.yaml b/charts/accumulo/templates/serviceaccount.yaml new file mode 100644 index 00000000000..63fb578a591 --- /dev/null +++ b/charts/accumulo/templates/serviceaccount.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.auth.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "accumulo.serviceAccountName" . }} + labels: + {{- include "accumulo.labels" . | nindent 4 }} + {{- with .Values.auth.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: true +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/templates/tests/smoke-test.yaml b/charts/accumulo/templates/tests/smoke-test.yaml new file mode 100644 index 00000000000..7deeb0ce288 --- /dev/null +++ b/charts/accumulo/templates/tests/smoke-test.yaml @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{{- if .Values.dev.smokeTest.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "accumulo.fullname" . }}-smoke-test + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.global.commonAnnotations }} + {{ toYaml . | nindent 4 }} + {{- end }} +spec: + restartPolicy: Never + serviceAccountName: {{ include "accumulo.serviceAccountName" . }} + initContainers: + - name: wait-for-services + image: busybox:1.35 + command: + - /bin/sh + - -c + - | + echo "Waiting for all services to be ready..." + + echo "Checking ZooKeeper..." + until nc -z {{ include "accumulo.zookeeperHosts" . | replace ":2181" "" }} 2181; do + echo "Waiting for ZooKeeper..." + sleep 5 + done + + echo "Checking Alluxio master..." + until nc -z {{ include "accumulo.fullname" . }}-alluxio-master 19998; do + echo "Waiting for Alluxio master..." + sleep 5 + done + + echo "Checking Accumulo manager..." + until nc -z {{ include "accumulo.fullname" . }}-manager 9999; do + echo "Waiting for Accumulo manager..." + sleep 5 + done + + echo "Checking TabletServer..." + until nc -z {{ include "accumulo.fullname" . }}-tserver 9997; do + echo "Waiting for TabletServer..." + sleep 5 + done + + echo "All services are ready!" + containers: + - name: smoke-test + image: {{ .Values.dev.smokeTest.image.registry }}/{{ .Values.dev.smokeTest.image.repository }}:{{ .Values.dev.smokeTest.image.tag }} + command: + - /bin/bash + - -c + - | + set -e + + echo "=== Accumulo Smoke Test ===" + echo "Instance: {{ .Values.accumulo.instance.name }}" + echo "ZooKeeper: {{ include "accumulo.zookeeperHosts" . }}" + echo "Alluxio: {{ include "accumulo.fullname" . }}-alluxio-master:19998" + + # Wait a bit more for full initialization + echo "Waiting for system initialization..." + sleep 30 + + echo "=== Testing Accumulo Shell Commands ===" + + # Create test table + echo "Creating test table..." + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "createtable testtable" + + # Insert test data + echo "Inserting test data..." + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "insert row1 cf1 cq1 value1" -t testtable + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "insert row2 cf1 cq1 value2" -t testtable + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "insert row3 cf1 cq1 value3" -t testtable + + # Scan test data + echo "Scanning test data..." + SCAN_OUTPUT=$(/opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "scan" -t testtable) + echo "Scan output: $SCAN_OUTPUT" + + # Verify we have 3 rows + ROW_COUNT=$(echo "$SCAN_OUTPUT" | grep -c "value" || true) + echo "Found $ROW_COUNT rows" + + if [ "$ROW_COUNT" -eq 3 ]; then + echo "SUCCESS: All 3 test rows found" + else + echo "FAILED: Expected 3 rows, found $ROW_COUNT" + exit 1 + fi + + # Test table operations + echo "Testing table operations..." + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "flush -t testtable" + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "compact -t testtable" + + echo "=== Testing Alluxio Integration ===" + + # Check if data is being stored in Alluxio + echo "Checking Alluxio master status..." + curl -f {{ include "accumulo.fullname" . }}-alluxio-master:19999/ > /dev/null + echo "SUCCESS: Alluxio master is accessible" + + echo "=== Testing Monitor Web Interface ===" + + # Check if Monitor is accessible + echo "Checking Monitor web interface..." + curl -f {{ include "accumulo.fullname" . }}-monitor:{{ .Values.accumulo.monitor.service.port }}/ > /dev/null + echo "SUCCESS: Monitor web interface is accessible" + + echo "=== Cleanup ===" + + # Clean up test table + echo "Dropping test table..." + /opt/accumulo/bin/accumulo shell -u root -p {{ .Values.accumulo.instance.secret }} -e "deletetable -f testtable" + + echo "=== ALL TESTS PASSED! ===" + echo "Accumulo cluster is working correctly with Alluxio storage" + env: + {{- include "accumulo.commonEnv" . | nindent 4 }} + - name: ACCUMULO_HOME + value: "/opt/accumulo" + volumeMounts: + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo.properties + subPath: accumulo.properties + - name: accumulo-config + mountPath: /opt/accumulo/conf/accumulo-env.sh + subPath: accumulo-env.sh + volumes: + - name: accumulo-config + configMap: + name: {{ include "accumulo.fullname" . }}-config + defaultMode: 0755 +{{- end }} \ No newline at end of file diff --git a/charts/accumulo/values-dev.yaml b/charts/accumulo/values-dev.yaml new file mode 100644 index 00000000000..ddd2fc748a7 --- /dev/null +++ b/charts/accumulo/values-dev.yaml @@ -0,0 +1,153 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Development mode values for Apache Accumulo with Alluxio +# This configuration uses MinIO for local development and testing + +# Enable development mode +dev: + enabled: true + +# Accumulo configuration - reduced resources for development +accumulo: + instance: + name: "accumulo-dev" + secret: "dev-secret-change-me" + volumes: "alluxio://accumulo-dev-alluxio-master:19998/accumulo" + + # Reduced resource requirements for local development + resources: + manager: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + tserver: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1000m" + monitor: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "250m" + gc: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "250m" + compactor: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "250m" + + # Reduced replicas for development + tserver: + replicaCount: 2 + compactor: + replicaCount: 1 + +# Alluxio configuration for development +alluxio: + # Reduced resource requirements + master: + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + # Use emptyDir for journal in dev mode + journal: + storageClass: "" + size: "1Gi" + + worker: + replicaCount: 2 + resources: + requests: + memory: "1Gi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + # Use emptyDir for worker storage in dev mode + storage: + storageClass: "" + size: "5Gi" + + properties: + alluxio.worker.memory.size: "512MB" + +# Use MinIO for development storage +storage: + provider: "minio" + minio: + endpoint: "http://accumulo-dev-minio:9000" + bucket: "accumulo-data" + accessKey: "minioadmin" + secretKey: "minioadmin" + +# Enable built-in MinIO +minio: + enabled: true + defaultBuckets: "accumulo-data" + auth: + rootUser: minioadmin + rootPassword: minioadmin + persistence: + enabled: false # Use emptyDir for development + size: 5Gi + resources: + requests: + memory: 256Mi + cpu: 250m + +# Enable built-in ZooKeeper with reduced resources +zookeeper: + enabled: true + replicaCount: 1 + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 512Mi + cpu: 500m + persistence: + enabled: false # Use emptyDir for development + size: 1Gi + +# Enable smoke tests +dev: + smokeTest: + enabled: true \ No newline at end of file diff --git a/charts/accumulo/values-production-aws.yaml b/charts/accumulo/values-production-aws.yaml new file mode 100644 index 00000000000..64e5c95c980 --- /dev/null +++ b/charts/accumulo/values-production-aws.yaml @@ -0,0 +1,195 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Production values for Apache Accumulo on AWS with S3 storage +# This configuration is optimized for production workloads on AWS EKS + +# Accumulo configuration for production +accumulo: + instance: + name: "accumulo-prod" + secret: "CHANGE_THIS_SECRET_IN_PRODUCTION" + volumes: "alluxio://accumulo-prod-alluxio-master:19998/accumulo" + + # Production resource allocations + resources: + manager: + requests: + memory: "1Gi" + cpu: "1000m" + limits: + memory: "2Gi" + cpu: "2000m" + tserver: + requests: + memory: "4Gi" + cpu: "2000m" + limits: + memory: "8Gi" + cpu: "4000m" + monitor: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1000m" + gc: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1000m" + compactor: + requests: + memory: "1Gi" + cpu: "1000m" + limits: + memory: "2Gi" + cpu: "2000m" + + # High availability configuration + manager: + replicaCount: 2 + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + + tserver: + replicaCount: 6 + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + + compactor: + replicaCount: 4 + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + + # Expose Monitor via LoadBalancer for external access + monitor: + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + +# Alluxio configuration for production +alluxio: + # High availability Alluxio masters + master: + replicaCount: 3 + resources: + requests: + memory: "2Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "2000m" + # Persistent journal storage + journal: + storageClass: "gp3" + size: "100Gi" + + # Alluxio workers with local SSD caching + worker: + replicaCount: 6 + resources: + requests: + memory: "8Gi" + cpu: "2000m" + limits: + memory: "16Gi" + cpu: "4000m" + # Local NVMe SSD for caching + storage: + storageClass: "local-nvme" + size: "500Gi" + + # Production Alluxio configuration + properties: + alluxio.worker.memory.size: "4GB" + # Enhanced performance settings + alluxio.user.file.write.location.policy.class: "alluxio.client.file.policy.LocalFirstAvoidEvictionPolicy" + alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes: "2GB" + + # Optimized write modes for Accumulo workloads + pathWriteModes: + "/accumulo/wal": "THROUGH" # WAL needs immediate durability + "/accumulo/tables": "CACHE_THROUGH" # Tables benefit from caching + "/accumulo/tmp": "ASYNC_THROUGH" # Temp files can be async + "/accumulo/recovery": "THROUGH" # Recovery logs need durability + +# AWS S3 storage configuration +storage: + provider: "s3" + s3: + endpoint: "https://s3.amazonaws.com" + bucket: "your-company-accumulo-prod" + region: "us-west-2" + # Using IRSA - credentials will be provided by service account + accessKey: "" + secretKey: "" + +# External ZooKeeper (recommended for production) +zookeeper: + enabled: false + external: + hosts: "zk-cluster.your-domain.com:2181" + +# Disable built-in MinIO for production +minio: + enabled: false + +# Production authentication with IRSA +auth: + method: "serviceAccount" + serviceAccount: + create: true + name: "accumulo-prod" + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/AccumuloProdRole" + +# Global settings +global: + storageClass: "gp3" + commonLabels: + environment: "production" + team: "data-engineering" + commonAnnotations: + deployed-by: "helm" + contact: "data-team@company.com" + +# Enable monitoring for production +monitoring: + prometheus: + enabled: true + +# Network configuration +networking: + istio: + enabled: true + +# Disable development features +dev: + enabled: false + smokeTest: + enabled: false \ No newline at end of file diff --git a/charts/accumulo/values.yaml b/charts/accumulo/values.yaml new file mode 100644 index 00000000000..8528815f30f --- /dev/null +++ b/charts/accumulo/values.yaml @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Default values for Apache Accumulo with Alluxio +# This is a YAML-formatted file. + +# Global settings +global: + # Image registry, can be overridden + imageRegistry: "" + # Common labels to apply to all resources + commonLabels: {} + # Common annotations to apply to all resources + commonAnnotations: {} + # Storage class for persistent volumes + storageClass: "" + +# Accumulo configuration +accumulo: + # Accumulo instance configuration + instance: + # Instance name + name: "accumulo" + # Instance secret (change before deployment!) + secret: "DEFAULT_CHANGE_ME" + # Instance volumes - will use Alluxio + volumes: "alluxio://alluxio-master:19998/accumulo" + + # Accumulo image configuration + image: + registry: "" + repository: accumulo/accumulo + tag: "4.0.0-SNAPSHOT" + pullPolicy: IfNotPresent + # To use custom built images, set registry to your registry + # and run: scripts/build-docker.sh -r your-registry -t your-tag -p + + # Resource configurations for different components + resources: + manager: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1000m" + tserver: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + monitor: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + gc: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + compactor: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + + # Component deployment configuration + manager: + enabled: true + replicaCount: 1 + # Pod anti-affinity for high availability + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + + tserver: + enabled: true + replicaCount: 3 + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + + monitor: + enabled: true + replicaCount: 1 + service: + type: ClusterIP + port: 9995 + + gc: + enabled: true + replicaCount: 1 + + compactor: + enabled: true + replicaCount: 2 + podAntiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + +# Alluxio configuration +alluxio: + enabled: true + + # Alluxio image + image: + registry: docker.io + repository: alluxio/alluxio + tag: "2.9.4" + pullPolicy: IfNotPresent + + # Master configuration + master: + enabled: true + replicaCount: 1 + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" + # Journal storage for master metadata + journal: + storageClass: "" + size: "10Gi" + + # Worker configuration + worker: + enabled: true + replicaCount: 3 + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "2000m" + # Local cache storage + storage: + storageClass: "" + size: "50Gi" + + # Alluxio properties configuration + properties: + # Under storage configuration - will be set based on storage provider + alluxio.master.mount.table.root.ufs: "" + # Cache settings + alluxio.user.file.write.location.policy.class: "alluxio.client.file.policy.LocalFirstPolicy" + alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes: "512MB" + # Memory allocation + alluxio.worker.memory.size: "1GB" + alluxio.master.journal.type: "UFS" + + # Per-path write modes for different Accumulo data + pathWriteModes: + "/accumulo/wal": "THROUGH" # WAL needs immediate durability + "/accumulo/tables": "CACHE_THROUGH" # Tables benefit from caching + "/accumulo/tmp": "ASYNC_THROUGH" # Temp files can be async + +# Storage provider configuration +storage: + # Storage provider: s3, gcs, azure, or minio (for local dev) + provider: "minio" + + # S3 configuration + s3: + endpoint: "" + bucket: "accumulo-data" + region: "us-west-2" + accessKey: "" + secretKey: "" + + # GCS configuration + gcs: + projectId: "" + bucket: "accumulo-data" + keyFile: "" + + # Azure configuration + azure: + account: "" + container: "accumulo-data" + key: "" + + # MinIO configuration (for local development) + minio: + endpoint: "http://minio:9000" + bucket: "accumulo-data" + accessKey: "minioadmin" + secretKey: "minioadmin" + +# ZooKeeper configuration +zookeeper: + # Enable embedded ZooKeeper (set to false to use external) + enabled: true + replicaCount: 3 + + # External ZooKeeper configuration (when enabled: false) + external: + hosts: "localhost:2181" + +# Built-in MinIO for local development +minio: + # Enable MinIO for local development + enabled: true + defaultBuckets: "accumulo-data" + auth: + rootUser: minioadmin + rootPassword: minioadmin + persistence: + enabled: true + size: 10Gi + +# Security and authentication +auth: + # Cloud provider authentication method + # Options: serviceAccount, workloadIdentity, managedIdentity, accessKeys + method: "accessKeys" + + # Service account configuration (for IRSA, Workload Identity, etc.) + serviceAccount: + create: true + name: "" + annotations: {} + +# Monitoring and observability +monitoring: + # Enable Prometheus metrics + prometheus: + enabled: false + + # Enable tracing + tracing: + enabled: false + jaegerEndpoint: "" + +# Networking +networking: + # Service mesh integration + istio: + enabled: false + +# Development and testing +dev: + # Enable development mode (uses MinIO, reduces resource requirements) + enabled: false + + # Smoke test configuration + smokeTest: + enabled: true + image: + registry: docker.io + repository: accumulo/accumulo + tag: "4.0.0-SNAPSHOT" \ No newline at end of file diff --git a/docker/accumulo/Dockerfile b/docker/accumulo/Dockerfile new file mode 100644 index 00000000000..d024ac8ae5b --- /dev/null +++ b/docker/accumulo/Dockerfile @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Apache Accumulo Docker Image +# Based on the official Accumulo distribution + +FROM eclipse-temurin:17-jre-jammy + +# Set environment variables +ENV ACCUMULO_VERSION=4.0.0-SNAPSHOT +ENV HADOOP_VERSION=3.3.6 +ENV ZOOKEEPER_VERSION=3.8.4 +ENV ACCUMULO_HOME=/opt/accumulo +ENV HADOOP_HOME=/opt/hadoop +ENV ZOOKEEPER_HOME=/opt/zookeeper +ENV JAVA_HOME=/opt/java/openjdk + +# Install required packages +RUN apt-get update && \ + apt-get install -y \ + curl \ + wget \ + netcat \ + procps \ + bash \ + gettext-base \ + && rm -rf /var/lib/apt/lists/* + +# Create accumulo user +RUN groupadd -r accumulo && \ + useradd -r -g accumulo -d $ACCUMULO_HOME -s /bin/bash accumulo + +# Create directories +RUN mkdir -p $ACCUMULO_HOME $HADOOP_HOME $ZOOKEEPER_HOME && \ + chown -R accumulo:accumulo $ACCUMULO_HOME $HADOOP_HOME $ZOOKEEPER_HOME + +# Download and install Hadoop (client libraries only) +RUN wget -q https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt && \ + mv /opt/hadoop-${HADOOP_VERSION} $HADOOP_HOME && \ + rm hadoop-${HADOOP_VERSION}.tar.gz && \ + chown -R accumulo:accumulo $HADOOP_HOME + +# Download and install ZooKeeper (client libraries only) +RUN wget -q https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && \ + tar -xzf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /opt && \ + mv /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin $ZOOKEEPER_HOME && \ + rm apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && \ + chown -R accumulo:accumulo $ZOOKEEPER_HOME + +# Copy Accumulo distribution (built from source) +COPY --chown=accumulo:accumulo dist/ $ACCUMULO_HOME/ + +# Create necessary directories +RUN mkdir -p $ACCUMULO_HOME/logs $ACCUMULO_HOME/walogs $ACCUMULO_HOME/conf && \ + chown -R accumulo:accumulo $ACCUMULO_HOME + +# Set up classpath and environment +RUN echo 'export JAVA_HOME=/opt/java/openjdk' >> /etc/environment && \ + echo 'export HADOOP_HOME=/opt/hadoop' >> /etc/environment && \ + echo 'export ZOOKEEPER_HOME=/opt/zookeeper' >> /etc/environment && \ + echo 'export ACCUMULO_HOME=/opt/accumulo' >> /etc/environment + +# Copy entrypoint script +COPY docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# Switch to accumulo user +USER accumulo +WORKDIR $ACCUMULO_HOME + +# Set default environment variables +ENV PATH=$ACCUMULO_HOME/bin:$HADOOP_HOME/bin:$ZOOKEEPER_HOME/bin:$PATH +ENV ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs +ENV HADOOP_CONF_DIR=$ACCUMULO_HOME/conf +ENV ACCUMULO_CONF_DIR=$ACCUMULO_HOME/conf + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD $ACCUMULO_HOME/bin/accumulo info || exit 1 + +# Default command +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["help"] \ No newline at end of file diff --git a/docker/accumulo/docker-entrypoint.sh b/docker/accumulo/docker-entrypoint.sh new file mode 100755 index 00000000000..b57c0b2e12c --- /dev/null +++ b/docker/accumulo/docker-entrypoint.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +# Default configuration directory +ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME"/conf} + +# Function to wait for a service to be available +wait_for_service() { + local host=$1 + local port=$2 + local service_name=$3 + local timeout=${4:-300} + + echo "Waiting for $service_name at $host:$port..." + local count=0 + until nc -z "$host" "$port" || [ "$count" -eq "$timeout" ]; do + sleep 1 + ((count++)) + done + + if [ "$count" -eq "$timeout" ]; then + echo "ERROR: Timeout waiting for $service_name at $host:$port" + exit 1 + fi + + echo "$service_name is available at $host:$port" +} + +# Function to setup configuration templates +setup_config() { + echo "Setting up Accumulo configuration..." + + # Set default values if not provided + export ACCUMULO_INSTANCE_NAME=${ACCUMULO_INSTANCE_NAME:-accumulo} + export ACCUMULO_INSTANCE_SECRET=${ACCUMULO_INSTANCE_SECRET:-DEFAULT} + export ZOOKEEPER_HOSTS=${ZOOKEEPER_HOSTS:-localhost:2181} + export ACCUMULO_INSTANCE_VOLUMES=${ACCUMULO_INSTANCE_VOLUMES:-file:///accumulo} + + # Process configuration templates if they exist + if [ -d "$ACCUMULO_CONF_DIR/templates" ]; then + echo "Processing configuration templates..." + for template in "$ACCUMULO_CONF_DIR/templates"/*.template; do + if [ -f "$template" ]; then + filename=$(basename "$template" .template) + echo "Processing template: $template -> $ACCUMULO_CONF_DIR/$filename" + envsubst <"$template" >"$ACCUMULO_CONF_DIR/$filename" + fi + done + fi + + # Ensure log directory exists + mkdir -p "$ACCUMULO_LOG_DIR" +} + +# Function to initialize Accumulo instance +init_accumulo() { + echo "Checking if Accumulo instance needs initialization..." + + # Wait for ZooKeeper + local zk_host + local zk_port + zk_host=$(echo "$ZOOKEEPER_HOSTS" | cut -d: -f1) + zk_port=$(echo "$ZOOKEEPER_HOSTS" | cut -d: -f2) + wait_for_service "$zk_host" "$zk_port" "ZooKeeper" + + # Check if instance already exists + if "$ACCUMULO_HOME"/bin/accumulo org.apache.accumulo.server.util.ListInstances 2>/dev/null | grep -q "$ACCUMULO_INSTANCE_NAME"; then + echo "Accumulo instance '$ACCUMULO_INSTANCE_NAME' already exists" + else + echo "Initializing Accumulo instance '$ACCUMULO_INSTANCE_NAME'..." + "$ACCUMULO_HOME"/bin/accumulo init \ + --instance-name "$ACCUMULO_INSTANCE_NAME" \ + --password "$ACCUMULO_INSTANCE_SECRET" \ + --clear-instance-name + fi +} + +# Function to start specific Accumulo service +start_service() { + local service=$1 + echo "Starting Accumulo $service..." + + case "$service" in + manager | master) + # Wait for ZooKeeper and optionally initialize + if [ "${ACCUMULO_AUTO_INIT:-true}" = "true" ]; then + init_accumulo + fi + exec "$ACCUMULO_HOME"/bin/accumulo manager + ;; + tserver) + # Wait for manager to be available + if [ -n "${ACCUMULO_MANAGER_HOST:-}" ]; then + wait_for_service "${ACCUMULO_MANAGER_HOST}" "${ACCUMULO_MANAGER_PORT:-9999}" "Accumulo Manager" + fi + exec "$ACCUMULO_HOME"/bin/accumulo tserver + ;; + monitor) + # Wait for manager to be available + if [ -n "${ACCUMULO_MANAGER_HOST:-}" ]; then + wait_for_service "${ACCUMULO_MANAGER_HOST}" "${ACCUMULO_MANAGER_PORT:-9999}" "Accumulo Manager" + fi + exec "$ACCUMULO_HOME"/bin/accumulo monitor + ;; + gc) + # Wait for manager to be available + if [ -n "${ACCUMULO_MANAGER_HOST:-}" ]; then + wait_for_service "${ACCUMULO_MANAGER_HOST}" "${ACCUMULO_MANAGER_PORT:-9999}" "Accumulo Manager" + fi + exec "$ACCUMULO_HOME"/bin/accumulo gc + ;; + compactor) + # Wait for manager to be available + if [ -n "${ACCUMULO_MANAGER_HOST:-}" ]; then + wait_for_service "${ACCUMULO_MANAGER_HOST}" "${ACCUMULO_MANAGER_PORT:-9999}" "Accumulo Manager" + fi + local queue="${ACCUMULO_COMPACTOR_QUEUE:-default}" + exec "$ACCUMULO_HOME"/bin/accumulo compactor -q "$queue" + ;; + shell) + exec "$ACCUMULO_HOME"/bin/accumulo shell "$@" + ;; + *) + # Pass through any other accumulo commands + exec "$ACCUMULO_HOME"/bin/accumulo "$@" + ;; + esac +} + +# Main execution +echo "Accumulo Docker Container Starting..." +echo "Command: $*" + +# Setup configuration +setup_config + +# Check if this is an Accumulo service command +if [ $# -eq 0 ]; then + echo "No command specified. Use: manager, tserver, monitor, gc, compactor, shell, or any accumulo command" + exec "$ACCUMULO_HOME"/bin/accumulo help +elif [ "$1" = "manager" ] || [ "$1" = "master" ] || [ "$1" = "tserver" ] || [ "$1" = "monitor" ] || [ "$1" = "gc" ] || [ "$1" = "compactor" ]; then + start_service "$@" +else + # Pass through to accumulo binary + exec "$ACCUMULO_HOME"/bin/accumulo "$@" +fi diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000000..f9f7d239bee --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,198 @@ + + +# Accumulo Deployment Scripts + +This directory contains helper scripts for building, configuring, and deploying Apache Accumulo with Alluxio on Kubernetes. + +## Scripts Overview + +### `build-docker.sh` +Builds Docker images for Apache Accumulo from the source code in this repository. + +**Usage:** +```bash +# Build for local use +./scripts/build-docker.sh + +# Build and push to registry +./scripts/build-docker.sh -r myregistry.com/accumulo -t latest -p + +# Build for multiple platforms +./scripts/build-docker.sh --platform linux/amd64,linux/arm64 +``` + +**Prerequisites:** +- Docker installed and running +- Maven (for building Accumulo distribution) +- Source code built: `mvn clean package -DskipTests` + +### `generate-secrets.sh` +Generates secure configuration values and secrets for Helm deployment. + +**Usage:** +```bash +# Interactive mode (recommended) +./scripts/generate-secrets.sh -o my-values.yaml + +# Non-interactive with defaults +./scripts/generate-secrets.sh --non-interactive -i prod-accumulo -o prod-values.yaml + +# For specific namespace +./scripts/generate-secrets.sh -n accumulo-prod -o prod-values.yaml +``` + +**Features:** +- Generates cryptographically secure instance secrets +- Interactive configuration for different cloud providers +- Support for AWS S3, GCS, Azure Blob Storage, and MinIO +- Configures authentication methods (IRSA, Workload Identity, etc.) + +### `helm-deploy.sh` +Comprehensive Helm deployment helper with dependency management. + +**Usage:** +```bash +# Install with development values +./scripts/helm-deploy.sh install -r accumulo-dev -f ./charts/accumulo/values-dev.yaml + +# Install with generated configuration +./scripts/helm-deploy.sh install -r my-accumulo -f values-generated.yaml --create-namespace -n accumulo + +# Upgrade existing deployment +./scripts/helm-deploy.sh upgrade -r accumulo-prod -f production-values.yaml + +# Run tests +./scripts/helm-deploy.sh test -r accumulo-dev + +# Check status +./scripts/helm-deploy.sh status -r accumulo-dev +``` + +**Features:** +- Automatic dependency management (creates embedded ZooKeeper and MinIO charts) +- Validation of environment and prerequisites +- Support for all Helm operations (install, upgrade, uninstall, test, status) +- Comprehensive error handling and logging + +## Quick Start Workflow + +### 1. Development Setup +```bash +# Generate development configuration +./scripts/generate-secrets.sh -o values-dev-generated.yaml --non-interactive + +# Deploy to local Kubernetes cluster +./scripts/helm-deploy.sh install -r accumulo-dev -f values-dev-generated.yaml --create-namespace -n accumulo-dev + +# Run smoke tests +./scripts/helm-deploy.sh test -r accumulo-dev -n accumulo-dev +``` + +### 2. Production Setup +```bash +# Generate production configuration interactively +./scripts/generate-secrets.sh -o values-production.yaml -i accumulo-prod + +# Review and customize the generated configuration +vim values-production.yaml + +# Build and push custom images (optional) +./scripts/build-docker.sh -r your-registry.com/accumulo -t v1.0.0 -p + +# Deploy to production +./scripts/helm-deploy.sh install -r accumulo-prod -f values-production.yaml --create-namespace -n accumulo-prod +``` + +### 3. Building Custom Images + +If you want to use custom Accumulo images built from this repository: + +```bash +# Build the Accumulo distribution first +mvn clean package -DskipTests -pl assemble -am + +# Build Docker image +./scripts/build-docker.sh -r your-registry.com/accumulo -t 4.0.0-SNAPSHOT + +# Push to registry +./scripts/build-docker.sh -r your-registry.com/accumulo -t 4.0.0-SNAPSHOT -p + +# Update values file to use custom image +# Set accumulo.image.registry to "your-registry.com" +``` + +## Troubleshooting + +### Common Issues + +1. **Helm dependency errors** + - The `helm-deploy.sh` script automatically creates embedded dependencies + - No need to run `helm dependency build` manually + +2. **Image pull errors** + - If using custom images, ensure they are built and pushed to a registry accessible by your cluster + - Check image registry and tag configuration in values file + +3. **Permission errors** + - Ensure scripts have execute permissions: `chmod +x scripts/*.sh` + - Check Kubernetes RBAC permissions for the service account + +4. **Network connectivity** + - For development, ensure MinIO and ZooKeeper are accessible within the cluster + - For production, verify cloud storage and authentication configuration + +### Debug Commands + +```bash +# Check Helm deployment status +./scripts/helm-deploy.sh status -r your-release -n your-namespace + +# Run tests to validate deployment +./scripts/helm-deploy.sh test -r your-release -n your-namespace + +# Check pod logs +kubectl logs -l app.kubernetes.io/name=accumulo -n your-namespace + +# Access Accumulo shell +kubectl exec -it deployment/your-release-manager -n your-namespace -- /opt/accumulo/bin/accumulo shell -u root +``` + +## Environment Variables + +Scripts support the following environment variables: + +- `DOCKER_REGISTRY`: Default registry for Docker images +- `DOCKER_TAG`: Default tag for Docker images +- `KUBECONFIG`: Path to Kubernetes configuration file + +## Security Notes + +- **Instance Secrets**: The `generate-secrets.sh` script creates cryptographically secure secrets. Store these safely. +- **Cloud Credentials**: Use cloud-native authentication methods (IRSA, Workload Identity) instead of access keys when possible. +- **Container Images**: Consider using signed images and admission controllers in production. + +## Contributing + +When adding new scripts: +1. Follow the existing error handling and logging patterns +2. Add comprehensive help text and examples +3. Include validation for prerequisites +4. Test with both interactive and non-interactive modes +5. Update this README with usage information \ No newline at end of file diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh new file mode 100755 index 00000000000..88b3e91cba1 --- /dev/null +++ b/scripts/build-docker.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Build script for Apache Accumulo Docker images + +set -euo pipefail + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +# Default values +REGISTRY="${DOCKER_REGISTRY:-accumulo}" +TAG="${DOCKER_TAG:-4.0.0-SNAPSHOT}" +BUILD_ARGS="" +PUSH=false +PLATFORM="" + +# Usage function +usage() { + cat </dev/null; then + log_error "Maven is required to build Accumulo distribution" + exit 1 + fi + + # Build the distribution + if ! mvn clean package -DskipTests -pl assemble -am; then + log_error "Failed to build Accumulo distribution" + exit 1 + fi + fi + + # Extract distribution for Docker build + local dist_dir="$PROJECT_DIR/docker/accumulo/dist" + mkdir -p "$dist_dir" + + local tarball + tarball=$(find "$PROJECT_DIR/assemble/target" -name "accumulo-*-bin.tar.gz" | head -1) + if [ -z "$tarball" ]; then + log_error "No Accumulo distribution found in assemble/target" + exit 1 + fi + + log_info "Extracting distribution: $(basename "$tarball")" + tar -xzf "$tarball" -C "$dist_dir" --strip-components=1 + + log_success "Accumulo distribution prepared" +} + +# Build Docker image +build_docker_image() { + local image_name="$REGISTRY/accumulo:$TAG" + local dockerfile="$PROJECT_DIR/docker/accumulo/Dockerfile" + local context="$PROJECT_DIR/docker/accumulo" + + log_info "Building Docker image: $image_name" + + # Prepare build command + local build_cmd="docker build" + + if [ -n "$PLATFORM" ]; then + build_cmd="$build_cmd --platform $PLATFORM" + fi + + build_cmd="$build_cmd $BUILD_ARGS -t $image_name -f $dockerfile $context" + + log_info "Build command: $build_cmd" + + # Execute build + if eval "$build_cmd"; then + log_success "Successfully built $image_name" + else + log_error "Failed to build $image_name" + exit 1 + fi + + # Push if requested + if [ "$PUSH" = true ]; then + log_info "Pushing image: $image_name" + if docker push "$image_name"; then + log_success "Successfully pushed $image_name" + else + log_error "Failed to push $image_name" + exit 1 + fi + fi +} + +# Validate environment +validate_environment() { + log_info "Validating build environment..." + + # Check Docker + if ! command -v docker &>/dev/null; then + log_error "Docker is required but not installed" + exit 1 + fi + + # Check Docker daemon + if ! docker info &>/dev/null; then + log_error "Docker daemon is not running" + exit 1 + fi + + log_success "Environment validation passed" +} + +# Main execution +main() { + log_info "Starting Accumulo Docker build process" + + validate_environment + build_accumulo_dist + build_docker_image + + log_success "Build process completed successfully!" + log_info "Image: $REGISTRY/accumulo:$TAG" + + # Show image info + docker images "$REGISTRY/accumulo:$TAG" +} + +# Execute main function +main "$@" diff --git a/scripts/generate-secrets.sh b/scripts/generate-secrets.sh new file mode 100755 index 00000000000..ebe660649aa --- /dev/null +++ b/scripts/generate-secrets.sh @@ -0,0 +1,438 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Generate secrets and configuration for Accumulo Helm deployment + +set -euo pipefail + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +# Default values +OUTPUT_FILE="" +INSTANCE_NAME="accumulo" +NAMESPACE="default" +INTERACTIVE=true +OVERWRITE=false + +# Usage function +usage() { + cat </dev/null; then + uuidgen | tr '[:upper:]' '[:lower:]' + else + cat /proc/sys/kernel/random/uuid 2>/dev/null || echo "$(date +%s)-$(shuf -i 1000-9999 -n 1)" + fi +} + +# Interactive input function +get_input() { + local prompt="$1" + local default="$2" + local secret="${3:-false}" + + if [ "$INTERACTIVE" = false ]; then + echo "$default" + return + fi + + if [ "$secret" = true ]; then + echo -n "$prompt [$default]: " >&2 + read -rs input + echo >&2 + else + echo -n "$prompt [$default]: " >&2 + read -r input + fi + + echo "${input:-$default}" +} + +# Validate tools +validate_tools() { + local missing_tools=() + + if ! command -v openssl &>/dev/null; then + missing_tools+=("openssl") + fi + + if [ ${#missing_tools[@]} -gt 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + log_info "Please install the missing tools and try again" + exit 1 + fi +} + +# Generate configuration +generate_config() { + log_info "Generating Accumulo configuration..." + + # Check if output file exists + if [ -f "$OUTPUT_FILE" ] && [ "$OVERWRITE" = false ]; then + log_error "Output file already exists: $OUTPUT_FILE" + log_info "Use --overwrite to overwrite existing file" + exit 1 + fi + + # Collect configuration values + log_info "Collecting configuration values..." + + local instance_secret + local storage_provider + local s3_bucket + local s3_region + local s3_access_key + local s3_secret_key + local gcs_project + local gcs_bucket + local azure_account + local azure_container + local azure_key + + # Instance configuration + if [ "$INTERACTIVE" = true ]; then + echo + echo "=== Accumulo Instance Configuration ===" + fi + + INSTANCE_NAME=$(get_input "Instance name" "$INSTANCE_NAME") + instance_secret=$(generate_secret 32) + + if [ "$INTERACTIVE" = true ]; then + log_info "Generated instance secret: $instance_secret" + echo + echo "=== Storage Configuration ===" + echo "Choose storage provider:" + echo "1) AWS S3" + echo "2) Google Cloud Storage" + echo "3) Azure Blob Storage" + echo "4) MinIO (development)" + echo -n "Selection [4]: " + read -r storage_choice + storage_choice=${storage_choice:-4} + else + storage_choice=4 # Default to MinIO for non-interactive + fi + + case $storage_choice in + 1) + storage_provider="s3" + s3_bucket=$(get_input "S3 bucket name" "${INSTANCE_NAME}-data") + s3_region=$(get_input "AWS region" "us-west-2") + s3_access_key=$(get_input "AWS access key (leave empty for IRSA)" "") + if [ -n "$s3_access_key" ]; then + s3_secret_key=$(get_input "AWS secret key" "" true) + fi + ;; + 2) + storage_provider="gcs" + gcs_project=$(get_input "GCP project ID" "") + gcs_bucket=$(get_input "GCS bucket name" "${INSTANCE_NAME}-data") + ;; + 3) + storage_provider="azure" + azure_account=$(get_input "Azure storage account" "") + azure_container=$(get_input "Azure container name" "${INSTANCE_NAME}-data") + azure_key=$(get_input "Azure access key (leave empty for Managed Identity)" "" true) + ;; + *) + storage_provider="minio" + ;; + esac + + # Generate values file + log_info "Generating values file: $OUTPUT_FILE" + + cat >"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" <>"$OUTPUT_FILE" </dev/null; then + log_error "Helm is required but not installed" + exit 1 + fi + + # Check kubectl + if ! command -v kubectl &>/dev/null; then + log_error "kubectl is required but not installed" + exit 1 + fi + + # Check cluster connectivity + if ! kubectl cluster-info &>/dev/null; then + log_error "Cannot connect to Kubernetes cluster" + exit 1 + fi + + # Check chart exists + if [ ! -f "$CHART_DIR/Chart.yaml" ]; then + log_error "Helm chart not found at $CHART_DIR" + exit 1 + fi + + log_success "Environment validation passed" +} + +# Setup dependencies +setup_dependencies() { + log_info "Setting up Helm chart dependencies..." + + # Create embedded dependencies instead of external ones + # This avoids the network connectivity issues + local deps_dir="$CHART_DIR/charts" + mkdir -p "$deps_dir" + + # Create simple ZooKeeper subchart + if [ ! -f "$deps_dir/zookeeper/Chart.yaml" ]; then + log_info "Creating embedded ZooKeeper chart..." + mkdir -p "$deps_dir/zookeeper/templates" + + cat >"$deps_dir/zookeeper/Chart.yaml" <<'EOF' +apiVersion: v2 +name: zookeeper +description: ZooKeeper for Accumulo +version: 1.0.0 +appVersion: "3.8.4" +EOF + + cat >"$deps_dir/zookeeper/values.yaml" <<'EOF' +enabled: true +replicaCount: 1 +image: + registry: docker.io + repository: zookeeper + tag: "3.8.4" + pullPolicy: IfNotPresent +resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 512Mi + cpu: 500m +persistence: + enabled: false + size: 1Gi +EOF + + cat >"$deps_dir/zookeeper/templates/deployment.yaml" <<'EOF' +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-zookeeper + labels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: zookeeper + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + env: + - name: ALLOW_ANONYMOUS_LOGIN + value: "yes" + resources: + {{- toYaml .Values.resources | nindent 10 }} + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "accumulo.fullname" . }}-zookeeper-data + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +EOF + + cat >"$deps_dir/zookeeper/templates/service.yaml" <<'EOF' +{{- if .Values.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-zookeeper + labels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + type: ClusterIP + ports: + - port: 2181 + targetPort: client + protocol: TCP + name: client + selector: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +EOF + fi + + # Create simple MinIO subchart + if [ ! -f "$deps_dir/minio/Chart.yaml" ]; then + log_info "Creating embedded MinIO chart..." + mkdir -p "$deps_dir/minio/templates" + + cat >"$deps_dir/minio/Chart.yaml" <<'EOF' +apiVersion: v2 +name: minio +description: MinIO for Accumulo development +version: 1.0.0 +appVersion: "2024.1.1" +EOF + + cat >"$deps_dir/minio/values.yaml" <<'EOF' +enabled: true +defaultBuckets: "accumulo-data" +auth: + rootUser: minioadmin + rootPassword: minioadmin +image: + registry: docker.io + repository: minio/minio + tag: "RELEASE.2024-01-01T16-36-33Z" + pullPolicy: IfNotPresent +resources: + requests: + memory: 256Mi + cpu: 250m +persistence: + enabled: false + size: 10Gi +EOF + + cat >"$deps_dir/minio/templates/deployment.yaml" <<'EOF' +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "accumulo.fullname" . }}-minio + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: minio + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + mkdir -p /data/{{ .Values.defaultBuckets }} + /usr/bin/docker-entrypoint.sh minio server /data --console-address ":9001" + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: MINIO_ROOT_USER + value: {{ .Values.auth.rootUser }} + - name: MINIO_ROOT_PASSWORD + value: {{ .Values.auth.rootPassword }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "accumulo.fullname" . }}-minio-data + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +EOF + + cat >"$deps_dir/minio/templates/service.yaml" <<'EOF' +{{- if .Values.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-minio + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + type: ClusterIP + ports: + - port: 9000 + targetPort: api + protocol: TCP + name: api + - port: 9001 + targetPort: console + protocol: TCP + name: console + selector: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +EOF + fi + + log_success "Dependencies setup complete" +} + +# Execute Helm action +execute_action() { + local cmd_args=() + + case "$ACTION" in + install) + if [ -z "$RELEASE_NAME" ]; then + log_error "Release name is required for install action" + exit 1 + fi + + cmd_args=("install" "$RELEASE_NAME" "$CHART_DIR") + ;; + upgrade) + if [ -z "$RELEASE_NAME" ]; then + log_error "Release name is required for upgrade action" + exit 1 + fi + + cmd_args=("upgrade" "$RELEASE_NAME" "$CHART_DIR") + ;; + uninstall) + if [ -z "$RELEASE_NAME" ]; then + log_error "Release name is required for uninstall action" + exit 1 + fi + + cmd_args=("uninstall" "$RELEASE_NAME") + ;; + test) + if [ -z "$RELEASE_NAME" ]; then + log_error "Release name is required for test action" + exit 1 + fi + + cmd_args=("test" "$RELEASE_NAME") + ;; + status) + if [ -z "$RELEASE_NAME" ]; then + log_error "Release name is required for status action" + exit 1 + fi + + cmd_args=("status" "$RELEASE_NAME") + ;; + *) + log_error "Unknown action: $ACTION" + exit 1 + ;; + esac + + # Add common options + if [ "$ACTION" = "install" ] || [ "$ACTION" = "upgrade" ]; then + if [ -n "$VALUES_FILE" ]; then + cmd_args+=("-f" "$VALUES_FILE") + fi + + cmd_args+=("--timeout" "$TIMEOUT") + + if [ "$WAIT" = true ]; then + cmd_args+=("--wait") + fi + + if [ "$CREATE_NAMESPACE" = true ]; then + cmd_args+=("--create-namespace") + fi + fi + + # Add namespace + cmd_args+=("--namespace" "$NAMESPACE") + + # Add dry-run if requested + if [ "$DRY_RUN" = true ]; then + cmd_args+=("--dry-run") + fi + + # Execute command + log_info "Executing: helm ${cmd_args[*]}" + + if helm "${cmd_args[@]}"; then + log_success "$ACTION completed successfully" + else + log_error "$ACTION failed" + exit 1 + fi +} + +# Main execution +main() { + log_info "Starting Helm deployment for Accumulo" + log_info "Action: $ACTION" + log_info "Release: ${RELEASE_NAME:-N/A}" + log_info "Namespace: $NAMESPACE" + + validate_environment + + if [ "$ACTION" = "install" ] || [ "$ACTION" = "upgrade" ]; then + setup_dependencies + fi + + execute_action + + log_success "Operation completed successfully!" +} + +# Execute main function +main "$@"