Skip to content

Commit ee69d10

Browse files
ritwik-gritwik-g
committed
Added some low level design documents
1 parent e37424f commit ee69d10

File tree

4 files changed

+846
-135
lines changed

4 files changed

+846
-135
lines changed

‎docs/ADRs/001-helm-values-manager.md‎

Lines changed: 134 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ Managing configurations and secrets across multiple Kubernetes deployments is a
99
Key challenges include:
1010
- Ensuring configurations remain consistent across different environments (e.g., dev, staging, production).
1111
- Managing sensitive values securely using external secret management systems.
12-
- Automating the generation of `values.yaml` while integrating with GitOps tools like ArgoCD.
12+
- Automating the generation of `values.json` while integrating with GitOps tools like ArgoCD.
1313
- Providing a user-friendly CLI that integrates well with Helm workflows.
1414

1515
## Decision
@@ -22,110 +22,145 @@ We have decided to implement the **Helm Values Manager** as a **Helm plugin writ
2222
4. **Secret Storage Abstraction:** Securely manages sensitive values by integrating with AWS Secrets Manager, Azure Key Vault, and HashiCorp Vault.
2323
5. **CLI-Based Approach:** Interactive commands for managing configurations and secrets.
2424
6. **Autocomplete Support:** Smooth CLI experience.
25-
7. **ArgoCD Compatibility:** Generates `values.yaml` dynamically for GitOps workflows.
25+
7. **ArgoCD Compatibility:** Generates `values.json` dynamically for GitOps workflows.
26+
8. **JSON for Configuration:** Using JSON for configuration files provides better schema validation and consistent parsing across different platforms.
2627

27-
## YAML Structure
28+
## Configuration Structure
2829

2930
The configuration follows this structure:
3031

31-
```yaml
32-
version: "1.0" # Schema version
33-
release: my-release
34-
35-
deployments:
36-
dev:
37-
secrets_backend: aws_secrets_manager
38-
secrets_config:
39-
region: us-west-2
40-
secret_prefix: "/dev/myapp/"
41-
auth:
42-
type: env # Use AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
43-
# Alternative: type: file, path: "~/.aws/credentials"
44-
# Alternative: type: direct
45-
# access_key_id: "AKIA..."
46-
# secret_access_key: "xyz..."
47-
48-
staging:
49-
secrets_backend: google_secret_manager
50-
secrets_config:
51-
project_id: "my-gcp-project"
52-
secret_prefix: "myapp-staging-"
53-
auth:
54-
type: file
55-
path: "/path/to/gcp-service-account.json"
56-
# Alternative: type: env, credential_env: "GOOGLE_APPLICATION_CREDENTIALS"
57-
# Alternative: type: direct
58-
# credentials_json: "{...}"
59-
60-
prod:
61-
secrets_backend: azure_key_vault
62-
secrets_config:
63-
vault_url: "https://my-prod-vault.vault.azure.net"
64-
auth:
65-
type: managed_identity # Use Azure Managed Identity
66-
# Alternative: type: service_principal
67-
# tenant_id: "${AZURE_TENANT_ID}"
68-
# client_id: "${AZURE_CLIENT_ID}"
69-
# client_secret: "${AZURE_CLIENT_SECRET}"
70-
71-
local:
72-
secrets_backend: git_secret
73-
secrets_config:
74-
gpg_key: "${GPG_KEY}" # GPG key for decryption
75-
secret_files_path: "./.gitsecret" # Path to git-secret files
76-
auth:
77-
type: file
78-
path: "~/.gnupg/secring.gpg"
79-
# Alternative: type: env
80-
# passphrase_env: "GIT_SECRET_PASSPHRASE"
81-
# Alternative: type: direct
82-
# passphrase: "your-passphrase"
83-
84-
config:
85-
- key: DATABASE_URL
86-
path: global.database.url
87-
description: "Database connection string for the application"
88-
required: true
89-
sensitive: true
90-
values:
91-
dev: "mydb://dev-connection"
92-
staging: "mydb://staging-connection"
93-
prod: "mydb://prod-connection"
94-
local: "mydb://localhost"
95-
96-
- key: LOG_LEVEL
97-
path: global.logging.level
98-
description: "Application logging verbosity level"
99-
required: false
100-
sensitive: false
101-
values:
102-
dev: "debug"
103-
staging: "info"
104-
prod: "warn"
105-
local: "debug"
32+
```json
33+
{
34+
"version": "1.0",
35+
"release": "my-release",
36+
"deployments": {
37+
"dev": {
38+
"secrets_backend": "aws_secrets_manager",
39+
"secrets_config": {
40+
"region": "us-west-2",
41+
"secret_prefix": "/dev/myapp/",
42+
"auth": {
43+
"type": "env"
44+
}
45+
}
46+
},
47+
"staging": {
48+
"secrets_backend": "google_secret_manager",
49+
"secrets_config": {
50+
"project_id": "my-gcp-project",
51+
"secret_prefix": "myapp-staging-",
52+
"auth": {
53+
"type": "file",
54+
"path": "/path/to/gcp-service-account.json"
55+
}
56+
}
57+
},
58+
"prod": {
59+
"secrets_backend": "azure_key_vault",
60+
"secrets_config": {
61+
"vault_url": "https://my-prod-vault.vault.azure.net",
62+
"auth": {
63+
"type": "managed_identity"
64+
}
65+
}
66+
},
67+
"local": {
68+
"secrets_backend": "git_secret",
69+
"secrets_config": {
70+
"gpg_key": "${GPG_KEY}",
71+
"secret_files_path": "./.gitsecret",
72+
"auth": {
73+
"type": "file",
74+
"path": "~/.gnupg/secring.gpg"
75+
}
76+
}
77+
}
78+
},
79+
"config": [
80+
{
81+
"path": "global.database.url",
82+
"description": "Database connection string for the application",
83+
"required": true,
84+
"sensitive": true,
85+
"values": {
86+
"dev": "mydb://dev-connection",
87+
"staging": "mydb://staging-connection",
88+
"prod": "mydb://prod-connection",
89+
"local": "mydb://localhost"
90+
}
91+
},
92+
{
93+
"path": "global.logging.level",
94+
"description": "Application logging verbosity level",
95+
"required": false,
96+
"sensitive": false,
97+
"values": {
98+
"dev": "DEBUG",
99+
"staging": "INFO",
100+
"prod": "WARN",
101+
"local": "DEBUG"
102+
}
103+
}
104+
]
105+
}
106106
```
107107

108-
### Secret Backend Configuration
109-
110-
The configuration supports multiple secret backend types with flexible authentication methods:
111-
112-
1. **Authentication Methods**:
113-
- `env`: Use environment variables
114-
- `file`: Use credential files
115-
- `direct`: Direct credential specification (not recommended for production)
116-
- `managed_identity`: For cloud-native authentication (Azure)
117-
118-
2. **Supported Secret Backends**:
119-
- AWS Secrets Manager
120-
- Google Secret Manager
121-
- Azure Key Vault
122-
- git-secret (for local development)
123-
124-
3. **Authentication Patterns**:
125-
- Environment variables for cloud credentials
126-
- Credential files for service accounts
127-
- Direct credentials (development only)
128-
- Managed identities for cloud services
108+
Alternative authentication configurations:
109+
- AWS:
110+
```json
111+
{
112+
"type": "file",
113+
"path": "~/.aws/credentials"
114+
}
115+
```
116+
or
117+
```json
118+
{
119+
"type": "direct",
120+
"access_key_id": "AKIA...",
121+
"secret_access_key": "xyz..."
122+
}
123+
```
124+
125+
- Google Cloud:
126+
```json
127+
{
128+
"type": "env",
129+
"credential_env": "GOOGLE_APPLICATION_CREDENTIALS"
130+
}
131+
```
132+
or
133+
```json
134+
{
135+
"type": "direct",
136+
"credentials_json": "{...}"
137+
}
138+
```
139+
140+
- Azure:
141+
```json
142+
{
143+
"type": "service_principal",
144+
"tenant_id": "${AZURE_TENANT_ID}",
145+
"client_id": "${AZURE_CLIENT_ID}",
146+
"client_secret": "${AZURE_CLIENT_SECRET}"
147+
}
148+
```
149+
150+
- Git Secret:
151+
```json
152+
{
153+
"type": "env",
154+
"passphrase_env": "GIT_SECRET_PASSPHRASE"
155+
}
156+
```
157+
or
158+
```json
159+
{
160+
"type": "direct",
161+
"passphrase": "your-passphrase"
162+
}
163+
```
129164

130165
## Consequences
131166
- The project will be built as a Helm plugin with Python as the core language.

‎docs/Design/architecture-overview.md‎

Lines changed: 64 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,52 +7,63 @@ Helm Values Manager simplifies **configuration and secret management** across mu
77

88
The Helm plugin consists of:
99
- **CLI Command Interface (Python Typer-based)**: Handles command execution.
10-
- **Validation Engine**: Ensures required keys have values for each deployment.
11-
- **Secret Manager Integration**: Supports AWS Secrets Manager, Azure Key Vault, HashiCorp Vault, and Git-Secrets.
10+
- **Validation Engine**: Ensures required values have values for each deployment.
11+
- **Value Storage Backend**: Supports AWS Secrets Manager, Azure Key Vault, HashiCorp Vault, and Git-Secrets.
1212
- **values.yaml Generator**: Produces the final Helm-compatible values file.
1313
- **Helm Plugin System**: Integrates seamlessly with Helm commands.
14+
- **JSON Schema Validation**: Ensures configuration files follow the correct structure.
1415

15-
## Configuration YAML Structure
16-
17-
```yaml
18-
release: my-release
19-
20-
deployments:
21-
dev:
22-
secrets_backend: aws_secrets_manager
23-
prod:
24-
secrets_backend: azure_key_vault
25-
26-
config:
27-
- key: DATABASE_URL
28-
path: global.database.url
29-
required: true
30-
sensitive: true
31-
values:
32-
dev: "mydb://dev-connection"
33-
prod: "mydb://prod-connection"
34-
- key: LOG_LEVEL
35-
path: global.logging.level
36-
required: false
37-
sensitive: false
38-
values:
39-
dev: "debug"
40-
prod: "warn"
16+
## Configuration Structure
17+
18+
```json
19+
{
20+
"release": "my-release",
21+
"deployments": {
22+
"dev": {
23+
"secrets_backend": "aws_secrets_manager"
24+
},
25+
"prod": {
26+
"secrets_backend": "azure_key_vault"
27+
}
28+
},
29+
"config": [
30+
{
31+
"path": "global.database.url",
32+
"required": true,
33+
"sensitive": true,
34+
"values": {
35+
"dev": "mydb://dev-connection",
36+
"prod": "mydb://prod-connection"
37+
}
38+
},
39+
{
40+
"path": "global.logging.level",
41+
"required": false,
42+
"sensitive": false,
43+
"values": {
44+
"dev": "DEBUG",
45+
"prod": "WARN"
46+
}
47+
}
48+
]
49+
}
4150
```
4251

43-
## Secret Manager Extensibility
52+
## Value Backend Extensibility
4453
Implemented using **Abstract Base Class (ABC)**:
4554

4655
```python
4756
from abc import ABC, abstractmethod
4857

49-
class SecretManager(ABC):
58+
class ValueBackend(ABC):
5059
@abstractmethod
51-
def get_secret(self, secret_name: str) -> str:
60+
def get_value(self, path: str, environment: str) -> str:
61+
"""Get a value from the secrets backend."""
5262
pass
5363

5464
@abstractmethod
55-
def store_secret(self, secret_name: str, secret_value: str):
65+
def set_value(self, path: str, environment: str, value: str) -> None:
66+
"""Set a value in the secrets backend."""
5667
pass
5768
```
5869

@@ -61,16 +72,33 @@ class SecretManager(ABC):
6172
### Example Commands
6273
```sh
6374
helm values-manager init my-release
64-
helm values-manager add-key DATABASE_URL --required --sensitive --path=global.database.url
65-
helm values-manager add-secret DATABASE_URL=mydb://connection --deployment=dev
75+
helm values-manager add-value --path=global.database.url --required --sensitive
76+
helm values-manager set-value global.database.url=mydb://connection --deployment=dev
6677
helm values-manager validate
6778
helm values-manager generate --deployment=dev
6879
```
6980

7081
## Testing Strategy
7182
- **Unit Tests:** CLI commands, validation logic, storage handling.
72-
- **Integration Tests:** Secret manager interactions.
83+
- **Integration Tests:** Backend interactions and value management.
7384
- **E2E Tests:** Full workflow validation.
7485

86+
## Implementation Details
87+
88+
### JSON Schema Validation
89+
- Uses Python's `jsonschema` library for configuration validation
90+
- Schema version control for backward compatibility
91+
- Strict validation of required fields and value types
92+
93+
### Value Storage
94+
- Values stored in JSON format for consistency
95+
- Supports both plain text and encrypted storage
96+
- Environment-specific value management
97+
98+
### Security Considerations
99+
- Sensitive values stored in secure backends
100+
- Support for various authentication methods
101+
- No sensitive data in version control
102+
75103
## Conclusion
76-
Helm Values Manager is a **secure, scalable, and extensible solution** for Helm-based deployments.
104+
Helm Values Manager is a **secure, scalable, and extensible solution** for Helm-based deployments, using JSON for consistent configuration management and robust schema validation.

0 commit comments

Comments
 (0)