Skip to content

Commit ff5393d

Browse files
pascalbreuningerpascalbreuninger
committed
feat(pro): move daemon config into secrets and mount under /var/run/secrets/devpod/daemon_config instead of using environment variable
1 parent 7cafcdf commit ff5393d

File tree

8 files changed

+165
-25
lines changed

8 files changed

+165
-25
lines changed

‎cmd/agent/container/daemon.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515

1616
"github.com/loft-sh/devpod/pkg/agent"
1717
agentd "github.com/loft-sh/devpod/pkg/daemon/agent"
18-
"github.com/loft-sh/devpod/pkg/devcontainer"
18+
"github.com/loft-sh/devpod/pkg/devcontainer/config"
1919
"github.com/loft-sh/devpod/pkg/platform/client"
2020
"github.com/loft-sh/devpod/pkg/ts"
2121
"github.com/loft-sh/log"
@@ -25,7 +25,8 @@ import (
2525
)
2626

2727
const (
28-
RootDir = "/var/devpod"
28+
RootDir = "/var/devpod"
29+
DaemonConfigPath = "/var/run/secrets/devpod/daemon_config"
2930
)
3031

3132
type DaemonCmd struct {
@@ -133,7 +134,21 @@ func (cmd *DaemonCmd) Run(c *cobra.Command, args []string) error {
133134
// loadConfig loads the daemon configuration from base64-encoded JSON.
134135
// If a CLI-provided timeout exists, it will override the timeout in the config.
135136
func (cmd *DaemonCmd) loadConfig() error {
136-
if encodedCfg := os.Getenv(devcontainer.WorkspaceDaemonConfigExtraEnvVar); strings.TrimSpace(encodedCfg) != "" {
137+
// check local file
138+
encodedCfg := ""
139+
configBytes, err := os.ReadFile(DaemonConfigPath)
140+
if err != nil {
141+
if errors.Is(err, os.ErrNotExist) {
142+
// check environment variable
143+
encodedCfg = os.Getenv(config.WorkspaceDaemonConfigExtraEnvVar)
144+
} else {
145+
return fmt.Errorf("get daemon config file %s: %w", DaemonConfigPath, err)
146+
}
147+
} else {
148+
encodedCfg = string(configBytes)
149+
}
150+
151+
if strings.TrimSpace(encodedCfg) != "" {
137152
decoded, err := base64.StdEncoding.DecodeString(encodedCfg)
138153
if err != nil {
139154
return fmt.Errorf("error decoding daemon config: %w", err)
@@ -147,6 +162,7 @@ func (cmd *DaemonCmd) loadConfig() error {
147162
}
148163
cmd.Config = &cfg
149164
}
165+
150166
return nil
151167
}
152168

‎pkg/daemon/agent/daemon.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ type DaemonConfig struct {
3030
Timeout string `json:"timeout"`
3131
}
3232

33-
func BuildDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig) (*DaemonConfig, error) {
33+
func BuildWorkspaceDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig) (*DaemonConfig, error) {
3434
var workdir string
3535
if workspaceConfig.Source.GitSubPath != "" {
3636
substitutionContext.ContainerWorkspaceFolder = filepath.Join(substitutionContext.ContainerWorkspaceFolder, workspaceConfig.Source.GitSubPath)
@@ -49,6 +49,9 @@ func BuildDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *
4949
user = "root"
5050
}
5151

52+
// build info isn't required in the workspace and can be omitted
53+
platformOptions.Build = nil
54+
5255
daemonConfig := &DaemonConfig{
5356
Platform: platformOptions,
5457
Ssh: SshConfig{
@@ -60,8 +63,8 @@ func BuildDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *
6063
return daemonConfig, nil
6164
}
6265

63-
func GetEncodedDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig) (string, error) {
64-
daemonConfig, err := BuildDaemonConfig(platformOptions, workspaceConfig, substitutionContext, mergedConfig)
66+
func GetEncodedWorkspaceDaemonConfig(platformOptions devpod.PlatformOptions, workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig) (string, error) {
67+
daemonConfig, err := BuildWorkspaceDaemonConfig(platformOptions, workspaceConfig, substitutionContext, mergedConfig)
6568
if err != nil {
6669
return "", err
6770
}

‎pkg/devcontainer/config/build.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ const (
1010

1111
DevPodContextFeatureFolder = ".devpod-internal"
1212
DevPodDockerlessBuildInfoFolder = "/workspaces/.dockerless"
13+
14+
WorkspaceDaemonConfigExtraEnvVar = "DEVPOD_WORKSPACE_DAEMON_CONFIG"
1315
)
1416

1517
func GetDockerLabelForID(id string) []string {

‎pkg/devcontainer/single.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,10 @@ import (
1818
var dockerlessImage = "ghcr.io/loft-sh/dockerless:0.2.0"
1919

2020
const (
21-
DevPodExtraEnvVar = "DEVPOD"
22-
RemoteContainersExtraEnvVar = "REMOTE_CONTAINERS"
23-
WorkspaceIDExtraEnvVar = "DEVPOD_WORKSPACE_ID"
24-
WorkspaceUIDExtraEnvVar = "DEVPOD_WORKSPACE_UID"
25-
WorkspaceDaemonConfigExtraEnvVar = "DEVPOD_WORKSPACE_DAEMON_CONFIG"
21+
DevPodExtraEnvVar = "DEVPOD"
22+
RemoteContainersExtraEnvVar = "REMOTE_CONTAINERS"
23+
WorkspaceIDExtraEnvVar = "DEVPOD_WORKSPACE_ID"
24+
WorkspaceUIDExtraEnvVar = "DEVPOD_WORKSPACE_UID"
2625

2726
DefaultEntrypoint = `
2827
while ! command -v /usr/local/bin/devpod >/dev/null 2>&1; do
@@ -132,11 +131,11 @@ func (r *runner) runSingleContainer(
132131
if options.CLIOptions.Platform.AccessKey != "" {
133132
r.Log.Debugf("Platform config detected, injecting DevPod daemon entrypoint.")
134133

135-
data, err := agent.GetEncodedDaemonConfig(options.Platform, r.WorkspaceConfig.Workspace, substitutionContext, mergedConfig)
134+
data, err := agent.GetEncodedWorkspaceDaemonConfig(options.Platform, r.WorkspaceConfig.Workspace, substitutionContext, mergedConfig)
136135
if err != nil {
137136
r.Log.Errorf("Failed to marshal daemon config: %v", err)
138137
} else {
139-
mergedConfig.ContainerEnv[WorkspaceDaemonConfigExtraEnvVar] = data
138+
mergedConfig.ContainerEnv[config.WorkspaceDaemonConfigExtraEnvVar] = data
140139
}
141140
}
142141

‎pkg/driver/kubernetes/daemonsecret.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package kubernetes
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
k8sv1 "k8s.io/api/core/v1"
8+
kerrors "k8s.io/apimachinery/pkg/api/errors"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
)
11+
12+
const DaemonConfigKey = "daemon_config"
13+
14+
func (k *KubernetesDriver) EnsureDaemonConfigSecret(
15+
ctx context.Context,
16+
secretName string,
17+
data string,
18+
) error {
19+
k.Log.Debugf("Ensure daemon config secret")
20+
21+
if k.secretExists(ctx, secretName) {
22+
if !k.shouldRecreateDaemonConfigSecret(ctx, data, secretName) {
23+
k.Log.Debugf("Daemon config secret '%s' already exists and is up to date", secretName)
24+
return nil
25+
}
26+
27+
k.Log.Debugf("Daemon config secret '%s' already exists, but is outdated. Recreating...", secretName)
28+
err := k.DeleteSecret(ctx, secretName)
29+
if err != nil {
30+
return err
31+
}
32+
}
33+
34+
_, err := k.client.Client().CoreV1().Secrets(k.namespace).Create(ctx, &k8sv1.Secret{
35+
ObjectMeta: metav1.ObjectMeta{
36+
Name: secretName,
37+
},
38+
Type: k8sv1.SecretTypeOpaque,
39+
Data: map[string][]byte{DaemonConfigKey: []byte(data)},
40+
}, metav1.CreateOptions{})
41+
if err != nil {
42+
return fmt.Errorf("create daemon config secret: %w", err)
43+
}
44+
45+
k.Log.Infof("Daemon config secret '%s' created", secretName)
46+
return nil
47+
}
48+
func (k *KubernetesDriver) shouldRecreateDaemonConfigSecret(ctx context.Context, newData string, secretName string) bool {
49+
secret, err := k.client.Client().CoreV1().Secrets(k.namespace).Get(ctx, secretName, metav1.GetOptions{})
50+
if err != nil {
51+
return true
52+
}
53+
54+
daemonConfigBytes, ok := secret.Data[DaemonConfigKey]
55+
if !ok {
56+
return true
57+
}
58+
59+
return string(daemonConfigBytes) != newData
60+
}
61+
62+
func (k *KubernetesDriver) DeleteDaemonConfigSecret(
63+
ctx context.Context,
64+
secretName string) error {
65+
if !k.secretExists(ctx, secretName) {
66+
return nil
67+
}
68+
69+
err := k.client.Client().CoreV1().Secrets(k.namespace).Delete(ctx, secretName, metav1.DeleteOptions{})
70+
if err != nil && !kerrors.IsNotFound(err) {
71+
return fmt.Errorf("delete daemon config secret: %w", err)
72+
}
73+
74+
return nil
75+
}

‎pkg/driver/kubernetes/driver.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,10 +127,19 @@ func (k *KubernetesDriver) DeleteDevContainer(ctx context.Context, workspaceId s
127127
}
128128
}
129129

130+
// delete daemon config secret
131+
if k.secretExists(ctx, getDaemonSecretName(workspaceId)) {
132+
k.Log.Infof("Delete daemon config secret '%s'...", workspaceId)
133+
err := k.DeleteSecret(ctx, getDaemonSecretName(workspaceId))
134+
if err != nil {
135+
return err
136+
}
137+
}
138+
130139
// delete pull secret
131140
if k.options.KubernetesPullSecretsEnabled != "" {
132141
k.Log.Infof("Delete pull secret '%s'...", workspaceId)
133-
err := k.DeletePullSecret(ctx, getPullSecretsName(workspaceId))
142+
err := k.DeleteSecret(ctx, getPullSecretsName(workspaceId))
134143
if err != nil {
135144
return err
136145
}

‎pkg/driver/kubernetes/pullsecrets.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func (k *KubernetesDriver) EnsurePullSecret(
3636
}
3737

3838
k.Log.Debugf("Pull secret '%s' already exists, but is outdated. Recreating...", pullSecretName)
39-
err := k.DeletePullSecret(ctx, pullSecretName)
39+
err := k.DeleteSecret(ctx, pullSecretName)
4040
if err != nil {
4141
return false, err
4242
}
@@ -64,16 +64,16 @@ func (k *KubernetesDriver) ReadSecretContents(
6464
return DecodeAuthTokenFromPullSecret(secret, host)
6565
}
6666

67-
func (k *KubernetesDriver) DeletePullSecret(
67+
func (k *KubernetesDriver) DeleteSecret(
6868
ctx context.Context,
69-
pullSecretName string) error {
70-
if !k.secretExists(ctx, pullSecretName) {
69+
secretName string) error {
70+
if !k.secretExists(ctx, secretName) {
7171
return nil
7272
}
7373

74-
err := k.client.Client().CoreV1().Secrets(k.namespace).Delete(ctx, pullSecretName, metav1.DeleteOptions{})
74+
err := k.client.Client().CoreV1().Secrets(k.namespace).Delete(ctx, secretName, metav1.DeleteOptions{})
7575
if err != nil && !kerrors.IsNotFound(err) {
76-
return perrors.Wrap(err, "delete pull secret")
76+
return perrors.Wrap(err, "delete secret")
7777
}
7878

7979
return nil
@@ -89,9 +89,9 @@ func (k *KubernetesDriver) shouldRecreateSecret(ctx context.Context, dockerCrede
8989

9090
func (k *KubernetesDriver) secretExists(
9191
ctx context.Context,
92-
pullSecretName string,
92+
secretName string,
9393
) bool {
94-
_, err := k.client.Client().CoreV1().Secrets(k.namespace).Get(ctx, pullSecretName, metav1.GetOptions{})
94+
_, err := k.client.Client().CoreV1().Secrets(k.namespace).Get(ctx, secretName, metav1.GetOptions{})
9595
return err == nil
9696
}
9797

‎pkg/driver/kubernetes/run.go‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,13 @@ func (k *KubernetesDriver) runContainer(
155155

156156
// env vars
157157
envVars := []corev1.EnvVar{}
158+
daemonConfig := ""
158159
for k, v := range options.Env {
160+
// filter out daemon config, that's going to be mounted through a secret
161+
if k == config.WorkspaceDaemonConfigExtraEnvVar {
162+
daemonConfig = v
163+
continue
164+
}
159165
envVars = append(envVars, corev1.EnvVar{
160166
Name: k,
161167
Value: v,
@@ -196,6 +202,16 @@ func (k *KubernetesDriver) runContainer(
196202
resources = parseResources(k.options.Resources, k.Log)
197203
}
198204

205+
// ensure daemon config secret
206+
daemonConfigSecretName := ""
207+
if daemonConfig != "" {
208+
daemonConfigSecretName = getDaemonSecretName(id)
209+
err = k.EnsureDaemonConfigSecret(ctx, daemonConfigSecretName, daemonConfig)
210+
if err != nil {
211+
return err
212+
}
213+
}
214+
199215
// ensure pull secrets
200216
pullSecretsCreated := false
201217
if k.options.KubernetesPullSecretsEnabled == "true" {
@@ -212,8 +228,8 @@ func (k *KubernetesDriver) runContainer(
212228
pod.Spec.ServiceAccountName = serviceAccount
213229
pod.Spec.NodeSelector = nodeSelector
214230
pod.Spec.InitContainers = initContainers
215-
pod.Spec.Containers = getContainers(pod, options.Image, options.Entrypoint, options.Cmd, envVars, volumeMounts, capabilities, resources, options.Privileged, k.options.StrictSecurity)
216-
pod.Spec.Volumes = getVolumes(pod, id)
231+
pod.Spec.Containers = getContainers(pod, options.Image, options.Entrypoint, options.Cmd, envVars, volumeMounts, capabilities, resources, options.Privileged, k.options.StrictSecurity, daemonConfigSecretName)
232+
pod.Spec.Volumes = getVolumes(pod, id, daemonConfigSecretName)
217233
// avoids a problem where attaching volumes with large repositories would cause an extremely long pod startup time
218234
// because changing the ownership of all files takes longer than the kubelet expects it to
219235
if pod.Spec.SecurityContext == nil {
@@ -311,7 +327,12 @@ func getContainers(
311327
resources corev1.ResourceRequirements,
312328
privileged *bool,
313329
strictSecurity string,
330+
daemonConfigSecretName string,
314331
) []corev1.Container {
332+
volumeMounts = append(volumeMounts, corev1.VolumeMount{
333+
Name: "devpod-daemon-config",
334+
MountPath: "/var/run/secrets/devpod",
335+
})
315336
devPodContainer := corev1.Container{
316337
Name: DevContainerName,
317338
Image: imageName,
@@ -362,7 +383,7 @@ func getContainers(
362383
return retContainers
363384
}
364385

365-
func getVolumes(pod *corev1.Pod, id string) []corev1.Volume {
386+
func getVolumes(pod *corev1.Pod, id string, daemonConfigSecretName string) []corev1.Volume {
366387
volumes := []corev1.Volume{
367388
{
368389
Name: "devpod",
@@ -374,6 +395,17 @@ func getVolumes(pod *corev1.Pod, id string) []corev1.Volume {
374395
},
375396
}
376397

398+
if daemonConfigSecretName != "" {
399+
volumes = append(volumes, corev1.Volume{
400+
Name: "devpod-daemon-config",
401+
VolumeSource: corev1.VolumeSource{
402+
Secret: &corev1.SecretVolumeSource{
403+
SecretName: daemonConfigSecretName,
404+
},
405+
},
406+
})
407+
}
408+
377409
if pod.Spec.Volumes != nil {
378410
volumes = append(volumes, pod.Spec.Volumes...)
379411
}
@@ -467,6 +499,10 @@ func getPullSecretsName(workspaceID string) string {
467499
return fmt.Sprintf("devpod-pull-secret-%s", workspaceID)
468500
}
469501

502+
func getDaemonSecretName(workspaceID string) string {
503+
return fmt.Sprintf("devpod-daemon-secret-%s", workspaceID)
504+
}
505+
470506
func optionsEqual(a, b *provider2.ProviderKubernetesDriverConfig) bool {
471507
// copy a and b and the compare them without the context, config, namespace and podTimeout
472508
aCopy := *a

0 commit comments

Comments
 (0)