Skip to content

Commit 55b5f23

Browse files
gloursndeloof
authored andcommitted
use Defang secret-detector to identify potential secret leaks before publishing OCI artifacts
Signed-off-by: Guillaume Lours <[email protected]>
1 parent c3a0c35 commit 55b5f23

File tree

9 files changed

+189
-12
lines changed

9 files changed

+189
-12
lines changed

go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.23.6
44

55
require (
66
github.com/AlecAivazis/survey/v2 v2.3.7
7+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f
78
github.com/Microsoft/go-winio v0.6.2
89
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
910
github.com/buger/goterm v1.0.4
@@ -107,6 +108,7 @@ require (
107108
github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
108109
github.com/gofrs/flock v0.12.1 // indirect
109110
github.com/gogo/protobuf v1.3.2 // indirect
111+
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
110112
github.com/golang/protobuf v1.5.4 // indirect
111113
github.com/google/gnostic-models v0.6.8 // indirect
112114
github.com/google/gofuzz v1.2.0 // indirect
@@ -120,6 +122,7 @@ require (
120122
github.com/imdario/mergo v0.3.16 // indirect
121123
github.com/in-toto/in-toto-golang v0.5.0 // indirect
122124
github.com/inconshreveable/mousetrap v1.1.0 // indirect
125+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
123126
github.com/josharian/intern v1.0.0 // indirect
124127
github.com/json-iterator/go v1.1.12 // indirect
125128
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
@@ -190,6 +193,7 @@ require (
190193
google.golang.org/protobuf v1.36.4 // indirect
191194
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
192195
gopkg.in/inf.v0 v0.9.1 // indirect
196+
gopkg.in/ini.v1 v1.66.2 // indirect
193197
gopkg.in/yaml.v2 v2.4.0 // indirect
194198
k8s.io/api v0.31.2 // indirect
195199
k8s.io/apimachinery v0.31.2 // indirect

go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
1010
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
1111
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
1212
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
13+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f h1:RTbUqLhPxejgK92ifVdMTIW9H23QLlscy8QXPDTfaL4=
14+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f/go.mod h1:2UjtD/G/Sy2FxoHpxKnzHTXMpRURecwYal8HgbxcvkY=
1315
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
1416
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
1517
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
@@ -195,6 +197,8 @@ github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
195197
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
196198
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
197199
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
200+
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
201+
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
198202
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
199203
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
200204
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -249,6 +253,8 @@ github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1Gd
249253
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
250254
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
251255
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
256+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
257+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
252258
github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE=
253259
github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
254260
github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc=
@@ -642,6 +648,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
642648
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
643649
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
644650
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
651+
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
652+
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
645653
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM=
646654
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw=
647655
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

pkg/compose/publish.go

Lines changed: 133 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,17 @@
1717
package compose
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"crypto/sha256"
2223
"errors"
2324
"fmt"
25+
"io"
2426
"os"
2527

28+
"github.com/DefangLabs/secret-detector/pkg/scanner"
29+
"github.com/DefangLabs/secret-detector/pkg/secrets"
30+
2631
"github.com/compose-spec/compose-go/v2/loader"
2732
"github.com/compose-spec/compose-go/v2/types"
2833
"github.com/distribution/reference"
@@ -226,15 +231,37 @@ func (s *composeService) generateImageDigestsOverride(ctx context.Context, proje
226231
return override.MarshalYAML()
227232
}
228233

234+
//nolint:gocyclo
229235
func (s *composeService) preChecks(project *types.Project, options api.PublishOptions) (bool, error) {
230-
if ok, err := s.checkOnlyBuildSection(project); !ok {
236+
if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil {
237+
return false, err
238+
}
239+
if ok, err := s.checkForBindMount(project); !ok || err != nil {
231240
return false, err
232241
}
242+
if options.AssumeYes {
243+
return true, nil
244+
}
245+
detectedSecrets, err := s.checkForSensitiveData(project)
246+
if err != nil {
247+
return false, err
248+
}
249+
if len(detectedSecrets) > 0 {
250+
fmt.Println("you are about to publish sensitive data within your OCI artifact.\n" +
251+
"please double check that you are not leaking sensitive data")
252+
for _, val := range detectedSecrets {
253+
_, _ = fmt.Fprintln(s.dockerCli.Out(), val.Type)
254+
_, _ = fmt.Fprintf(s.dockerCli.Out(), "%q: %s\n", val.Key, val.Value)
255+
}
256+
if ok, err := acceptPublishSensitiveData(s.dockerCli); err != nil || !ok {
257+
return false, err
258+
}
259+
}
233260
envVariables, err := s.checkEnvironmentVariables(project, options)
234261
if err != nil {
235262
return false, err
236263
}
237-
if !options.AssumeYes && len(envVariables) > 0 {
264+
if len(envVariables) > 0 {
238265
fmt.Println("you are about to publish environment variables within your OCI artifact.\n" +
239266
"please double check that you are not leaking sensitive data")
240267
for key, val := range envVariables {
@@ -243,17 +270,10 @@ func (s *composeService) preChecks(project *types.Project, options api.PublishOp
243270
_, _ = fmt.Fprintf(s.dockerCli.Out(), "%s=%v\n", k, *v)
244271
}
245272
}
246-
return acceptPublishEnvVariables(s.dockerCli)
247-
}
248-
249-
for name, config := range project.Services {
250-
for _, volume := range config.Volumes {
251-
if volume.Type == types.VolumeTypeBind {
252-
return false, fmt.Errorf("cannot publish compose file: service %q relies on bind-mount. You should use volumes", name)
253-
}
273+
if ok, err := acceptPublishEnvVariables(s.dockerCli); err != nil || !ok {
274+
return false, err
254275
}
255276
}
256-
257277
return true, nil
258278
}
259279

@@ -299,6 +319,12 @@ func acceptPublishEnvVariables(cli command.Cli) (bool, error) {
299319
return confirm, err
300320
}
301321

322+
func acceptPublishSensitiveData(cli command.Cli) (bool, error) {
323+
msg := "Are you ok to publish these sensitive data? [y/N]: "
324+
confirm, err := prompt.NewPrompt(cli.In(), cli.Out()).Confirm(msg, false)
325+
return confirm, err
326+
}
327+
302328
func envFileLayers(project *types.Project) []ocipush.Pushable {
303329
var layers []ocipush.Pushable
304330
for _, service := range project.Services {
@@ -334,3 +360,99 @@ func (s *composeService) checkOnlyBuildSection(project *types.Project) (bool, er
334360
}
335361
return true, nil
336362
}
363+
364+
func (s *composeService) checkForBindMount(project *types.Project) (bool, error) {
365+
for name, config := range project.Services {
366+
for _, volume := range config.Volumes {
367+
if volume.Type == types.VolumeTypeBind {
368+
return false, fmt.Errorf("cannot publish compose file: service %q relies on bind-mount. You should use volumes", name)
369+
}
370+
}
371+
}
372+
return true, nil
373+
}
374+
375+
func (s *composeService) checkForSensitiveData(project *types.Project) ([]secrets.DetectedSecret, error) {
376+
var allFindings []secrets.DetectedSecret
377+
scan := scanner.NewDefaultScanner()
378+
// Check all compose files
379+
for _, file := range project.ComposeFiles {
380+
in, err := composeFileAsByteReader(file, project)
381+
if err != nil {
382+
return nil, err
383+
}
384+
385+
findings, err := scan.ScanReader(in)
386+
if err != nil {
387+
return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err)
388+
}
389+
allFindings = append(allFindings, findings...)
390+
}
391+
for _, service := range project.Services {
392+
// Check env files
393+
for _, envFile := range service.EnvFiles {
394+
findings, err := scan.ScanFile(envFile.Path)
395+
if err != nil {
396+
return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err)
397+
}
398+
allFindings = append(allFindings, findings...)
399+
}
400+
}
401+
402+
// Check configs defined by files
403+
for _, config := range project.Configs {
404+
if config.File != "" {
405+
findings, err := scan.ScanFile(config.File)
406+
if err != nil {
407+
return nil, fmt.Errorf("failed to scan config file %s: %w", config.File, err)
408+
}
409+
allFindings = append(allFindings, findings...)
410+
}
411+
}
412+
413+
// Check secrets defined by files
414+
for _, secret := range project.Secrets {
415+
if secret.File != "" {
416+
findings, err := scan.ScanFile(secret.File)
417+
if err != nil {
418+
return nil, fmt.Errorf("failed to scan secret file %s: %w", secret.File, err)
419+
}
420+
allFindings = append(allFindings, findings...)
421+
}
422+
}
423+
424+
return allFindings, nil
425+
}
426+
427+
func composeFileAsByteReader(filePath string, project *types.Project) (io.Reader, error) {
428+
composeFile, err := os.ReadFile(filePath)
429+
if err != nil {
430+
return nil, fmt.Errorf("failed to open compose file %s: %w", filePath, err)
431+
}
432+
base, err := loader.LoadWithContext(context.TODO(), types.ConfigDetails{
433+
WorkingDir: project.WorkingDir,
434+
Environment: project.Environment,
435+
ConfigFiles: []types.ConfigFile{
436+
{
437+
Filename: filePath,
438+
Content: composeFile,
439+
},
440+
},
441+
}, func(options *loader.Options) {
442+
options.SkipValidation = true
443+
options.SkipExtends = true
444+
options.SkipConsistencyCheck = true
445+
options.ResolvePaths = true
446+
options.SkipInterpolation = true
447+
options.SkipResolveEnvironment = true
448+
})
449+
if err != nil {
450+
return nil, err
451+
}
452+
453+
in, err := base.MarshalYAML()
454+
if err != nil {
455+
return nil, err
456+
}
457+
return bytes.NewBuffer(in), nil
458+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
services:
2+
serviceA:
3+
image: "alpine:3.12"
4+
environment:
5+
- AWS_ACCESS_KEY_ID=A3TX1234567890ABCDEF
6+
- AWS_SECRET_ACCESS_KEY=aws"12345+67890/abcdefghijklm+NOPQRSTUVWXYZ+"
7+
configs:
8+
- myconfig
9+
serviceB:
10+
image: "alpine:3.12"
11+
env_file:
12+
- publish-sensitive.env
13+
secrets:
14+
- mysecret
15+
configs:
16+
myconfig:
17+
file: config.txt
18+
secrets:
19+
mysecret:
20+
file: secret.txt

pkg/e2e/fixtures/publish/config.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuvwxyz

pkg/e2e/fixtures/publish/publish.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
FOO=bar
2-
QUIX=
2+
QUIX=

pkg/e2e/fixtures/publish/secret.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
-----BEGIN DSA PRIVATE KEY-----
2+
wxyz+ABC=
3+
-----END DSA PRIVATE KEY-----

pkg/e2e/publish_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,22 @@ FOO=bar`), res.Combined())
134134
"-p", projectName, "alpha", "publish", "test/test", "--dry-run")
135135
res.Assert(t, icmd.Expected{ExitCode: 1, Err: "cannot publish compose file with local includes"})
136136
})
137+
138+
t.Run("detect sensitive data", func(t *testing.T) {
139+
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/publish/compose-sensitive.yml",
140+
"-p", projectName, "alpha", "publish", "test/test", "--with-env", "--dry-run")
141+
cmd.Stdin = strings.NewReader("n\n")
142+
res := icmd.RunCmd(cmd)
143+
res.Assert(t, icmd.Expected{ExitCode: 0})
144+
145+
output := res.Combined()
146+
assert.Assert(t, strings.Contains(output, "you are about to publish sensitive data within your OCI artifact.\n"), output)
147+
assert.Assert(t, strings.Contains(output, "please double check that you are not leaking sensitive data"), output)
148+
assert.Assert(t, strings.Contains(output, "AWS Client ID\n\"services.serviceA.environment.AWS_ACCESS_KEY_ID\": A3TX1234567890ABCDEF"), output)
149+
assert.Assert(t, strings.Contains(output, "AWS Secret Key\n\"services.serviceA.environment.AWS_SECRET_ACCESS_KEY\": aws\"12345+67890/abcdefghijklm+NOPQRSTUVWXYZ+\""), output)
150+
assert.Assert(t, strings.Contains(output, "Github authentication\n\"GITHUB_TOKEN\": ghp_1234567890abcdefghijklmnopqrstuvwxyz"), output)
151+
assert.Assert(t, strings.Contains(output, "JSON Web Token\n\"\": eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."+
152+
"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw"), output)
153+
assert.Assert(t, strings.Contains(output, "Private Key\n\"\": -----BEGIN DSA PRIVATE KEY-----\nwxyz+ABC=\n-----END DSA PRIVATE KEY-----"), output)
154+
})
137155
}

0 commit comments

Comments
 (0)