Skip to content

Commit b5279a1

Browse files
commit111nullfunclionello
authored
Add detection for sensitive config values in Compose files (#1038)
* draft config detector * add transformers (may be broken) * fixed scanner to use detectors * move into its own function * added tests for config detector * revise fn to return a list of types * make the keyword and high entropy detectors work * add go mod * comments for config detector function * run config detect on env vars in compose file in CLI * add compose tests * edit dx message * edit description of function * add aws and github detectors * improve dx message * fix nil check for value * update vendorhash * Delete .vscode/launch.json * add comments for threshold * remove package main comment * fix comma * added back the json transformer for url_password detector * Apply suggestions from code review Co-authored-by: Lio李歐 <[email protected]> * use fmt to wrap errors instead of errors.New() * fix compose warnings test data * change name to sensitive (to not trigger the keyword detector) * make test become subtests * use logical expression instead of json to create config --------- Co-authored-by: Eric Liu <[email protected]> Co-authored-by: Lio李歐 <[email protected]>
1 parent f1f837e commit b5279a1

File tree

10 files changed

+178
-1
lines changed

10 files changed

+178
-1
lines changed

‎pkgs/defang/cli.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ buildGoModule {
77
pname = "defang-cli";
88
version = "git";
99
src = ../../src;
10-
vendorHash = "sha256-of8K2h3gYoOdxHmBDXKiRfm35YeVE5R4WOy0pcSim4c=";
10+
vendorHash = "sha256-tLjXqnA8zkLtJrchO/FpIjimtC5YzW/0f7mL6RJLnSQ=";
1111

1212
subPackages = [ "cmd/cli" ];
1313

‎src/go.mod‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
cloud.google.com/go/secretmanager v1.14.2
1212
cloud.google.com/go/storage v1.46.0
1313
github.com/AlecAivazis/survey/v2 v2.3.7
14+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f
1415
github.com/aws/aws-sdk-go-v2 v1.32.6
1516
github.com/aws/aws-sdk-go-v2/config v1.26.6
1617
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.42.6
@@ -74,11 +75,15 @@ require (
7475
github.com/envoyproxy/go-control-plane v0.13.0 // indirect
7576
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
7677
github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
78+
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
7779
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
7880
github.com/google/go-querystring v1.1.0 // indirect
7981
github.com/google/s2a-go v0.1.8 // indirect
8082
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
83+
github.com/hashicorp/errwrap v1.0.0 // indirect
8184
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
85+
github.com/hashicorp/go-multierror v1.1.1 // indirect
86+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
8287
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
8388
github.com/mattn/go-runewidth v0.0.14 // indirect
8489
github.com/morikuni/aec v1.0.0 // indirect
@@ -95,6 +100,7 @@ require (
95100
google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 // indirect
96101
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
97102
google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
103+
gopkg.in/ini.v1 v1.66.2 // indirect
98104
gopkg.in/yaml.v2 v2.4.0 // indirect
99105
)
100106

‎src/go.sum‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1r
3434
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
3535
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
3636
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
37+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f h1:RTbUqLhPxejgK92ifVdMTIW9H23QLlscy8QXPDTfaL4=
38+
github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f/go.mod h1:2UjtD/G/Sy2FxoHpxKnzHTXMpRURecwYal8HgbxcvkY=
3739
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg=
3840
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=
3941
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s=
@@ -159,6 +161,8 @@ github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAp
159161
github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
160162
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
161163
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
164+
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
165+
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
162166
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
163167
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
164168
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
@@ -203,10 +207,14 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
203207
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
204208
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
205209
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
210+
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
211+
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
206212
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
207213
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
208214
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
209215
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
216+
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
217+
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
210218
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
211219
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
212220
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
@@ -215,6 +223,8 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u
215223
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
216224
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
217225
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
226+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
227+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
218228
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
219229
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
220230
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -450,6 +460,8 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt
450460
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
451461
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
452462
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
463+
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
464+
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
453465
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
454466
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
455467
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

‎src/pkg/cli/compose/config_detector.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package compose
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/DefangLabs/secret-detector/pkg/scanner"
7+
)
8+
9+
// assume that the input is a key-value pair string
10+
func detectConfig(input string) (detectorTypes []string, err error) {
11+
// Detectors check for certain formats in a string to determine if it contains a secret.
12+
// Some detectors allow additional configuration options, such as:
13+
// keyword: key contains a keyword (e.g. KEY, PASSWORD, SECRET, TOKEN, etc.)
14+
// high_entropy_string: calculated high entropy (randomness) in a string
15+
// These detectors require an entropy threshold value (0 = low entropy, 4+ = very high entropy).
16+
17+
// create a custom scanner config
18+
cfg := scanner.NewConfigWithDefaults()
19+
cfg.Transformers = []string{"json"}
20+
cfg.DetectorConfigs["keyword"] = []string{"3"}
21+
cfg.DetectorConfigs["high_entropy_string"] = []string{"3"}
22+
23+
// create a scanner from scanner config
24+
scannerClient, err := scanner.NewScannerFromConfig(cfg)
25+
if err != nil {
26+
return nil, fmt.Errorf("Failed to make a config detector: %w", err)
27+
}
28+
29+
// scan the input
30+
ds, err := scannerClient.Scan(input)
31+
if err != nil {
32+
return nil, fmt.Errorf("Failed to scan input: %w", err)
33+
}
34+
35+
// return a list of detector types
36+
list := []string{}
37+
for _, d := range ds {
38+
list = append(list, d.Type)
39+
}
40+
41+
return list, nil
42+
}

‎src/pkg/cli/compose/config_detector_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package compose
2+
3+
import "testing"
4+
5+
func TestDetectConfig(t *testing.T) {
6+
tests := []struct {
7+
input string
8+
expectedOutput []string
9+
}{
10+
{"", nil},
11+
{"not a secret", nil},
12+
{"https://user:[email protected]", []string{"URL with password"}},
13+
{"LINK: https://user:[email protected], LINK: https://user:[email protected]", []string{"URL with password", "URL with password"}},
14+
{"api-key=50m34p1k3y", []string{"Keyword Detector"}},
15+
{"1234567890abcdef", []string{"High entropy string"}},
16+
{"ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890", []string{"Github authentication"}},
17+
{"AROA1234567890ABCDEF", []string{"AWS Client ID"}},
18+
}
19+
20+
for _, tt := range tests {
21+
t.Run(tt.input, func(t *testing.T) {
22+
ds, err := detectConfig(tt.input)
23+
24+
//check for error
25+
if err != nil {
26+
if len(tt.expectedOutput) > 0 && tt.expectedOutput[0] != "" {
27+
t.Errorf("Error: %v", err)
28+
}
29+
return
30+
}
31+
32+
// check for length of the output
33+
if len(ds) != len(tt.expectedOutput) {
34+
t.Errorf("Expected %d detector types, but got %d", len(tt.expectedOutput), len(ds))
35+
return
36+
}
37+
38+
// check for the output values
39+
for i, d := range ds {
40+
if d != tt.expectedOutput[i] {
41+
t.Errorf("Expected detector type %s, but got %s", tt.expectedOutput[i], d)
42+
}
43+
}
44+
})
45+
}
46+
}

‎src/pkg/cli/compose/validation.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,27 @@ func validateService(svccfg *composeTypes.ServiceConfig, project *composeTypes.P
194194
term.Warnf("unsupported secret %q: not marked external:true", secret.Source) // TODO: support secrets from environment/file
195195
}
196196
}
197+
198+
// check for compose file environment variables that may be sensitive
199+
for key, value := range svccfg.Environment {
200+
if value != nil {
201+
// format input as a key-value pair string
202+
input := key + "=" + *value
203+
204+
// call detectConfig to check for sensitive information
205+
ds, err := detectConfig(input)
206+
if err != nil {
207+
return fmt.Errorf("service %q: %w", svccfg.Name, err)
208+
}
209+
210+
// show warning if sensitive information is detected
211+
if len(ds) > 0 {
212+
term.Warnf("service %q: environment %q may contain sensitive information; consider using 'defang config set %s' to securely store this value", svccfg.Name, key, key)
213+
term.Debugf("service %q: environment %q may contain detected secrets of type: %q", svccfg.Name, key, ds)
214+
}
215+
}
216+
}
217+
197218
err := validatePorts(svccfg.Ports)
198219
if err != nil {
199220
return err

‎src/testdata/configdetection/compose.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
services:
2+
configdetection:
3+
image: alpine
4+
environment:
5+
- API_KEY=50m34p1k3y # keyword detector
6+
- AWS_CLIENT_ID=AROA1234567890ABCDEF # aws_client_id detector
7+
- GH_PAT=ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890 # github detector
8+
- HIGH_ENTROPY_STRING=1234567890abcdef # high_entropy_string detector
9+
- MY_URL=https://user:[email protected] # url_password detector
10+
- NOT_SENSITIVE=notsensitive

‎src/testdata/configdetection/compose.yaml.fixup‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"configdetection": {
3+
"command": null,
4+
"entrypoint": null,
5+
"environment": {
6+
"API_KEY": "50m34p1k3y",
7+
"AWS_CLIENT_ID": "AROA1234567890ABCDEF",
8+
"GH_PAT": "ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890",
9+
"HIGH_ENTROPY_STRING": "1234567890abcdef",
10+
"MY_URL": "https://user:[email protected]",
11+
"NOT_SENSITIVE": "notsensitive"
12+
},
13+
"image": "alpine",
14+
"networks": {
15+
"default": null
16+
}
17+
}
18+
}

‎src/testdata/configdetection/compose.yaml.golden‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: configdetection
2+
services:
3+
configdetection:
4+
environment:
5+
API_KEY: 50m34p1k3y
6+
AWS_CLIENT_ID: AROA1234567890ABCDEF
7+
GH_PAT: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890
8+
HIGH_ENTROPY_STRING: 1234567890abcdef
9+
MY_URL: https://user:[email protected]
10+
NOT_SENSITIVE: notsensitive
11+
image: alpine
12+
networks:
13+
default: null
14+
networks:
15+
default:
16+
name: configdetection_default

‎src/testdata/configdetection/compose.yaml.warnings‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
! service "configdetection": environment "API_KEY" may contain sensitive information; consider using 'defang config set API_KEY' to securely store this value
2+
! service "configdetection": environment "AWS_CLIENT_ID" may contain sensitive information; consider using 'defang config set AWS_CLIENT_ID' to securely store this value
3+
! service "configdetection": environment "GH_PAT" may contain sensitive information; consider using 'defang config set GH_PAT' to securely store this value
4+
! service "configdetection": environment "HIGH_ENTROPY_STRING" may contain sensitive information; consider using 'defang config set HIGH_ENTROPY_STRING' to securely store this value
5+
! service "configdetection": environment "MY_URL" may contain sensitive information; consider using 'defang config set MY_URL' to securely store this value
6+
! service "configdetection": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors

0 commit comments

Comments
 (0)