Skip to content

Commit 8161a16

Browse files
nickajacks1knadh
nickajacks1
and
knadh
authored
Refactor env provider into a major breaking version /v2 (#341)
- `providers/env` is now `/providers/env/v2` - Add new `Opt{}` struct with transform functions that can be passed to the constructor instead of the old `WithProvider(..)` functions. This allow future callbacks and options to be added without breaking changes. - `Opt.TransformFunc` is now a single transformation function for keys and values. - `Opt.EnvirnFunc` allows the source of environment variables to be plugged for testing and mocking. It defaults to `os.Environ`. --------- Co-authored-by: Kailash Nadh <kailash@nadh.in>
1 parent 53e7d90 commit 8161a16

File tree

9 files changed

+271
-333
lines changed

9 files changed

+271
-333
lines changed

‎README.md‎

Lines changed: 29 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ All external dependencies in providers and parsers are detached from the core an
1717
go get -u github.com/knadh/koanf/v2
1818

1919
# Install the necessary Provider(s).
20-
# Available: file, env, posflag, basicflag, confmap, rawbytes,
20+
# Available: file, env/v2, posflag, basicflag, confmap, rawbytes,
2121
# structs, fs, s3, appconfig/v2, consul/v2, etcd/v2, vault/v2, parameterstore/v2
2222
# eg: go get -u github.com/knadh/koanf/providers/s3
2323
# eg: go get -u github.com/knadh/koanf/providers/consul/v2
@@ -229,7 +229,7 @@ import (
229229

230230
"github.com/knadh/koanf/v2"
231231
"github.com/knadh/koanf/parsers/json"
232-
"github.com/knadh/koanf/providers/env"
232+
"github.com/knadh/koanf/providers/env/v2"
233233
"github.com/knadh/koanf/providers/file"
234234
)
235235

@@ -242,45 +242,35 @@ func main() {
242242
log.Fatalf("error loading config: %v", err)
243243
}
244244

245-
// Load environment variables and merge into the loaded config.
246-
// "MYVAR" is the prefix to filter the env vars by.
247-
// "." is the delimiter used to represent the key hierarchy in env vars.
248-
// The (optional, or can be nil) function can be used to transform
249-
// the env var names, for instance, to lowercase them.
250-
//
251-
// For example, env vars: MYVAR_TYPE and MYVAR_PARENT1_CHILD1_NAME
252-
// will be merged into the "type" and the nested "parent1.child1.name"
253-
// keys in the config file here as we lowercase the key,
254-
// replace `_` with `.` and strip the MYVAR_ prefix so that
255-
// only "parent1.child1.name" remains.
256-
k.Load(env.Provider("MYVAR_", ".", func(s string) string {
257-
return strings.Replace(strings.ToLower(
258-
strings.TrimPrefix(s, "MYVAR_")), "_", ".", -1)
245+
// Load only environment variables with prefix "MYVAR_" and merge into config.
246+
// Transform var names by:
247+
// 1. Converting to lowercase
248+
// 2. Removing "MYVAR_" prefix
249+
// 3. Replacing "_" with "." to representing nesting using the . delimiter.
250+
// Example: MYVAR_PARENT1_CHILD1_NAME becomes "parent1.child1.name"
251+
k.Load(env.Provider(".", env.Opt{
252+
Prefix: "MYVAR_",
253+
TransformFunc: func(k, v string) (string, any) {
254+
// Transform the key.
255+
k = strings.ReplaceAll(strings.ToLower(strings.TrimPrefix(k, "MYVAR_")), "_", ".")
256+
257+
// Transform the value into slices, if they contain spaces.
258+
// Eg: MYVAR_TAGS="foo bar baz" -> tags: ["foo", "bar", "baz"]
259+
// This is to demonstrate that string values can be transformed to any type
260+
// where necessary.
261+
if strings.Contains(v, " ") {
262+
return k, strings.Split(v, " ")
263+
}
264+
265+
return k, v
266+
},
259267
}), nil)
260268

261-
fmt.Println("name is = ", k.String("parent1.child1.name"))
262-
}
263-
```
269+
fmt.Println("name is =", k.String("parent1.child1.name"))
270+
fmt.Println("time is =", k.Time("time", time.DateOnly))
271+
fmt.Println("ids are =", k.Strings("parent1.child1.grandchild1.ids"))
272+
}```
264273
265-
You can also use the `env.ProviderWithValue` with a callback that supports mutating both the key and value
266-
to return types other than a string. For example, here, env values separated by spaces are
267-
returned as string slices or arrays. eg: `MYVAR_slice=a b c` becomes `slice: [a, b, c]`.
268-
269-
```go
270-
k.Load(env.ProviderWithValue("MYVAR_", ".", func(s string, v string) (string, interface{}) {
271-
// Strip out the MYVAR_ prefix and lowercase and get the key while also replacing
272-
// the _ character with . in the key (koanf delimiter).
273-
key := strings.Replace(strings.ToLower(strings.TrimPrefix(s, "MYVAR_")), "_", ".", -1)
274-
275-
// If there is a space in the value, split the value into a slice by the space.
276-
if strings.Contains(v, " ") {
277-
return key, strings.Split(v, " ")
278-
}
279-
280-
// Otherwise, return the plain string.
281-
return key, v
282-
}), nil)
283-
```
284274
285275
### Reading from an S3 bucket
286276
@@ -662,7 +652,7 @@ Install with `go get -u github.com/knadh/koanf/providers/$provider`
662652
| fs | `fs.Provider(f fs.FS, filepath string)` | (**Experimental**) Reads a file from fs.FS and returns the raw bytes to be parsed. The provider requires `go v1.16` or higher. |
663653
| basicflag | `basicflag.Provider(f *flag.FlagSet, delim string)` | Takes a stdlib `flag.FlagSet` |
664654
| posflag | `posflag.Provider(f *pflag.FlagSet, delim string)` | Takes an `spf13/pflag.FlagSet` (advanced POSIX compatible flags with multiple types) and provides a nested config map based on delim. |
665-
| env | `env.Provider(prefix, delim string, f func(s string) string)` | Takes an optional prefix to filter env variables by, an optional function that takes and returns a string to transform env variables, and returns a nested config map based on delim. |
655+
| env/v2 | `env.Provider(prefix, delim string, f func(s string) string)` | Takes an optional prefix to filter env variables by, an optional function that takes and returns a string to transform env variables, and returns a nested config map based on delim. |
666656
| confmap | `confmap.Provider(mp map[string]interface{}, delim string)` | Takes a premade `map[string]interface{}` conf map. If delim is provided, the keys are assumed to be flattened, thus unflattened using delim. |
667657
| structs | `structs.Provider(s interface{}, tag string)` | Takes a struct and struct tag. |
668658
| s3 | `s3.Provider(s3.S3Config{})` | Takes a s3 config struct. |

‎examples/read-environment/main.go‎

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,28 @@ package main
33
import (
44
"fmt"
55
"log"
6+
"os"
7+
"slices"
68
"strings"
9+
"time"
710

811
"github.com/knadh/koanf/parsers/json"
9-
"github.com/knadh/koanf/providers/env"
12+
"github.com/knadh/koanf/providers/env/v2"
1013
"github.com/knadh/koanf/providers/file"
1114
"github.com/knadh/koanf/v2"
1215
)
1316

1417
var k = koanf.New(".")
1518

19+
// Run this example:
20+
//
21+
// go run ./examples/read-environment
22+
//
23+
// Try setting environment variables to see what changes (and what doesn't!):
24+
//
25+
// MYVAR_PARENT1_CHILD1_NAME=FooBar go run ./examples/read-environment
26+
// MYVAR_TIME=2020-02-02 go run ./examples/read-environment
27+
// MYVAR_PARENT1_CHILD1_GRANDCHILD1_IDS="3 2 1" go run ./examples/read-environment
1628
func main() {
1729
// Load JSON config.
1830
if err := k.Load(file.Provider("mock/mock.json"), json.Parser()); err != nil {
@@ -22,34 +34,40 @@ func main() {
2234
// Load environment variables and merge into the loaded config.
2335
// "MYVAR" is the prefix to filter the env vars by.
2436
// "." is the delimiter used to represent the key hierarchy in env vars.
25-
// The (optional, or can be nil) function can be used to transform
26-
// the env var names, for instance, to lowercase them.
37+
// The optional TransformFunc can be used to transform the env var names,
38+
// for instance, to lowercase them. Values can also be modified into types
39+
// other than strings, for example, to turn space separated env vars into
40+
// slices.
2741
//
28-
// For example, env vars: MYVAR_TYPE and MYVAR_PARENT1_CHILD1_NAME
2942
// will be merged into the "type" and the nested "parent1.child1.name"
43+
// For example, env vars: MYVAR_TYPE and MYVAR_PARENT1_CHILD1_NAME
3044
// keys in the config file here as we lowercase the key,
3145
// replace `_` with `.` and strip the MYVAR_ prefix so that
3246
// only "parent1.child1.name" remains.
33-
k.Load(env.Provider("MYVAR_", ".", func(s string) string {
34-
return strings.Replace(strings.ToLower(
35-
strings.TrimPrefix(s, "MYVAR_")), "_", ".", -1)
36-
}), nil)
37-
38-
// Use ProviderWithValue() to process both keys and values into types other than strings,
39-
// for example, turn space separated env vars into slices.
40-
// k.Load(env.ProviderWithValue("MYVAR_", ".", func(s string, v string) (string, interface{}) {
41-
// // Strip out the MYVAR_ prefix and lowercase and get the key while also replacing
42-
// // the _ character with . in the key (koanf delimiter).
43-
// key := strings.Replace(strings.ToLower(strings.TrimPrefix(s, "MYVAR_")), "_", ".", -1)
44-
45-
// // If there is a space in the value, split the value into a slice by the space.
46-
// if strings.Contains(v, " ") {
47-
// return key, strings.Split(v, " ")
48-
// }
47+
//
48+
// The optional EnvironFunc can be used to control what environment
49+
// variables are read by Koanf. In this example, we ensure that the time
50+
// variable cannot be overridden by env vars.
51+
k.Load(env.Provider(".", env.Opt{
52+
Prefix: "MYVAR_",
53+
TransformFunc: func(k, v string) (string, any) {
54+
k = strings.ReplaceAll(strings.ToLower(
55+
strings.TrimPrefix(k, "MYVAR_")), "_", ".")
4956

50-
// // Otherwise, return the plain string.
51-
// return key, v
52-
// }), nil)
57+
// If there is a space in the value, split the value into a slice by the space.
58+
if strings.Contains(v, " ") {
59+
return k, strings.Split(v, " ")
60+
}
61+
return k, v
62+
},
63+
EnvironFunc: func() []string {
64+
return slices.DeleteFunc(os.Environ(), func(s string) bool {
65+
return strings.HasPrefix(s, "MYVAR_TIME")
66+
})
67+
},
68+
}), nil)
5369

54-
fmt.Println("name is = ", k.String("parent1.child1.name"))
70+
fmt.Println("name is =", k.String("parent1.child1.name"))
71+
fmt.Println("time is =", k.Time("time", time.DateOnly))
72+
fmt.Println("ids are =", k.Strings("parent1.child1.grandchild1.ids"))
5573
}

‎go.work.sum‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,9 @@ cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNp
239239
cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg=
240240
cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw=
241241
cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g=
242+
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
242243
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
244+
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
243245
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
244246
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
245247
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
@@ -266,6 +268,7 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
266268
github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE=
267269
github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
268270
github.com/dgryski/go-ddmin v0.0.0-20210904190556-96a6d69f1034/go.mod h1:zz4KxBkcXUWKjIcrc+uphJ1gPh/t18ymGm3PmQ+VGTk=
271+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
269272
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
270273
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
271274
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -293,6 +296,7 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
293296
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
294297
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
295298
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
299+
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
296300
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
297301
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
298302
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -316,6 +320,10 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
316320
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
317321
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
318322
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
323+
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
324+
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
325+
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
326+
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
319327
github.com/lyft/protoc-gen-star/v2 v2.0.4-0.20230330145011-496ad1ac90a4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk=
320328
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
321329
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
@@ -335,6 +343,7 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1
335343
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
336344
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
337345
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
346+
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
338347
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
339348
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
340349
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=

‎providers/env/env.go‎

Lines changed: 44 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,30 @@ import (
1212

1313
// Env implements an environment variables provider.
1414
type Env struct {
15-
prefix string
16-
delim string
17-
cb func(key string, value string) (string, interface{})
15+
prefix string
16+
delim string
17+
transform func(key string, value string) (string, any)
18+
environ func() []string
19+
}
20+
21+
// Opt represents optional configuration passed to the provider.
22+
type Opt struct {
23+
// If specified (case-sensitive), only env vars beginning with
24+
// the prefix are captured. eg: "APP_"
25+
Prefix string
26+
27+
// TransformFunc is an optional callback that takes an environment
28+
// variable's string name and value, runs arbitrary transformations
29+
// on them and returns a transformed string key and value of any type.
30+
// Common usecase are stripping prefixes from keys, lowercasing variable names,
31+
// replacing _ with . etc. Eg: APP_DB_HOST -> db.host
32+
// If the returned variable name is an empty string (""), it is ignored altogether.
33+
TransformFunc func(k, v string) (string, any)
34+
35+
// EnvironFunc is the optional function that provides the environment
36+
// variables to the provider. If it's not set, os.Environ is used.
37+
// This can be used to inject environment variables in tests and mocks.
38+
EnvironFunc func() []string
1839
}
1940

2041
// Provider returns an environment variables provider that returns
@@ -23,36 +44,23 @@ type Env struct {
2344
// delim "." will convert the key `parent.child.key: 1`
2445
// to `{parent: {child: {key: 1}}}`.
2546
//
26-
// If prefix is specified (case-sensitive), only the env vars with
27-
// the prefix are captured. cb is an optional callback that takes
28-
// a string and returns a string (the env variable name) in case
29-
// transformations have to be applied, for instance, to lowercase
30-
// everything, strip prefixes and replace _ with . etc.
31-
// If the callback returns an empty string, the variable will be
32-
// ignored.
33-
func Provider(prefix, delim string, cb func(s string) string) *Env {
47+
// It takes an optional Opt argument containing a function to override
48+
// the default source for environment variables, which can be useful
49+
// for mocking and parallel unit tests.
50+
func Provider(delim string, o Opt) *Env {
3451
e := &Env{
35-
prefix: prefix,
36-
delim: delim,
37-
}
38-
if cb != nil {
39-
e.cb = func(key string, value string) (string, interface{}) {
40-
return cb(key), value
41-
}
52+
delim: delim,
53+
prefix: o.Prefix,
54+
environ: o.EnvironFunc,
55+
transform: o.TransformFunc,
4256
}
43-
return e
44-
}
4557

46-
// ProviderWithValue works exactly the same as Provider except the callback
47-
// takes a (key, value) with the variable name and value and allows you
48-
// to modify both. This is useful for cases where you may want to return
49-
// other types like a string slice instead of just a string.
50-
func ProviderWithValue(prefix, delim string, cb func(key string, value string) (string, interface{})) *Env {
51-
return &Env{
52-
prefix: prefix,
53-
delim: delim,
54-
cb: cb,
58+
// No environ function provided, use the default os.Environ.
59+
if e.environ == nil {
60+
e.environ = os.Environ
5561
}
62+
63+
return e
5664
}
5765

5866
// ReadBytes is not supported by the env provider.
@@ -62,10 +70,10 @@ func (e *Env) ReadBytes() ([]byte, error) {
6270

6371
// Read reads all available environment variables into a key:value map
6472
// and returns it.
65-
func (e *Env) Read() (map[string]interface{}, error) {
73+
func (e *Env) Read() (map[string]any, error) {
6674
// Collect the environment variable keys.
6775
var keys []string
68-
for _, k := range os.Environ() {
76+
for _, k := range e.environ() {
6977
if e.prefix != "" {
7078
if strings.HasPrefix(k, e.prefix) {
7179
keys = append(keys, k)
@@ -75,18 +83,19 @@ func (e *Env) Read() (map[string]interface{}, error) {
7583
}
7684
}
7785

78-
mp := make(map[string]interface{})
86+
mp := make(map[string]any)
7987
for _, k := range keys {
8088
parts := strings.SplitN(k, "=", 2)
8189

8290
// If there's a transformation callback,
8391
// run it through every key/value.
84-
if e.cb != nil {
85-
key, value := e.cb(parts[0], parts[1])
86-
// If the callback blanked the key, it should be omitted
92+
if e.transform != nil {
93+
key, value := e.transform(parts[0], parts[1])
94+
// If the callback blanked the key, omit it.
8795
if key == "" {
8896
continue
8997
}
98+
9099
mp[key] = value
91100
} else {
92101
mp[parts[0]] = parts[1]

0 commit comments

Comments
 (0)