Skip to content

Commit a07f0bc

Browse files
authored
feat: add key-value store for sharing data across executables (#187)
1 parent aca34bf commit a07f0bc

File tree

13 files changed

+617
-0
lines changed

13 files changed

+617
-0
lines changed

cmd/internal/exec.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/jahvon/flow/internal/runner/render"
2222
"github.com/jahvon/flow/internal/runner/request"
2323
"github.com/jahvon/flow/internal/runner/serial"
24+
"github.com/jahvon/flow/internal/services/store"
2425
argUtils "github.com/jahvon/flow/internal/utils/args"
2526
"github.com/jahvon/flow/internal/vault"
2627
"github.com/jahvon/flow/types/executable"
@@ -71,6 +72,7 @@ func execPreRun(_ *context.Context, _ *cobra.Command, _ []string) {
7172
runner.RegisterRunner(parallel.NewRunner())
7273
}
7374

75+
//nolint:funlen
7476
func execFunc(ctx *context.Context, cmd *cobra.Command, verb executable.Verb, args []string) {
7577
logger := ctx.Logger
7678
if err := verb.Validate(); err != nil {
@@ -108,6 +110,9 @@ func execFunc(ctx *context.Context, cmd *cobra.Command, verb executable.Verb, ar
108110
if err != nil {
109111
logger.FatalErr(err)
110112
}
113+
if err = store.SetProcessBucketID(ref.String(), false); err != nil {
114+
logger.FatalErr(err)
115+
}
111116
if envMap == nil {
112117
envMap = make(map[string]string)
113118
}
@@ -131,6 +136,16 @@ func execFunc(ctx *context.Context, cmd *cobra.Command, verb executable.Verb, ar
131136
logger.FatalErr(err)
132137
}
133138
dur := time.Since(startTime)
139+
processStore, err := store.NewStore()
140+
if err != nil {
141+
logger.Errorf("failed clearing process store\n%v", err)
142+
}
143+
if processStore != nil {
144+
if err = processStore.DeleteBucket(); err != nil {
145+
logger.Errorf("failed clearing process store\n%v", err)
146+
}
147+
_ = processStore.Close()
148+
}
134149
logger.Infox(fmt.Sprintf("%s flow completed", ref), "Elapsed", dur.Round(time.Millisecond))
135150
if TUIEnabled(ctx, cmd) {
136151
if dur > 1*time.Minute && ctx.Config.SendSoundNotification() {

cmd/internal/store.go

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package internal
2+
3+
import (
4+
"strings"
5+
6+
"github.com/jahvon/tuikit/views"
7+
"github.com/spf13/cobra"
8+
9+
"github.com/jahvon/flow/internal/context"
10+
"github.com/jahvon/flow/internal/io"
11+
"github.com/jahvon/flow/internal/services/store"
12+
)
13+
14+
func RegisterStoreCmd(ctx *context.Context, rootCmd *cobra.Command) {
15+
subCmd := &cobra.Command{
16+
Use: "store",
17+
Short: "Manage the data store.",
18+
Args: cobra.NoArgs,
19+
}
20+
registerStoreSetCmd(ctx, subCmd)
21+
registerStoreGetCmd(ctx, subCmd)
22+
registerStoreClearCmd(ctx, subCmd)
23+
rootCmd.AddCommand(subCmd)
24+
}
25+
26+
func registerStoreSetCmd(ctx *context.Context, rootCmd *cobra.Command) {
27+
subCmd := &cobra.Command{
28+
Use: "set",
29+
Short: "Set a key-value pair in the data store.",
30+
Long: dataStoreDescription + "This will overwrite any existing value for the key.",
31+
Args: cobra.MinimumNArgs(1),
32+
Run: func(cmd *cobra.Command, args []string) {
33+
storeSetFunc(ctx, cmd, args)
34+
},
35+
}
36+
rootCmd.AddCommand(subCmd)
37+
}
38+
39+
func storeSetFunc(ctx *context.Context, _ *cobra.Command, args []string) {
40+
key := args[0]
41+
42+
var value string
43+
switch {
44+
case len(args) == 1:
45+
form, err := views.NewForm(
46+
io.Theme(ctx.Config.Theme.String()),
47+
ctx.StdIn(),
48+
ctx.StdOut(),
49+
&views.FormField{
50+
Key: "value",
51+
Type: views.PromptTypeMultiline,
52+
Title: "Enter the value to store",
53+
})
54+
if err != nil {
55+
ctx.Logger.FatalErr(err)
56+
}
57+
if err = form.Run(ctx.Ctx); err != nil {
58+
ctx.Logger.FatalErr(err)
59+
}
60+
value = form.FindByKey("value").Value()
61+
case len(args) == 2:
62+
value = args[1]
63+
default:
64+
ctx.Logger.Warnx("merging multiple arguments into a single value", "count", len(args))
65+
value = strings.Join(args[1:], " ")
66+
}
67+
68+
s, err := store.NewStore()
69+
if err != nil {
70+
ctx.Logger.FatalErr(err)
71+
}
72+
if err = s.CreateBucket(); err != nil {
73+
ctx.Logger.FatalErr(err)
74+
}
75+
defer func() {
76+
if err = s.Close(); err != nil {
77+
ctx.Logger.Error(err, "cleanup failure")
78+
}
79+
}()
80+
if err = s.Set(key, value); err != nil {
81+
ctx.Logger.FatalErr(err)
82+
}
83+
ctx.Logger.Infof("Key %q set in the store", key)
84+
}
85+
86+
func registerStoreGetCmd(ctx *context.Context, rootCmd *cobra.Command) {
87+
subCmd := &cobra.Command{
88+
Use: "get",
89+
Aliases: []string{"view"},
90+
Short: "Get a value from the data store.",
91+
Long: dataStoreDescription + "This will retrieve the value for the given key.",
92+
Args: cobra.ExactArgs(1),
93+
Run: func(cmd *cobra.Command, args []string) {
94+
storeGetFunc(ctx, cmd, args)
95+
},
96+
}
97+
rootCmd.AddCommand(subCmd)
98+
}
99+
100+
func storeGetFunc(ctx *context.Context, _ *cobra.Command, args []string) {
101+
key := args[0]
102+
103+
s, err := store.NewStore()
104+
if err != nil {
105+
ctx.Logger.FatalErr(err)
106+
}
107+
if err = s.CreateBucket(); err != nil {
108+
ctx.Logger.FatalErr(err)
109+
}
110+
defer func() {
111+
if err := s.Close(); err != nil {
112+
ctx.Logger.Error(err, "cleanup failure")
113+
}
114+
}()
115+
value, err := s.Get(key)
116+
if err != nil {
117+
ctx.Logger.FatalErr(err)
118+
}
119+
ctx.Logger.PlainTextSuccess(value)
120+
}
121+
122+
func registerStoreClearCmd(ctx *context.Context, rootCmd *cobra.Command) {
123+
subCmd := &cobra.Command{
124+
Use: "clear",
125+
Aliases: []string{"reset"},
126+
Short: "Clear the data store.",
127+
Long: dataStoreDescription + "This will remove all keys and values from the data store.",
128+
Args: cobra.NoArgs,
129+
Run: func(cmd *cobra.Command, args []string) {
130+
storeClearFunc(ctx, cmd, args)
131+
},
132+
}
133+
rootCmd.AddCommand(subCmd)
134+
}
135+
136+
func storeClearFunc(ctx *context.Context, _ *cobra.Command, _ []string) {
137+
s, err := store.NewStore()
138+
if err != nil {
139+
ctx.Logger.FatalErr(err)
140+
}
141+
if err := s.DeleteBucket(); err != nil {
142+
ctx.Logger.FatalErr(err)
143+
}
144+
ctx.Logger.PlainTextSuccess("Data store cleared")
145+
}
146+
147+
var dataStoreDescription = "The data store is a key-value store that can be used to persist data across executions. " +
148+
"Values that are set outside of an executable will persist across all executions until they are cleared. " +
149+
"When set within an executable, the data will only persist across serial or parallel sub-executables but all " +
150+
"values will be cleared when the parent executable completes.\n\n"

cmd/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,6 @@ func RegisterSubCommands(ctx *context.Context, rootCmd *cobra.Command) {
7171
internal.RegisterWorkspaceCmd(ctx, rootCmd)
7272
internal.RegisterTemplateCmd(ctx, rootCmd)
7373
internal.RegisterLogsCmd(ctx, rootCmd)
74+
internal.RegisterStoreCmd(ctx, rootCmd)
7475
internal.RegisterSyncCmd(ctx, rootCmd)
7576
}

docs/cli/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ See github.com/jahvon/flow for more information.
2424
* [flow library](flow_library.md) - View and manage your library of workspaces and executables.
2525
* [flow logs](flow_logs.md) - List and view logs for previous flow executions.
2626
* [flow secret](flow_secret.md) - Manage flow secrets.
27+
* [flow store](flow_store.md) - Manage the data store.
2728
* [flow sync](flow_sync.md) - Scan workspaces and update flow cache.
2829
* [flow template](flow_template.md) - Manage flowfile templates.
2930
* [flow workspace](flow_workspace.md) - Manage flow workspaces.

docs/cli/flow_store.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
## flow store
2+
3+
Manage the data store.
4+
5+
### Options
6+
7+
```
8+
-h, --help help for store
9+
```
10+
11+
### Options inherited from parent commands
12+
13+
```
14+
-x, --non-interactive Disable displaying flow output via terminal UI rendering. This is only needed if the interactive output is enabled by default in flow's configuration.
15+
--sync Sync flow cache and workspaces
16+
--verbosity int Log verbosity level (-1 to 1)
17+
```
18+
19+
### SEE ALSO
20+
21+
* [flow](flow.md) - flow is a command line interface designed to make managing and running development workflows easier.
22+
* [flow store clear](flow_store_clear.md) - Clear the data store.
23+
* [flow store get](flow_store_get.md) - Get a value from the data store.
24+
* [flow store set](flow_store_set.md) - Set a key-value pair in the data store.
25+

docs/cli/flow_store_clear.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
## flow store clear
2+
3+
Clear the data store.
4+
5+
### Synopsis
6+
7+
The data store is a key-value store that can be used to persist data across executions. Values that are set outside of an executable will persist across all executions until they are cleared. When set within an executable, the data will only persist across serial or parallel sub-executables but all values will be cleared when the parent executable completes.
8+
9+
This will remove all keys and values from the data store.
10+
11+
```
12+
flow store clear [flags]
13+
```
14+
15+
### Options
16+
17+
```
18+
-h, --help help for clear
19+
```
20+
21+
### Options inherited from parent commands
22+
23+
```
24+
-x, --non-interactive Disable displaying flow output via terminal UI rendering. This is only needed if the interactive output is enabled by default in flow's configuration.
25+
--sync Sync flow cache and workspaces
26+
--verbosity int Log verbosity level (-1 to 1)
27+
```
28+
29+
### SEE ALSO
30+
31+
* [flow store](flow_store.md) - Manage the data store.
32+

docs/cli/flow_store_get.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
## flow store get
2+
3+
Get a value from the data store.
4+
5+
### Synopsis
6+
7+
The data store is a key-value store that can be used to persist data across executions. Values that are set outside of an executable will persist across all executions until they are cleared. When set within an executable, the data will only persist across serial or parallel sub-executables but all values will be cleared when the parent executable completes.
8+
9+
This will retrieve the value for the given key.
10+
11+
```
12+
flow store get [flags]
13+
```
14+
15+
### Options
16+
17+
```
18+
-h, --help help for get
19+
```
20+
21+
### Options inherited from parent commands
22+
23+
```
24+
-x, --non-interactive Disable displaying flow output via terminal UI rendering. This is only needed if the interactive output is enabled by default in flow's configuration.
25+
--sync Sync flow cache and workspaces
26+
--verbosity int Log verbosity level (-1 to 1)
27+
```
28+
29+
### SEE ALSO
30+
31+
* [flow store](flow_store.md) - Manage the data store.
32+

docs/cli/flow_store_set.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
## flow store set
2+
3+
Set a key-value pair in the data store.
4+
5+
### Synopsis
6+
7+
The data store is a key-value store that can be used to persist data across executions. Values that are set outside of an executable will persist across all executions until they are cleared. When set within an executable, the data will only persist across serial or parallel sub-executables but all values will be cleared when the parent executable completes.
8+
9+
This will overwrite any existing value for the key.
10+
11+
```
12+
flow store set [flags]
13+
```
14+
15+
### Options
16+
17+
```
18+
-h, --help help for set
19+
```
20+
21+
### Options inherited from parent commands
22+
23+
```
24+
-x, --non-interactive Disable displaying flow output via terminal UI rendering. This is only needed if the interactive output is enabled by default in flow's configuration.
25+
--sync Sync flow cache and workspaces
26+
--verbosity int Log verbosity level (-1 to 1)
27+
```
28+
29+
### SEE ALSO
30+
31+
* [flow store](flow_store.md) - Manage the data store.
32+

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ require (
2020
github.com/pkg/errors v0.9.1
2121
github.com/spf13/cobra v1.8.1
2222
github.com/spf13/pflag v1.0.5
23+
go.etcd.io/bbolt v1.3.11
2324
go.uber.org/mock v0.4.0
2425
golang.org/x/crypto v0.28.0
2526
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
168168
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
169169
github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4=
170170
github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
171+
go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
172+
go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
171173
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
172174
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
173175
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=

0 commit comments

Comments
 (0)