Skip to content

Commit de96705

Browse files
committed
add: partitionFilesLimit param, confirm func, renamed continue-generation and F flag
1 parent 30e6d42 commit de96705

File tree

19 files changed

+303
-68
lines changed

19 files changed

+303
-68
lines changed

doc/en/usage.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,13 +173,17 @@ Structure `output.params` for format `csv`:
173173
- `datetime_format`: Date-time format. Default is `2006-01-02T15:04:05Z07:00`.
174174
- `without_headers`: Flag indicating if CSV headers should be excluded from data files.
175175
- `delimiter`: Single-character CSV delimiter. Default is `,`.
176+
- `partition_files_limit`: Limit on the number of partition files, upon reaching which a prompt will appear asking whether to continue.
177+
Ignored if the `--force` flag is specified. Default is `1000`.
176178

177179
Structure `output.params` for format `parquet`:
178180

179181
- `compression_codec`: Compression codec. Supported values: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZ4`, `ZSTD`.
180182
Default is `UNCOMPRESSED`.
181183
- `float_precision`: Floating-point number precision. Default is `2`.
182184
- `datetime_format`: Date-time format. Supported values: `millis`, `micros`. Default is `millis`.
185+
- `partition_files_limit`: Limit on the number of partition files, upon reaching which a prompt will appear asking whether to continue.
186+
Ignored if the `--force` flag is specified. Default is `1000`.
183187

184188
Structure `output.params` for format `http`:
185189

@@ -444,7 +448,7 @@ sdvg generate ./models.yml
444448
### Ignoring conflicts
445449

446450
If you want to automatically remove conflicting files from the output directory
447-
and continue generation without additional prompts, use the `-F` or `--force` flag:
451+
and continue generation without additional prompts, use the `-f` or `--force` flag:
448452

449453
```shell
450454
sdvg generate --force ./models.yml
@@ -455,7 +459,7 @@ sdvg generate --force ./models.yml
455459
To continue generation from the last recorded row:
456460

457461
```shell
458-
sdvg generate --continue-generation ./models.yml
462+
sdvg generate --continue ./models.yml
459463
```
460464

461465
> **Important**: To correctly continue generation, you must not change the generation configuration

doc/ru/usage.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,13 +179,17 @@ open_ai:
179179
- `datetime_format`: Формат даты и времени. По умолчанию `2006-01-02T15:04:05Z07:00`.
180180
- `without_headers`: Флаг, указывающий, исключать ли CSV заголовок из файлов с данными.
181181
- `delimiter`: Односимвольный CSV разделитель. По умолчанию `,`.
182+
- `partition_files_limit`: Ограничение количества файлов партиций, при достижении которого всплывет вопрос о продолжении.
183+
Игнорируется при указании флага `--force`. По умолчанию `1000`
182184

183185
Структура `output.params` для формата `parquet`:
184186

185187
- `compression_codec`: Кодек сжатия. Поддерживаемые значения: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZ4`, `ZSTD`.
186188
По умолчанию `UNCOMPRESSED`.
187189
- `float_precision`: Точность чисел с плавающей запятой. По умолчанию `2`.
188190
- `datetime_format`: Формат даты и времени. Поддерживаемые значения: `millis`, `micros`. По умолчанию `millis`.
191+
- `partition_files_limit`: Ограничение количества файлов партиций, при достижении которого всплывет вопрос о продолжении.
192+
Игнорируется при указании флага `--force`. По умолчанию `1000`
189193

190194
Структура `output.params` для формата `http`:
191195

@@ -450,7 +454,7 @@ sdvg generate ./models.yml
450454
### Игнорирование конфликтов
451455

452456
Если вы хотите автоматически удалить конфликтующие файлы в выходной директории
453-
и продолжить генерацию без дополнительных сообщений, используйте флаг `-F` или `--force`:
457+
и продолжить генерацию без дополнительных сообщений, используйте флаг `-f` или `--force`:
454458

455459
```shell
456460
sdvg generate --force ./models.yml
@@ -461,7 +465,7 @@ sdvg generate --force ./models.yml
461465
Для продолжения генерации с последней записанной строки:
462466

463467
```shell
464-
sdvg generate --continue-generation ./models.yml
468+
sdvg generate --continue ./models.yml
465469
```
466470

467471
> **Важно**: для корректного продолжения генерации нельзя менять конфигурацию генерации и уже сгенерированные данные.

internal/generator/cli/commands/consts.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@ const (
66
ConfigPathDefaultValue = ""
77
ConfigPathUsage = "Location of config file"
88

9-
ContinueGenerationFlag = "continue-generation"
10-
ContinueGenerationShortFlag = "C"
9+
ContinueGenerationFlag = "continue"
10+
ContinueGenerationShortFlag = "c"
1111
ContinueGenerationDefaultValue = false
1212
ContinueGenerationUsage = "Continue generation from the last recorded row"
1313

1414
ForceGenerationFlag = "force"
15-
ForceGenerationShortFlag = "F"
15+
ForceGenerationShortFlag = "f"
1616
ForceGenerationFlagDefaultValue = false
17-
ForceGenerationUsage = "Force generation even if output file conflicts found"
17+
ForceGenerationUsage = "Force generation even if output file conflicts found and partition files limit reached"
1818

1919
TTYFlag = "tty"
2020
TTYShortFlag = "t"

internal/generator/cli/commands/generate/generate.go

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/spf13/cobra"
1313
"github.com/spf13/pflag"
1414
"github.com/tarantool/sdvg/internal/generator/cli/commands"
15+
"github.com/tarantool/sdvg/internal/generator/cli/confirm"
1516
"github.com/tarantool/sdvg/internal/generator/cli/options"
1617
"github.com/tarantool/sdvg/internal/generator/cli/progress"
1718
"github.com/tarantool/sdvg/internal/generator/cli/progress/bar"
@@ -124,7 +125,9 @@ func runGenerate(ctx context.Context, opts *generateOptions) error {
124125
return err
125126
}
126127

127-
out := general.NewOutput(generationCfg, opts.continueGeneration, opts.forceGeneration)
128+
progressTrackerManager, confirm := initProgressTrackerManager(ctx, opts.renderer, opts.useTTY)
129+
130+
out := general.NewOutput(generationCfg, opts.continueGeneration, opts.forceGeneration, confirm)
128131

129132
taskID, err := opts.useCase.CreateTask(
130133
ctx, usecase.TaskConfig{
@@ -143,12 +146,11 @@ func runGenerate(ctx context.Context, opts *generateOptions) error {
143146
)
144147

145148
startProgressTracking(
146-
ctx,
149+
progressTrackerManager,
147150
opts.useCase,
148151
taskID,
149152
&finished,
150153
&wg,
151-
opts.useTTY,
152154
)
153155

154156
err = opts.useCase.WaitResult(taskID)
@@ -173,26 +175,37 @@ func runGenerate(ctx context.Context, opts *generateOptions) error {
173175
return nil
174176
}
175177

178+
// initProgressTrackerManager inits progress bar manager (progress.Tracker) and builds streams.Confirm func based on useTTY
179+
func initProgressTrackerManager(ctx context.Context, renderer render.Renderer, useTTY bool) (progress.Tracker, confirm.Confirm) {
180+
var progressTrackerManager progress.Tracker
181+
var confirmFunc confirm.Confirm
182+
183+
if useTTY {
184+
progressTrackerManager = bar.NewProgressBarManager(ctx)
185+
186+
confirmFunc = confirm.BuildConfirmTTY(renderer, progressTrackerManager)
187+
} else {
188+
isUpdatePaused := &atomic.Bool{}
189+
190+
progressTrackerManager = log.NewProgressLogManager(ctx, isUpdatePaused)
191+
192+
confirmFunc = confirm.BuildConfirmNoTTY(renderer, progressTrackerManager, isUpdatePaused)
193+
}
194+
195+
return progressTrackerManager, confirmFunc
196+
}
197+
176198
// startProgressTracking runs function to track progress of task
177199
// by getting progress from usecase object and displaying it.
178200
func startProgressTracking(
179-
ctx context.Context,
201+
progressTrackerManager progress.Tracker,
180202
uc usecase.UseCase,
181203
taskID string,
182204
finished *atomic.Bool,
183205
wg *sync.WaitGroup,
184-
useTTY bool,
185206
) {
186207
const delay = 500 * time.Millisecond
187208

188-
var progressTrackerManager progress.Tracker
189-
190-
if useTTY {
191-
progressTrackerManager = bar.NewProgressBarManager(ctx)
192-
} else {
193-
progressTrackerManager = log.NewProgressLogManager(ctx)
194-
}
195-
196209
wg.Add(1)
197210

198211
go func() {

internal/generator/cli/commands/serve/handlers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func handleGenerate(opts handlerOptions, c echo.Context) error {
5858

5959
generationConfig.OutputConfig.Dir = models.DefaultOutputDir
6060

61-
out := general.NewOutput(&generationConfig, false, true)
61+
out := general.NewOutput(&generationConfig, false, true, nil)
6262

6363
taskID, err := opts.useCase.CreateTask(
6464
c.Request().Context(), usecase.TaskConfig{
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package confirm
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"strings"
8+
"sync/atomic"
9+
10+
"github.com/manifoldco/promptui"
11+
"github.com/pkg/errors"
12+
"github.com/tarantool/sdvg/internal/generator/cli/render"
13+
"github.com/tarantool/sdvg/internal/generator/cli/utils"
14+
)
15+
16+
// Confirm asks user a yes/no question. Returns true for “yes”.
17+
type Confirm func(ctx context.Context, question string) (bool, error)
18+
19+
func BuildConfirmTTY(in io.Reader, out io.Writer) func(ctx context.Context, question string) (bool, error) {
20+
return func(ctx context.Context, question string) (bool, error) {
21+
fmt.Fprintln(out)
22+
23+
prompt := promptui.Prompt{
24+
Label: question + " [y/N]: ",
25+
Default: "y",
26+
Stdin: utils.DummyReadWriteCloser{Reader: in},
27+
Stdout: utils.DummyReadWriteCloser{Writer: out},
28+
}
29+
validate := func(s string) error {
30+
if len(s) == 1 && strings.Contains("YyNn", s) || prompt.Default != "" && len(s) == 0 {
31+
return nil
32+
}
33+
return errors.New("invalid input")
34+
}
35+
prompt.Validate = validate
36+
37+
var (
38+
input string
39+
err error
40+
promptFinished = make(chan struct{})
41+
)
42+
43+
go func() {
44+
input, err = prompt.Run() // goroutine will block here until user input
45+
46+
promptFinished <- struct{}{}
47+
}()
48+
49+
select {
50+
case <-ctx.Done():
51+
return false, ctx.Err()
52+
case <-promptFinished:
53+
}
54+
55+
if err != nil {
56+
return false, errors.WithMessage(err, "confirm prompt failed")
57+
}
58+
59+
return strings.Contains("Yy", input), nil
60+
}
61+
}
62+
63+
func BuildConfirmNoTTY(in render.Renderer, out io.Writer, isUpdatePaused *atomic.Bool) func(ctx context.Context, question string) (bool, error) {
64+
return func(ctx context.Context, question string) (bool, error) {
65+
// here we pause ProgressLogManager to stop sending progress messages
66+
isUpdatePaused.Store(true)
67+
defer isUpdatePaused.Store(false)
68+
69+
for {
70+
fmt.Fprintf(out, "%s [y/N]: ", question)
71+
72+
var (
73+
input string
74+
err error
75+
inputReadFinished = make(chan struct{})
76+
)
77+
78+
go func() {
79+
input, err = in.ReadLine() // goroutine will block here until user input
80+
81+
inputReadFinished <- struct{}{}
82+
}()
83+
84+
select {
85+
case <-ctx.Done():
86+
return false, ctx.Err()
87+
case <-inputReadFinished:
88+
}
89+
90+
if err != nil {
91+
return false, err
92+
}
93+
94+
if !in.IsTerminal() {
95+
fmt.Fprintln(out, input)
96+
}
97+
98+
switch strings.ToLower(strings.TrimSpace(input)) {
99+
case "y", "yes":
100+
return true, nil
101+
case "", "n", "no":
102+
return false, nil
103+
default:
104+
fmt.Fprintln(out, "Please enter y or n")
105+
}
106+
}
107+
}
108+
}

internal/generator/cli/progress/bar/bar.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,8 @@ func (p *ProgressBarManager) UpdateProgress(name string, progress usecase.Progre
7777
func (p *ProgressBarManager) Wait() {
7878
p.progressManager.Wait()
7979
}
80+
81+
// Write writes to stdout.
82+
func (p *ProgressBarManager) Write(b []byte) (int, error) {
83+
return p.progressManager.Write(b)
84+
}

internal/generator/cli/progress/interfaces.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ type Tracker interface {
1010
UpdateProgress(name string, progress usecase.Progress)
1111
// Wait function should wait for all tracked tasks to complete.
1212
Wait()
13+
// Write function should write to stdout.
14+
Write(b []byte) (int, error)
1315
}

internal/generator/cli/progress/log/log.go

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"fmt"
66
"log/slog"
77
"math"
8+
"os"
89
"sync"
10+
"sync/atomic"
911
"time"
1012

1113
"github.com/tarantool/sdvg/internal/generator/cli/progress"
@@ -42,13 +44,16 @@ type ProgressLogManager struct {
4244
ctx context.Context //nolint:containedctx
4345
tasks map[string]*task
4446
wg sync.WaitGroup
47+
48+
isUpdatePaused *atomic.Bool
4549
}
4650

47-
// NewProgressLogManager creates NewProgressLogManager object.
48-
func NewProgressLogManager(ctx context.Context) progress.Tracker {
51+
// NewProgressLogManager creates NewProgressLogManager object. isUpdatePaused is used to pause UpdateProgress.
52+
func NewProgressLogManager(ctx context.Context, isUpdatePaused *atomic.Bool) progress.Tracker {
4953
return &ProgressLogManager{
50-
ctx: ctx,
51-
tasks: make(map[string]*task),
54+
ctx: ctx,
55+
tasks: make(map[string]*task),
56+
isUpdatePaused: isUpdatePaused,
5257
}
5358
}
5459

@@ -78,6 +83,12 @@ func (p *ProgressLogManager) UpdateProgress(name string, progress usecase.Progre
7883
return
7984
}
8085

86+
for p.isUpdatePaused.Load() {
87+
if t.isDone() {
88+
return
89+
}
90+
}
91+
8192
p.updateIntervals(t, progress.Done)
8293

8394
t.current = progress.Done
@@ -138,3 +149,8 @@ func (p *ProgressLogManager) eta(t *task) string {
138149

139150
return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
140151
}
152+
153+
// Write writes to default stdout.
154+
func (p *ProgressLogManager) Write(b []byte) (int, error) {
155+
return os.Stdout.Write(b)
156+
}

internal/generator/cli/render/interfaces.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package render
22

3-
import "context"
3+
import (
4+
"context"
5+
)
46

57
// Renderer interface implementation should render interactive menu.
68
//
@@ -16,4 +18,10 @@ type Renderer interface {
1618
TextMenu(ctx context.Context, title string) (string, error)
1719
// WithSpinner should display spinner.
1820
WithSpinner(title string, fn func())
21+
// IsTerminal should return true if renderer is connected to a terminal.
22+
IsTerminal() bool
23+
// ReadLine should read input from input stream.
24+
ReadLine() (string, error)
25+
// Read should read from input stream.
26+
Read(p []byte) (int, error)
1927
}

0 commit comments

Comments
 (0)