Skip to content

Commit 277548e

Browse files
tonistiigitonistiigi
authored
Merge pull request #3152 from crazy-max/history-export-finalize
history: make sure build record is finalized before exporting
2 parents 3f0aec1 + 68ce10c commit 277548e

File tree

6 files changed

+209
-20
lines changed

6 files changed

+209
-20
lines changed

‎commands/history/export.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,11 @@ import (
2020
)
2121

2222
type exportOptions struct {
23-
builder string
24-
refs []string
25-
output string
26-
all bool
23+
builder string
24+
refs []string
25+
output string
26+
all bool
27+
finalize bool
2728
}
2829

2930
func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error {
@@ -62,6 +63,26 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e
6263
return errors.Errorf("no record found for ref %q", ref)
6364
}
6465

66+
if opts.finalize {
67+
var finalized bool
68+
for _, rec := range recs {
69+
if rec.Trace == nil {
70+
finalized = true
71+
if err := finalizeRecord(ctx, rec.Ref, nodes); err != nil {
72+
return err
73+
}
74+
}
75+
}
76+
if finalized {
77+
recs, err = queryRecords(ctx, ref, nodes, &queryOptions{
78+
CompletedOnly: true,
79+
})
80+
if err != nil {
81+
return err
82+
}
83+
}
84+
}
85+
6586
if ref == "" {
6687
slices.SortFunc(recs, func(a, b historyRecord) int {
6788
return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
@@ -154,7 +175,8 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
154175

155176
flags := cmd.Flags()
156177
flags.StringVarP(&options.output, "output", "o", "", "Output file path")
157-
flags.BoolVar(&options.all, "all", false, "Export all records for the builder")
178+
flags.BoolVar(&options.all, "all", false, "Export all build records for the builder")
179+
flags.BoolVar(&options.finalize, "finalize", false, "Ensure build records are finalized before exporting")
158180

159181
return cmd
160182
}

‎commands/history/trace.go‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
"github.com/docker/buildx/util/otelutil"
1818
"github.com/docker/buildx/util/otelutil/jaeger"
1919
"github.com/docker/cli/cli/command"
20-
controlapi "github.com/moby/buildkit/api/services/control"
2120
"github.com/opencontainers/go-digest"
2221
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
2322
"github.com/pkg/browser"
@@ -57,14 +56,7 @@ func loadTrace(ctx context.Context, ref string, nodes []builder.Node) (string, [
5756
// build is complete but no trace yet. try to finalize the trace
5857
time.Sleep(1 * time.Second) // give some extra time for last parts of trace to be written
5958

60-
c, err := rec.node.Driver.Client(ctx)
61-
if err != nil {
62-
return "", nil, err
63-
}
64-
_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
65-
Ref: rec.Ref,
66-
Finalize: true,
67-
})
59+
err := finalizeRecord(ctx, rec.Ref, []builder.Node{*rec.node})
6860
if err != nil {
6961
return "", nil, err
7062
}

‎commands/history/utils.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,28 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q
247247
return out, nil
248248
}
249249

250+
func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error {
251+
eg, ctx := errgroup.WithContext(ctx)
252+
for _, node := range nodes {
253+
node := node
254+
eg.Go(func() error {
255+
if node.Driver == nil {
256+
return nil
257+
}
258+
c, err := node.Driver.Client(ctx)
259+
if err != nil {
260+
return err
261+
}
262+
_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
263+
Ref: ref,
264+
Finalize: true,
265+
})
266+
return err
267+
})
268+
}
269+
return eg.Wait()
270+
}
271+
250272
func formatDuration(d time.Duration) string {
251273
if d < time.Minute {
252274
return fmt.Sprintf("%.1fs", d.Seconds())

‎docs/reference/buildx_history_export.md‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@ Export build records into Docker Desktop bundle
55

66
### Options
77

8-
| Name | Type | Default | Description |
9-
|:---------------------------------------|:---------|:--------|:-----------------------------------------|
10-
| [`--all`](#all) | `bool` | | Export all records for the builder |
11-
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
12-
| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging |
13-
| [`-o`](#output), [`--output`](#output) | `string` | | Output file path |
8+
| Name | Type | Default | Description |
9+
|:---------------------------------------|:---------|:--------|:----------------------------------------------------|
10+
| [`--all`](#all) | `bool` | | Export all build records for the builder |
11+
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
12+
| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging |
13+
| [`--finalize`](#finalize) | `bool` | | Ensure build records are finalized before exporting |
14+
| [`-o`](#output), [`--output`](#output) | `string` | | Output file path |
1415

1516

1617
<!---MARKER_GEN_END-->
@@ -49,6 +50,16 @@ docker buildx history export --builder builder0 ^1 -o builder0-build.dockerbuild
4950
docker buildx history export --debug qu2gsuo8ejqrwdfii23xkkckt -o debug-build.dockerbuild
5051
```
5152

53+
### <a name="finalize"></a> Ensure build records are finalized before exporting (--finalize)
54+
55+
Clients can report their own traces concurrently, and not all traces may be
56+
saved yet by the time of the export. Use the `--finalize` flag to ensure all
57+
traces are finalized before exporting.
58+
59+
```console
60+
docker buildx history export --finalize qu2gsuo8ejqrwdfii23xkkckt -o finalized-build.dockerbuild
61+
```
62+
5263
### <a name="output"></a> Export a single build to a custom file (--output)
5364

5465
```console

‎tests/history.go‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package tests
2+
3+
import (
4+
"encoding/json"
5+
"os"
6+
"path"
7+
"path/filepath"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
"github.com/moby/buildkit/util/testutil/integration"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
var historyTests = []func(t *testing.T, sb integration.Sandbox){
17+
testHistoryExport,
18+
testHistoryExportFinalize,
19+
testHistoryInspect,
20+
testHistoryLs,
21+
testHistoryRm,
22+
}
23+
24+
func testHistoryExport(t *testing.T, sb integration.Sandbox) {
25+
ref := buildTestProject(t, sb)
26+
require.NotEmpty(t, ref.Ref)
27+
28+
outFile := path.Join(t.TempDir(), "export.dockerbuild")
29+
cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile))
30+
out, err := cmd.Output()
31+
require.NoError(t, err, string(out))
32+
require.FileExists(t, outFile)
33+
}
34+
35+
func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) {
36+
ref := buildTestProject(t, sb)
37+
require.NotEmpty(t, ref.Ref)
38+
39+
outFile := path.Join(t.TempDir(), "export.dockerbuild")
40+
cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile))
41+
out, err := cmd.Output()
42+
require.NoError(t, err, string(out))
43+
require.FileExists(t, outFile)
44+
}
45+
46+
func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
47+
ref := buildTestProject(t, sb)
48+
require.NotEmpty(t, ref.Ref)
49+
50+
cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json"))
51+
out, err := cmd.Output()
52+
require.NoError(t, err, string(out))
53+
54+
type recT struct {
55+
Name string
56+
Ref string
57+
Context string
58+
Dockerfile string
59+
StartedAt *time.Time
60+
CompletedAt *time.Time
61+
Duration time.Duration
62+
Status string
63+
NumCompletedSteps int32
64+
NumTotalSteps int32
65+
NumCachedSteps int32
66+
}
67+
var rec recT
68+
err = json.Unmarshal(out, &rec)
69+
require.NoError(t, err)
70+
require.Equal(t, ref.Ref, rec.Ref)
71+
require.NotEmpty(t, rec.Name)
72+
}
73+
74+
func testHistoryLs(t *testing.T, sb integration.Sandbox) {
75+
ref := buildTestProject(t, sb)
76+
require.NotEmpty(t, ref.Ref)
77+
78+
cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json"))
79+
out, err := cmd.Output()
80+
require.NoError(t, err, string(out))
81+
82+
type recT struct {
83+
Ref string `json:"ref"`
84+
Name string `json:"name"`
85+
Status string `json:"status"`
86+
CreatedAt *time.Time `json:"created_at"`
87+
CompletedAt *time.Time `json:"completed_at"`
88+
TotalSteps int32 `json:"total_steps"`
89+
CompletedSteps int32 `json:"completed_steps"`
90+
CachedSteps int32 `json:"cached_steps"`
91+
}
92+
var rec recT
93+
err = json.Unmarshal(out, &rec)
94+
require.NoError(t, err)
95+
require.Equal(t, ref.String(), rec.Ref)
96+
require.NotEmpty(t, rec.Name)
97+
}
98+
99+
func testHistoryRm(t *testing.T, sb integration.Sandbox) {
100+
ref := buildTestProject(t, sb)
101+
require.NotEmpty(t, ref.Ref)
102+
103+
cmd := buildxCmd(sb, withArgs("history", "rm", ref.Ref))
104+
out, err := cmd.Output()
105+
require.NoError(t, err, string(out))
106+
}
107+
108+
type buildRef struct {
109+
Builder string
110+
Node string
111+
Ref string
112+
}
113+
114+
func (b buildRef) String() string {
115+
return b.Builder + "/" + b.Node + "/" + b.Ref
116+
}
117+
118+
func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef {
119+
dir := createTestProject(t)
120+
out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir))
121+
require.NoError(t, err, string(out))
122+
123+
dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
124+
require.NoError(t, err)
125+
126+
type mdT struct {
127+
BuildRef string `json:"buildx.build.ref"`
128+
}
129+
var md mdT
130+
err = json.Unmarshal(dt, &md)
131+
require.NoError(t, err)
132+
133+
refParts := strings.Split(md.BuildRef, "/")
134+
require.Len(t, refParts, 3)
135+
136+
return buildRef{
137+
Builder: refParts[0],
138+
Node: refParts[1],
139+
Ref: refParts[2],
140+
}
141+
}

‎tests/integration_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ func TestIntegration(t *testing.T) {
2424
tests = append(tests, commonTests...)
2525
tests = append(tests, buildTests...)
2626
tests = append(tests, bakeTests...)
27+
tests = append(tests, historyTests...)
2728
tests = append(tests, inspectTests...)
2829
tests = append(tests, lsTests...)
2930
tests = append(tests, imagetoolsTests...)

0 commit comments

Comments
 (0)