docs: add auto-generated sections to buildctl.md

Signed-off-by: Justin Chadwell <[email protected]>

Author: jedevc

Makefile

@@ -62,8 +62,12 @@ validate-generated-files:
 validate-doctoc:
 	$(BUILDX_CMD) bake validate-doctoc
 
+.PHONY: validate-docs
+validate-docs:
+	$(BUILDX_CMD) bake validate-docs
+
 .PHONY: validate-all
-validate-all: test lint validate-vendor validate-generated-files validate-doctoc
+validate-all: test lint validate-vendor validate-generated-files validate-doctoc validate-docs
 
 .PHONY: vendor
 vendor:
@@ -84,3 +88,7 @@ authors:
 .PHONY: doctoc
 doctoc:
 	$(BUILDX_CMD) bake doctoc
+
+.PHONY: docs
+docs:
+	$(BUILDX_CMD) bake docs

README.md

@@ -128,7 +128,7 @@ $ brew install buildkit
 
 To build BuildKit from source, see [`.github/CONTRIBUTING.md`](./.github/CONTRIBUTING.md).
 
-For a `buildctl` reference, see [this document](./docs/buildctl.md).
+For a `buildctl` reference, see [this document](./docs/reference/buildctl.md).
 
 ### Starting the `buildkitd` daemon
 

docker-bake.hcl

@@ -75,7 +75,7 @@ target "integration-tests-base" {
 }
 
 group "validate" {
-  targets = ["lint", "validate-vendor", "validate-doctoc", "validate-generated-files", "validate-shfmt"]
+  targets = ["lint", "validate-vendor", "validate-doctoc", "validate-generated-files", "validate-shfmt", "validate-docs"]
 }
 
 target "lint" {
@@ -119,6 +119,13 @@ target "validate-authors" {
   output = ["type=cacheonly"]
 }
 
+target "validate-docs" {
+  inherits = ["_common"]
+  dockerfile = "./hack/dockerfiles/docs.Dockerfile"
+  target = "validate"
+  output = ["type=cacheonly"]
+}
+
 target "vendor" {
   inherits = ["_common"]
   dockerfile = "./hack/dockerfiles/vendor.Dockerfile"
@@ -153,3 +160,10 @@ target "authors" {
   target = "update"
   output = ["."]
 }
+
+target "docs" {
+  inherits = ["_common"]
+  dockerfile = "./hack/dockerfiles/docs.Dockerfile"
+  target = "update"
+  output = ["./docs"]
+}

docs/generate.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+
+	"github.com/pkg/errors"
+)
+
+func main() {
+	re := regexp.MustCompile("(?s)<!---GENERATE_START (.*?)-->(.*?)<!---GENERATE_END-->\n")
+
+	err := filepath.Walk("./docs", func(path string, stat fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if stat.IsDir() {
+			return nil
+		}
+		if filepath.Ext(path) != ".md" {
+			return nil
+		}
+
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		dataNew := re.ReplaceAllFunc(data, func(match []byte) []byte {
+			groups := re.FindStringSubmatch(string(match))
+			stdout := bytes.NewBuffer(nil)
+			fmt.Fprintf(stdout, "<!---GENERATE_START %s-->\n", groups[1])
+			fmt.Fprintf(stdout, "```\n")
+			cmd := exec.Cmd{
+				Path:   "/bin/sh",
+				Args:   []string{"sh", "-c", groups[1]},
+				Stdout: stdout,
+			}
+			err = cmd.Start()
+			if err != nil {
+				err = errors.Wrapf(err, "could not start command %s", groups[1])
+				return nil
+			}
+			err = cmd.Wait()
+			if err != nil {
+				err = errors.Wrapf(err, "could not run command %s", groups[1])
+				return nil
+			}
+			fmt.Fprintf(stdout, "```\n")
+			fmt.Fprintf(stdout, "<!---GENERATE_END-->\n")
+
+			return stdout.Bytes()
+		})
+		if err != nil {
+			return err
+		}
+
+		if !bytes.Equal(data, dataNew) {
+			fmt.Println(path)
+			if err := os.WriteFile(path, dataNew, stat.Mode()); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

docs/buildctl.md renamed to docs/reference/buildctl.md

@@ -2,6 +2,7 @@
 
 `buildctl` is the command-line interface to `buildkitd`.
 
+<!---GENERATE_START buildctl --help-->
 ```
 NAME:
    buildctl - build utility
@@ -10,14 +11,15 @@ USAGE:
    buildctl [global options] command [command options] [arguments...]
 
 VERSION:
-   0.0.0+unknown
+   v0.0.0+unknown
 
 COMMANDS:
-   du        disk usage
-   prune     clean up build cache
-   build, b  build
-   debug     debug utilities
-   help, h   Shows a list of commands or help for one command
+   du               disk usage
+   prune            clean up build cache
+   prune-histories  clean up build histories
+   build, b         build
+   debug            debug utilities
+   help, h          Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
    --debug                enable debug output in logs
@@ -31,6 +33,7 @@ GLOBAL OPTIONS:
    --help, -h             show help
    --version, -v          print the version
 ```
+<!---GENERATE_END-->
 
 ## Connecting
 
@@ -45,9 +48,37 @@ Practically, that normally will be one of:
 
 Synopsis:
 
+<!---GENERATE_START buildctl build --help-->
 ```
-buildctl build --frontend dockerfile.v0 --opt target=foo --opt build-arg:foo=bar --local context=. --local dockerfile=. --output type=image,name=docker.io/username/image,push=true
-```
+NAME:
+   buildctl build - build
+
+USAGE:
+   
+  To build and push an image using Dockerfile:
+    $ buildctl build --frontend dockerfile.v0 --opt target=foo --opt build-arg:foo=bar --local context=. --local dockerfile=. --output type=image,name=docker.io/username/image,push=true
+  
+
+OPTIONS:
+   --output value, -o value    Define exports for build result, e.g. --output type=image,name=docker.io/username/image,push=true
+   --progress value            Set type of progress (auto, plain, tty). Use plain to show container output (default: "auto")
+   --trace value               Path to trace file. Defaults to no tracing.
+   --local value               Allow build access to the local directory
+   --oci-layout value          Allow build access to the local OCI layout
+   --frontend value            Define frontend used for build
+   --opt value                 Define custom options for frontend, e.g. --opt target=foo --opt build-arg:foo=bar
+   --no-cache                  Disable cache for all the vertices
+   --export-cache value        Export build cache, e.g. --export-cache type=registry,ref=example.com/foo/bar, or --export-cache type=local,dest=path/to/dir
+   --import-cache value        Import build cache, e.g. --import-cache type=registry,ref=example.com/foo/bar, or --import-cache type=local,src=path/to/dir
+   --secret value              Secret value exposed to the build. Format id=secretname,src=filepath
+   --allow value               Allow extra privileged entitlement, e.g. network.host, security.insecure
+   --ssh value                 Allow forwarding SSH agent to the builder. Format default|<id>[=<socket>|<key>[,<key>]]
+   --metadata-file value       Output build metadata (e.g., image digest) to a file as JSON
+   --source-policy-file value  Read source policy file from a JSON file
+   --ref-file value            Write build ref to a file
+   
+```
+<!---GENERATE_END-->
 
 `buildctl build` uses a buildkit daemon `buildkitd` to drive a build.
 

hack/dockerfiles/docs.Dockerfile

@@ -0,0 +1,54 @@
+# syntax=docker/dockerfile:1
+
+ARG GO_VERSION=1.20
+
+FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
+
+FROM golatest AS buildctl
+WORKDIR /src
+ENV CGO_ENABLED=0
+ARG TARGETPLATFORM
+RUN --mount=target=. \
+  --mount=target=/root/.cache,type=cache \
+  --mount=target=/go/pkg/mod,type=cache \
+  go build -mod=vendor -o /usr/bin/buildctl ./cmd/buildctl
+
+FROM golatest AS docsgen
+WORKDIR /src
+ENV CGO_ENABLED=0
+RUN --mount=target=. \
+  --mount=target=/root/.cache,type=cache \
+  --mount=target=/go/pkg/mod,type=cache \
+  go build -mod=vendor -o /out/docsgen ./docs/generate.go
+
+FROM alpine AS gen
+RUN apk add --no-cache rsync git
+WORKDIR /src
+COPY --from=docsgen /out/docsgen /usr/bin
+COPY --from=buildctl /usr/bin/buildctl /usr/bin/
+RUN --mount=target=/context \
+  --mount=target=.,type=tmpfs <<EOT
+set -e
+rsync -a /context/. .
+docsgen
+mkdir /out
+cp -r docs/* /out
+EOT
+
+FROM scratch AS update
+COPY --from=gen /out /
+
+FROM gen AS validate
+RUN --mount=target=/context \
+  --mount=target=.,type=tmpfs <<EOT
+set -e
+rsync -a /context/. .
+git add -A
+rm -rf docs/*
+cp -rf /out/* ./docs/
+if [ -n "$(git status --porcelain -- docs/)" ]; then
+  echo >&2 'ERROR: Docs result differs. Please update with "make docs"'
+  git status --porcelain -- docs/
+  exit 1
+fi
+EOT