Skip to content

buildflags: handle unknown values from cty #2965

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tonistiigi merged 1 commit into from
Feb 6, 2025
+88 −43
Merged

buildflags: handle unknown values from cty #2965

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion bake/hcl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package bake

import (
"reflect"
"regexp"
"testing"

hcl "github.com/hashicorp/hcl/v2"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -647,7 +649,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
require.Equal(t, []string{"default", "key=path/to/key"}, stringify(c.Targets[0].SSH))
}

func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
func TestHCLAttrsCapsuleType_ObjectVars(t *testing.T) {
dt := []byte(`
variable "foo" {
default = "bar"
Expand Down Expand Up @@ -716,6 +718,52 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
require.Equal(t, []string{"id=oci,src=/local/secret"}, stringify(web.Secrets))
}

func TestHCLAttrsCapsuleType_MissingVars(t *testing.T) {
dt := []byte(`
target "app" {
attest = [
"type=sbom,disabled=${SBOM}",
]

cache-from = [
{ type = "registry", ref = "user/app:${FOO1}" },
"type=local,src=path/to/cache:${FOO2}",
]

cache-to = [
{ type = "local", dest = "path/to/${BAR}" },
]

output = [
{ type = "oci", dest = "../${OUTPUT}.tar" },
]

secret = [
{ id = "mysecret", src = "/local/${SECRET}" },
]

ssh = [
{ id = "key", paths = ["path/to/${SSH_KEY}"] },
]
}
Comment on lines +723 to +748
Copy link
Member

@crazy-max crazy-max Feb 4, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was looking at how this example behaves in Buildx 0.19 and I got 13 diagnostics:

docker buildx bake --print
#1 [internal] load local bake definitions
#1 reading docker-bake.hcl 451B / 451B done
#1 DONE 0.0s
docker-bake.hcl:6
--------------------
   4 |                  ]
   5 |                  cache-from = [
   6 | >>>                      { type = "registry", ref = "user/app:${FOO1}" },
   7 |           "type=local,src=path/to/cache:${FOO2}",
   8 |                  ]
--------------------
ERROR: docker-bake.hcl:6,43-47: Unknown variable; There is no variable named "FOO1"., and 12 other diagnostic(s)

But here I got 8 diagnostics:

docker buildx bake --print
#1 [internal] load local bake definitions
#1 reading docker-bake.hcl 451B / 451B done
#1 DONE 0.0s
docker-bake.hcl:13
--------------------
  11 |                  ]
  12 |                  output = [
  13 | >>>                      { type = "oci", dest = "../${OUTPUT}.tar" },
  14 |                  ]
  15 |                  secret = [
--------------------
ERROR: docker-bake.hcl:13,33-39: Unknown variable; There is no variable named "OUTPUT"., and 7 other diagnostic(s)

Not for this PR but wonder if we could show all diags when --debug is set?

nit: Also seems to return OUTPUT as first diag with this PR but in older release returns what seems to be the first one FOO1

Copy link
Collaborator Author

@jsternberg jsternberg Feb 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd definitely say we should show all diagnostics. The number of diagnostics can be a bit strange sometimes because they will sometimes depend on the ordering of evaluation and one diagnostic can trigger another diagnostic. There will also be some diagnostics sometimes that say "cannot use unknown value" that are triggered somewhere.

`)

var diags hcl.Diagnostics
_, err := ParseFile(dt, "docker-bake.hcl")
require.ErrorAs(t, err, &diags)

re := regexp.MustCompile(`There is no variable named "([\w\d_]+)"`)
var actual []string
for _, diag := range diags {
if m := re.FindStringSubmatch(diag.Error()); m != nil {
actual = append(actual, m[1])
}
}
require.ElementsMatch(t,
[]string{"SBOM", "FOO1", "FOO2", "BAR", "OUTPUT", "SECRET", "SSH_KEY"},
actual)
}

func TestHCLMultiFileAttrs(t *testing.T) {
dt := []byte(`
variable "FOO" {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/docker/buildx

go 1.22.0
go 1.23.0
Copy link
Member

@thaJeztah thaJeztah Feb 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this intentional; i.e., go1.22 no longer supported with this change?

Copy link
Collaborator Author

@jsternberg jsternberg Feb 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was intentional to be able to use rangefunc. After an offline conversation, I was informed that buildx has some downstream dependencies that haven't moved to go 1.23 yet and we generally try to make our go.mod files compatible with stable - 1 go versions so we can introduce this usage when go 1.24 releases.

I've opened #2978 to keep the bones of this functionality but to revert the compiler usage of the feature.


require (
github.com/Masterminds/semver/v3 v3.2.1
Expand Down
2 changes: 1 addition & 1 deletion hack/dockerfiles/lint.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ ARG XX_VERSION=1.6.1
ARG GOLANGCI_LINT_VERSION=1.62.0
ARG GOPLS_VERSION=v0.26.0
# disabled: deprecated unusedvariable simplifyrange
ARG GOPLS_ANALYZERS="embeddirective fillreturns infertypeargs nonewvars norangeoverfunc noresultvalues simplifycompositelit simplifyslice undeclaredname unusedparams useany"
ARG GOPLS_ANALYZERS="embeddirective fillreturns infertypeargs nonewvars noresultvalues simplifycompositelit simplifyslice undeclaredname unusedparams useany"

FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx

Expand Down
8 changes: 5 additions & 3 deletions util/buildflags/attests_cty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ func (e *Attests) FromCtyValue(in cty.Value, p cty.Path) error {

func (e *Attests) fromCtyValue(in cty.Value, p cty.Path) error {
*e = make([]*Attest, 0, in.LengthInt())
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()

for value := range eachElement(in) {
entry := &Attest{}
if err := entry.FromCtyValue(value, p); err != nil {
return err
Expand Down Expand Up @@ -64,6 +62,10 @@ func (e *Attest) FromCtyValue(in cty.Value, p cty.Path) error {
e.Attrs = map[string]string{}
for it := conv.ElementIterator(); it.Next(); {
k, v := it.Element()
if !v.IsKnown() {
continue
}

switch key := k.AsString(); key {
case "type":
e.Type = v.AsString()
Expand Down
8 changes: 1 addition & 7 deletions util/buildflags/cache_cty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,7 @@ func (o *CacheOptions) FromCtyValue(in cty.Value, p cty.Path) error {

func (o *CacheOptions) fromCtyValue(in cty.Value, p cty.Path) error {
*o = make([]*CacheOptionsEntry, 0, in.LengthInt())
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()

if isEmpty(value) {
continue
}

for value := range eachElement(in) {
// Special handling for a string type to handle ref only format.
if value.Type() == cty.String {
entries, err := ParseCacheEntry([]string{value.AsString()})
Expand Down
8 changes: 1 addition & 7 deletions util/buildflags/export_cty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,7 @@ func (e *Exports) FromCtyValue(in cty.Value, p cty.Path) error {

func (e *Exports) fromCtyValue(in cty.Value, p cty.Path) error {
*e = make([]*ExportEntry, 0, in.LengthInt())
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()

if isEmpty(value) {
continue
}

for value := range eachElement(in) {
entry := &ExportEntry{}
if err := entry.FromCtyValue(value, p); err != nil {
return err
Expand Down
14 changes: 4 additions & 10 deletions util/buildflags/secrets_cty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,7 @@ func (s *Secrets) FromCtyValue(in cty.Value, p cty.Path) error {

func (s *Secrets) fromCtyValue(in cty.Value, p cty.Path) error {
*s = make([]*Secret, 0, in.LengthInt())
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()

if isEmpty(value) {
continue
}

for value := range eachElement(in) {
entry := &Secret{}
if err := entry.FromCtyValue(value, p); err != nil {
return err
Expand Down Expand Up @@ -71,13 +65,13 @@ func (e *Secret) FromCtyValue(in cty.Value, p cty.Path) error {
return err
}

if id := conv.GetAttr("id"); !id.IsNull() {
if id := conv.GetAttr("id"); !id.IsNull() && id.IsKnown() {
e.ID = id.AsString()
}
if src := conv.GetAttr("src"); !src.IsNull() {
if src := conv.GetAttr("src"); !src.IsNull() && src.IsKnown() {
e.FilePath = src.AsString()
}
if env := conv.GetAttr("env"); !env.IsNull() {
if env := conv.GetAttr("env"); !env.IsNull() && env.IsKnown() {
e.Env = env.AsString()
}
return nil
Expand Down
12 changes: 3 additions & 9 deletions util/buildflags/ssh_cty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,7 @@ func (s *SSHKeys) FromCtyValue(in cty.Value, p cty.Path) error {

func (s *SSHKeys) fromCtyValue(in cty.Value, p cty.Path) error {
*s = make([]*SSH, 0, in.LengthInt())
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()

if isEmpty(value) {
continue
}

for value := range eachElement(in) {
entry := &SSH{}
if err := entry.FromCtyValue(value, p); err != nil {
return err
Expand Down Expand Up @@ -71,10 +65,10 @@ func (e *SSH) FromCtyValue(in cty.Value, p cty.Path) error {
return err
}

if id := conv.GetAttr("id"); !id.IsNull() {
if id := conv.GetAttr("id"); !id.IsNull() && id.IsKnown() {
e.ID = id.AsString()
}
if paths := conv.GetAttr("paths"); !paths.IsNull() {
if paths := conv.GetAttr("paths"); !paths.IsNull() && paths.IsKnown() {
if err := gocty.FromCtyValue(paths, &e.Paths); err != nil {
return err
}
Expand Down
27 changes: 23 additions & 4 deletions util/buildflags/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package buildflags

import (
"iter"

"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
)
Expand Down Expand Up @@ -34,7 +36,7 @@ func removeDupes[E comparable[E]](s []E) []E {
}

func getAndDelete(m map[string]cty.Value, attr string, gv interface{}) error {
if v, ok := m[attr]; ok {
if v, ok := m[attr]; ok && v.IsKnown() {
delete(m, attr)
return gocty.FromCtyValue(v, gv)
}
Expand All @@ -44,11 +46,28 @@ func getAndDelete(m map[string]cty.Value, attr string, gv interface{}) error {
func asMap(m map[string]cty.Value) map[string]string {
out := make(map[string]string, len(m))
for k, v := range m {
out[k] = v.AsString()
if v.IsKnown() {
out[k] = v.AsString()
}
}
return out
}

func isEmpty(v cty.Value) bool {
return v.Type() == cty.String && v.AsString() == ""
func isEmptyOrUnknown(v cty.Value) bool {
return !v.IsKnown() || (v.Type() == cty.String && v.AsString() == "")
}

func eachElement(in cty.Value) iter.Seq[cty.Value] {
return func(yield func(v cty.Value) bool) {
for elem := in.ElementIterator(); elem.Next(); {
_, value := elem.Element()
if isEmptyOrUnknown(value) {
continue
}

if !yield(value) {
return
}
}
}
}