Skip to content

Commit 8f9d94e

Browse files
srenatussrenatus
committed
revision: add complete-bundle hash as input.bundle.hash
Computed on demand only, like the other hashes. This is for users that only need a stable revision, without combining existing sources' hashes, for example because there are too many of them to be convenient. Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>
1 parent 94df60a commit 8f9d94e

File tree

8 files changed

+310
-45
lines changed

8 files changed

+310
-45
lines changed

‎e2e/cli/build_revision_bundle_hash.txtar‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Test revision using input.bundle.hash
2+
exec $OPACTL build --config config.d/bundle.yml --data-dir tmp
3+
stderr '^1/1 bundles built and pushed successfully$'
4+
! stdout .
5+
6+
exec tar xf bundles/hello-world/bundle.tar.gz /.manifest
7+
cmp .manifest exp_manifest
8+
9+
-- config.d/bundle.yml --
10+
bundles:
11+
hello-world:
12+
object_storage:
13+
filesystem:
14+
path: bundles/hello-world/bundle.tar.gz
15+
requirements:
16+
- source: http-data
17+
revision: 'input.bundle.hash'
18+
sources:
19+
http-data:
20+
datasources:
21+
- type: http
22+
name: users
23+
path: users
24+
config:
25+
url: http://${HTTP_ENDPOINT}/users
26+
-- exp_manifest --
27+
{"revision":"0aa9261ac4e2c8c267a28b261acd98a17607b504bf2c9b51cddeb7dec7c750bb","roots":["users"],"rego_version":0}

‎internal/fs/hash.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import (
44
"crypto/sha256"
55
"encoding/hex"
66
"io"
7+
"io/fs"
78
"os"
8-
"path/filepath"
99
"sort"
1010
)
1111

@@ -14,18 +14,21 @@ import (
1414
// (null-terminated) followed by its contents to the hash. An empty directory produces
1515
// the hash of an empty input.
1616
func HashDirectory(root string) (string, error) {
17+
return HashFS(os.DirFS(root))
18+
}
19+
20+
// HashFS computes a deterministic SHA-256 hash over all files in an fs.FS.
21+
// Files are sorted by path, each contributing its path (null-terminated)
22+
// followed by its contents.
23+
func HashFS(fsys fs.FS) (string, error) {
1724
var files []string
1825

19-
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
26+
err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
2027
if err != nil {
2128
return err
2229
}
2330
if !d.IsDir() {
24-
relPath, err := filepath.Rel(root, path)
25-
if err != nil {
26-
return err
27-
}
28-
files = append(files, relPath)
31+
files = append(files, path)
2932
}
3033
return nil
3134
})
@@ -40,8 +43,7 @@ func HashDirectory(root string) (string, error) {
4043
h.Write([]byte(file))
4144
h.Write([]byte{0})
4245

43-
fullPath := filepath.Join(root, file)
44-
f, err := os.Open(fullPath)
46+
f, err := fsys.Open(file)
4547
if err != nil {
4648
return "", err
4749
}

‎internal/fs/hash_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"os"
55
"path/filepath"
66
"testing"
7+
"testing/fstest"
78
)
89

910
func TestHashDirectory(t *testing.T) {
@@ -118,3 +119,120 @@ func TestHashDirectory(t *testing.T) {
118119
}
119120
})
120121
}
122+
123+
func TestHashFS(t *testing.T) {
124+
t.Run("empty fs", func(t *testing.T) {
125+
fsys := fstest.MapFS{}
126+
127+
hash, err := HashFS(fsys)
128+
if err != nil {
129+
t.Fatalf("unexpected error: %v", err)
130+
}
131+
if hash != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
132+
t.Fatalf("unexpected hash for empty fs: %s", hash)
133+
}
134+
})
135+
136+
t.Run("single file", func(t *testing.T) {
137+
fsys := fstest.MapFS{
138+
"data.json": &fstest.MapFile{Data: []byte(`{"key":"value"}`)},
139+
}
140+
141+
hash, err := HashFS(fsys)
142+
if err != nil {
143+
t.Fatalf("unexpected error: %v", err)
144+
}
145+
if len(hash) != 64 {
146+
t.Fatalf("expected 64-char hex hash, got %d chars: %s", len(hash), hash)
147+
}
148+
})
149+
150+
t.Run("deterministic", func(t *testing.T) {
151+
fs1 := fstest.MapFS{
152+
"a.json": &fstest.MapFile{Data: []byte(`{"a":1}`)},
153+
"sub/b.json": &fstest.MapFile{Data: []byte(`{"b":2}`)},
154+
}
155+
fs2 := fstest.MapFS{
156+
"a.json": &fstest.MapFile{Data: []byte(`{"a":1}`)},
157+
"sub/b.json": &fstest.MapFile{Data: []byte(`{"b":2}`)},
158+
}
159+
160+
hash1, err := HashFS(fs1)
161+
if err != nil {
162+
t.Fatalf("unexpected error: %v", err)
163+
}
164+
hash2, err := HashFS(fs2)
165+
if err != nil {
166+
t.Fatalf("unexpected error: %v", err)
167+
}
168+
if hash1 != hash2 {
169+
t.Fatalf("expected identical hashes, got %s and %s", hash1, hash2)
170+
}
171+
})
172+
173+
t.Run("matches HashDirectory", func(t *testing.T) {
174+
dir := t.TempDir()
175+
if err := os.WriteFile(filepath.Join(dir, "data.json"), []byte(`{"key":"value"}`), 0644); err != nil {
176+
t.Fatal(err)
177+
}
178+
179+
dirHash, err := HashDirectory(dir)
180+
if err != nil {
181+
t.Fatalf("unexpected error: %v", err)
182+
}
183+
184+
fsys := fstest.MapFS{
185+
"data.json": &fstest.MapFile{Data: []byte(`{"key":"value"}`)},
186+
}
187+
fsHash, err := HashFS(fsys)
188+
if err != nil {
189+
t.Fatalf("unexpected error: %v", err)
190+
}
191+
192+
if dirHash != fsHash {
193+
t.Fatalf("HashDirectory and HashFS produced different hashes: %s vs %s", dirHash, fsHash)
194+
}
195+
})
196+
197+
t.Run("different content produces different hash", func(t *testing.T) {
198+
fs1 := fstest.MapFS{
199+
"data.json": &fstest.MapFile{Data: []byte(`{"a":1}`)},
200+
}
201+
fs2 := fstest.MapFS{
202+
"data.json": &fstest.MapFile{Data: []byte(`{"a":2}`)},
203+
}
204+
205+
hash1, err := HashFS(fs1)
206+
if err != nil {
207+
t.Fatalf("unexpected error: %v", err)
208+
}
209+
hash2, err := HashFS(fs2)
210+
if err != nil {
211+
t.Fatalf("unexpected error: %v", err)
212+
}
213+
if hash1 == hash2 {
214+
t.Fatal("expected different hashes for different content")
215+
}
216+
})
217+
218+
t.Run("different filenames produce different hash", func(t *testing.T) {
219+
fs1 := fstest.MapFS{
220+
"a.json": &fstest.MapFile{Data: []byte(`same`)},
221+
}
222+
fs2 := fstest.MapFS{
223+
"b.json": &fstest.MapFile{Data: []byte(`same`)},
224+
}
225+
226+
hash1, err := HashFS(fs1)
227+
if err != nil {
228+
t.Fatalf("unexpected error: %v", err)
229+
}
230+
hash2, err := HashFS(fs2)
231+
if err != nil {
232+
t.Fatalf("unexpected error: %v", err)
233+
}
234+
if hash1 == hash2 {
235+
t.Fatal("expected different hashes for different filenames")
236+
}
237+
})
238+
}

‎pkg/builder/builder.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ type Builder struct {
170170
target string
171171
optimizationLevel int
172172
revision string
173+
revisionFunc func(fs.FS) (string, error)
173174
}
174175

175176
func New() *Builder {
@@ -206,6 +207,13 @@ func (b *Builder) WithRevision(revision string) *Builder {
206207
return b
207208
}
208209

210+
// WithRevisionFunc sets a function to compute the revision from the assembled
211+
// bundle filesystem. It is called after FS assembly and before compilation.
212+
func (b *Builder) WithRevisionFunc(fn func(fs.FS) (string, error)) *Builder {
213+
b.revisionFunc = fn
214+
return b
215+
}
216+
209217
type PackageConflictErr struct {
210218
Requirement *Source
211219
Package *ast.Package
@@ -384,6 +392,14 @@ func (b *Builder) Build(ctx context.Context) error {
384392
fsBuild := mountfs.New(buildSources.fs())
385393
paths := slices.Collect(maps.Keys(fsBuild))
386394

395+
if b.revisionFunc != nil {
396+
revision, err := b.revisionFunc(fsBuild)
397+
if err != nil {
398+
return fmt.Errorf("revision: %w", err)
399+
}
400+
b.revision = revision
401+
}
402+
387403
target := cmp.Or(b.target, "rego")
388404
if target == "ir" { // fix naming convention
389405
target = "plan"

‎pkg/service/revision.go‎

Lines changed: 50 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,27 @@ type ReferencedSource struct {
1919
Fields []string // e.g., ["hashsum"], ["commit"]
2020
}
2121

22-
func ExtractRevisionRefs(revision string) ([]ReferencedSource, error) {
22+
func ExtractRevisionRefs(revision string) ([]ReferencedSource, bool, error) {
2323
if revision == "" {
24-
return nil, nil
24+
return nil, false, nil
2525
}
2626

2727
query, err := ast.ParseExpr(revision)
2828
if err != nil {
29-
return nil, fmt.Errorf("invalid rego query: %w", err)
29+
return nil, false, fmt.Errorf("invalid rego query: %w", err)
3030
}
3131

3232
sourcesRef := ast.InputRootRef.Append(ast.StringTerm("sources"))
33+
bundleRef := ast.InputRootRef.Append(ast.StringTerm("bundle"))
3334
references := make(map[string]map[string]bool)
35+
needsBundleHash := false
3436

3537
ast.WalkRefs(query, func(ref ast.Ref) bool {
38+
if ref.HasPrefix(bundleRef) {
39+
needsBundleHash = true
40+
return false
41+
}
42+
3643
if !ref.HasPrefix(sourcesRef) || len(ref) < 4 {
3744
return false
3845
}
@@ -64,33 +71,38 @@ func ExtractRevisionRefs(revision string) ([]ReferencedSource, error) {
6471
})
6572
}
6673

67-
return result, nil
74+
return result, needsBundleHash, nil
6875
}
6976

7077
// ResolveRevision evaluates a Rego revision expression with the given source metadata
7178
// and returns the final revision string.
72-
func ResolveRevision(ctx context.Context, revision string, sourceMetadata map[string]map[string]any) (string, error) {
79+
func ResolveRevision(ctx context.Context, revision string, sourceMetadata map[string]map[string]any, bundleHash string) (string, error) {
7380
if revision == "" {
7481
return "", nil
7582
}
7683

7784
input := map[string]any{
7885
"sources": sourceMetadata,
7986
}
87+
if bundleHash != "" {
88+
input["bundle"] = map[string]any{
89+
"hash": bundleHash,
90+
}
91+
}
8092

8193
query, err := ast.ParseExpr(revision)
8294
if err != nil {
8395
return "", fmt.Errorf("invalid rego query: %w", err)
8496
}
8597

86-
result, err := evaluateRego(ctx, query, input, sourceMetadata)
98+
result, err := evaluateRego(ctx, query, input, sourceMetadata, bundleHash)
8799
if err != nil {
88100
return "", fmt.Errorf("failed to resolve revision: %w", err)
89101
}
90102
return result, nil
91103
}
92104

93-
func buildInputSchema(sourceMetadata map[string]map[string]any) *ast.SchemaSet {
105+
func buildInputSchema(sourceMetadata map[string]map[string]any, bundleHash string) *ast.SchemaSet {
94106
sourceSchema := map[string]any{
95107
"type": "object",
96108
"properties": map[string]any{
@@ -127,21 +139,37 @@ func buildInputSchema(sourceMetadata map[string]map[string]any) *ast.SchemaSet {
127139
}
128140

129141
schemas := ast.NewSchemaSet()
130-
schemas.Put(ast.SchemaRootRef, map[string]any{
131-
"$schema": "http://json-schema.org/draft-07/schema",
132-
"type": "object",
133-
"properties": map[string]any{
134-
"sources": map[string]any{
135-
"type": "object",
136-
"properties": sourceProps,
137-
},
142+
143+
properties := map[string]any{
144+
"sources": map[string]any{
145+
"type": "object",
146+
"properties": sourceProps,
138147
},
139-
"required": []string{"sources"},
148+
}
149+
if bundleHash != "" {
150+
properties["bundle"] = map[string]any{
151+
"type": "object",
152+
"properties": map[string]any{
153+
"hash": map[string]any{"type": "string"},
154+
},
155+
}
156+
}
157+
158+
required := []string{"sources"}
159+
if bundleHash != "" {
160+
required = append(required, "bundle")
161+
}
162+
163+
schemas.Put(ast.SchemaRootRef, map[string]any{
164+
"$schema": "http://json-schema.org/draft-07/schema",
165+
"type": "object",
166+
"properties": properties,
167+
"required": required,
140168
})
141169
return schemas
142170
}
143171

144-
func evaluateRego(ctx context.Context, query *ast.Expr, input map[string]any, sourceMetadata map[string]map[string]any) (string, error) {
172+
func evaluateRego(ctx context.Context, query *ast.Expr, input map[string]any, sourceMetadata map[string]map[string]any, bundleHash string) (string, error) {
145173
opts := []func(*rego.Rego){
146174
rego.ParsedQuery([]*ast.Expr{query}),
147175
rego.Strict(true),
@@ -152,8 +180,11 @@ func evaluateRego(ctx context.Context, query *ast.Expr, input map[string]any, so
152180
opts = append(opts, rego.Input(input))
153181
}
154182

155-
if sourceMetadata != nil {
156-
opts = append(opts, rego.Schemas(buildInputSchema(sourceMetadata)))
183+
if sourceMetadata != nil || bundleHash != "" {
184+
if sourceMetadata == nil {
185+
sourceMetadata = make(map[string]map[string]any)
186+
}
187+
opts = append(opts, rego.Schemas(buildInputSchema(sourceMetadata, bundleHash)))
157188
}
158189

159190
rs, err := rego.New(opts...).Eval(ctx)

0 commit comments

Comments
 (0)