Skip to content

Commit fbf7ca2

Browse files
NaatanNaatan
authored
Merge pull request ActiveState#3575 from ActiveState/DX-3105
Fixes and tests for ingredientcall end-to-end
2 parents 3ef1501 + 41062d2 commit fbf7ca2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+4437
-194
lines changed

‎cmd/state-svc/internal/hash/file_hasher.go‎

Lines changed: 36 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"time"
1010

1111
"github.com/ActiveState/cli/internal/errs"
12-
"github.com/ActiveState/cli/internal/rtutils"
12+
"github.com/bmatcuk/doublestar/v4"
1313
"github.com/cespare/xxhash"
1414
"github.com/patrickmn/go-cache"
1515
)
@@ -36,64 +36,75 @@ func NewFileHasher() *FileHasher {
3636
}
3737

3838
func (fh *FileHasher) HashFiles(wd string, globs []string) (_ string, _ []hashedFile, rerr error) {
39-
sort.Strings(globs) // ensure consistent ordering
39+
fs := os.DirFS(wd)
4040
hashedFiles := []hashedFile{}
41-
hasher := xxhash.New()
41+
hashes := []string{}
4242
for _, glob := range globs {
43-
files, err := filepath.Glob(glob)
43+
files, err := doublestar.Glob(fs, glob)
4444
if err != nil {
4545
return "", nil, errs.Wrap(err, "Could not match glob: %s", glob)
4646
}
4747
sort.Strings(files) // ensure consistent ordering
48-
for _, f := range files {
49-
if !filepath.IsAbs(f) {
50-
af, err := filepath.Abs(filepath.Join(wd, f))
51-
if err != nil {
52-
return "", nil, errs.Wrap(err, "Could not get absolute path for file: %s", f)
53-
}
54-
f = af
48+
for _, relativePath := range files {
49+
absolutePath, err := filepath.Abs(filepath.Join(wd, relativePath))
50+
if err != nil {
51+
return "", nil, errs.Wrap(err, "Could not get absolute path for file: %s", relativePath)
5552
}
56-
file, err := os.Open(f)
53+
fileInfo, err := os.Stat(absolutePath)
5754
if err != nil {
58-
return "", nil, errs.Wrap(err, "Could not open file: %s", file.Name())
55+
return "", nil, errs.Wrap(err, "Could not stat file: %s", absolutePath)
5956
}
60-
defer rtutils.Closer(file.Close, &rerr)
6157

62-
fileInfo, err := file.Stat()
63-
if err != nil {
64-
return "", nil, errs.Wrap(err, "Could not stat file: %s", file.Name())
58+
if fileInfo.IsDir() {
59+
continue
6560
}
6661

6762
var hash string
68-
cachedHash, ok := fh.cache.Get(cacheKey(file.Name(), fileInfo.ModTime()))
63+
cachedHash, ok := fh.cache.Get(cacheKey(fileInfo.Name(), fileInfo.ModTime()))
6964
if ok {
7065
hash, ok = cachedHash.(string)
7166
if !ok {
7267
return "", nil, errs.New("Could not convert cache value to string")
7368
}
7469
} else {
7570
fileHasher := xxhash.New()
71+
// include filepath in hash, because moving files should affect the hash
72+
fmt.Fprintf(fileHasher, "%016x", relativePath)
73+
file, err := os.Open(absolutePath)
74+
if err != nil {
75+
return "", nil, errs.Wrap(err, "Could not open file: %s", absolutePath)
76+
}
77+
defer file.Close()
7678
if _, err := io.Copy(fileHasher, file); err != nil {
77-
return "", nil, errs.Wrap(err, "Could not hash file: %s", file.Name())
79+
return "", nil, errs.Wrap(err, "Could not hash file: %s", fileInfo.Name())
7880
}
7981

8082
hash = fmt.Sprintf("%016x", fileHasher.Sum64())
8183
}
8284

83-
fh.cache.Set(cacheKey(file.Name(), fileInfo.ModTime()), hash, cache.NoExpiration)
85+
fh.cache.Set(cacheKey(fileInfo.Name(), fileInfo.ModTime()), hash, cache.NoExpiration)
8486

87+
hashes = append(hashes, hash)
8588
hashedFiles = append(hashedFiles, hashedFile{
8689
Pattern: glob,
87-
Path: file.Name(),
90+
Path: relativePath,
8891
Hash: hash,
8992
})
90-
91-
// Incorporate the individual file hash into the overall hash in hex format
92-
fmt.Fprintf(hasher, "%016x", hash)
9393
}
9494
}
9595

96-
return fmt.Sprintf("%016x", hasher.Sum64()), hashedFiles, nil
96+
if hashedFiles == nil {
97+
return "", nil, nil
98+
}
99+
100+
// Ensure the overall hash is consistently calculated
101+
sort.Slice(hashedFiles, func(i, j int) bool { return hashedFiles[i].Path < hashedFiles[j].Path })
102+
h := xxhash.New()
103+
for _, f := range hashedFiles {
104+
fmt.Fprintf(h, "%016x", f.Hash)
105+
}
106+
107+
return fmt.Sprintf("%016x", h.Sum64()), hashedFiles, nil
97108
}
98109

99110
func cacheKey(file string, modTime time.Time) string {

‎cmd/state-svc/internal/hash/file_hasher_test.go‎

Lines changed: 70 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
package hash
22

33
import (
4+
"fmt"
45
"os"
6+
"path/filepath"
7+
"sort"
8+
"strings"
59
"testing"
610
"time"
711

12+
"github.com/ActiveState/cli/internal/errs"
13+
"github.com/ActiveState/cli/internal/fileutils"
14+
"github.com/ActiveState/cli/internal/osutils"
815
"github.com/patrickmn/go-cache"
916
"github.com/stretchr/testify/assert"
17+
"github.com/stretchr/testify/require"
1018
)
1119

1220
type testCache struct {
@@ -31,23 +39,39 @@ func (tc *testCache) Set(key string, value interface{}, expiration time.Duration
3139
}
3240

3341
func TestFileHasher_HashFiles(t *testing.T) {
34-
file1 := createTempFile(t, "file1")
35-
file2 := createTempFile(t, "file2")
42+
dir := fileutils.TempDirUnsafe()
43+
file1 := createTempFile(t, dir, "file1.txt")
44+
file2 := createTempFile(t, dir, "file2.info")
45+
subfile1 := createTempFile(t, dir, "dir1/subfile1.txt")
3646

3747
hasher := NewFileHasher()
3848

39-
hash1, err := hasher.HashFiles([]string{file1, file2})
40-
assert.NoError(t, err)
49+
hash1, files1, err := hasher.HashFiles(dir, []string{file1, file2, subfile1})
50+
require.NoError(t, err)
4151

42-
hash2, err := hasher.HashFiles([]string{file1, file2})
43-
assert.NoError(t, err)
52+
hash2, files2, err := hasher.HashFiles(dir, []string{"./**/*"})
53+
require.NoError(t, err, errs.JoinMessage(err))
54+
55+
for _, f := range files1 {
56+
assert.False(t, strings.HasPrefix(f.Path, dir), fmt.Sprintf("'%s' should not be prefixed with '%s'", f.Path, dir))
57+
}
58+
59+
sort.Slice(files1, func(i, j int) bool { return files1[i].Path < files1[j].Path })
60+
sort.Slice(files2, func(i, j int) bool { return files2[i].Path < files2[j].Path })
61+
require.Len(t, files2, 3)
62+
require.Len(t, files2, len(files1))
63+
64+
for i, f := range files1 {
65+
assert.Equal(t, f.Path, files2[i].Path)
66+
assert.Equal(t, f.Hash, files2[i].Hash)
67+
}
4468

4569
assert.Equal(t, hash1, hash2)
4670
}
4771

4872
func TestFileHasher_CacheHit(t *testing.T) {
49-
file1 := createTempFile(t, "file1")
50-
file2 := createTempFile(t, "file2")
73+
file1 := createTempFile(t, "", "file1")
74+
file2 := createTempFile(t, "", "file2")
5175

5276
tc := &testCache{
5377
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -57,10 +81,10 @@ func TestFileHasher_CacheHit(t *testing.T) {
5781
cache: tc,
5882
}
5983

60-
hash1, err := hasher.HashFiles([]string{file1, file2})
84+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
6185
assert.NoError(t, err)
6286

63-
hash2, err := hasher.HashFiles([]string{file1, file2})
87+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
6488
assert.NoError(t, err)
6589

6690
assert.Equal(t, hash1, hash2)
@@ -69,8 +93,8 @@ func TestFileHasher_CacheHit(t *testing.T) {
6993
}
7094

7195
func TestFileHasher_CacheMiss(t *testing.T) {
72-
file1 := createTempFile(t, "file1")
73-
file2 := createTempFile(t, "file2")
96+
file1 := createTempFile(t, "", "file1")
97+
file2 := createTempFile(t, "", "file2")
7498

7599
tc := &testCache{
76100
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -80,7 +104,7 @@ func TestFileHasher_CacheMiss(t *testing.T) {
80104
cache: tc,
81105
}
82106

83-
hash1, err := hasher.HashFiles([]string{file1, file2})
107+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
84108
assert.NoError(t, err)
85109

86110
if err := os.Chtimes(file1, time.Now(), time.Now()); err != nil {
@@ -92,7 +116,7 @@ func TestFileHasher_CacheMiss(t *testing.T) {
92116
err = file.Sync()
93117
assert.NoError(t, err)
94118

95-
hash2, err := hasher.HashFiles([]string{file1, file2})
119+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
96120
assert.NoError(t, err)
97121

98122
assert.Equal(t, hash1, hash2)
@@ -102,11 +126,11 @@ func TestFileHasher_CacheMiss(t *testing.T) {
102126

103127
func TestFileHasher_ContentAgnostic(t *testing.T) {
104128
// Files have same content but different names and modification times
105-
file1 := createTempFile(t, "file1")
129+
file1 := createTempFile(t, "", "file1")
106130

107131
// Ensure mod times are different
108132
time.Sleep(1 * time.Millisecond)
109-
file2 := createTempFile(t, "file1")
133+
file2 := createTempFile(t, "", "file1")
110134

111135
tc := &testCache{
112136
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -116,10 +140,10 @@ func TestFileHasher_ContentAgnostic(t *testing.T) {
116140
cache: tc,
117141
}
118142

119-
hash1, err := hasher.HashFiles([]string{file1, file2})
143+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
120144
assert.NoError(t, err)
121145

122-
hash2, err := hasher.HashFiles([]string{file1, file2})
146+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
123147
assert.NoError(t, err)
124148

125149
assert.Equal(t, hash1, hash2)
@@ -128,9 +152,9 @@ func TestFileHasher_ContentAgnostic(t *testing.T) {
128152
}
129153

130154
func TestFileHasher_NotEqualFileAdded(t *testing.T) {
131-
file1 := createTempFile(t, "file1")
132-
file2 := createTempFile(t, "file2")
133-
file3 := createTempFile(t, "file3")
155+
file1 := createTempFile(t, "", "file1")
156+
file2 := createTempFile(t, "", "file2")
157+
file3 := createTempFile(t, "", "file3")
134158

135159
tc := &testCache{
136160
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -140,10 +164,10 @@ func TestFileHasher_NotEqualFileAdded(t *testing.T) {
140164
cache: tc,
141165
}
142166

143-
hash1, err := hasher.HashFiles([]string{file1, file2})
167+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
144168
assert.NoError(t, err)
145169

146-
hash2, err := hasher.HashFiles([]string{file1, file2, file3})
170+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2, file3})
147171
assert.NoError(t, err)
148172

149173
assert.NotEqual(t, hash1, hash2)
@@ -152,9 +176,9 @@ func TestFileHasher_NotEqualFileAdded(t *testing.T) {
152176
}
153177

154178
func TestFileHasher_NotEqualFileRemoved(t *testing.T) {
155-
file1 := createTempFile(t, "file1")
156-
file2 := createTempFile(t, "file2")
157-
file3 := createTempFile(t, "file3")
179+
file1 := createTempFile(t, "", "file1")
180+
file2 := createTempFile(t, "", "file2")
181+
file3 := createTempFile(t, "", "file3")
158182

159183
tc := &testCache{
160184
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -164,10 +188,10 @@ func TestFileHasher_NotEqualFileRemoved(t *testing.T) {
164188
cache: tc,
165189
}
166190

167-
hash1, err := hasher.HashFiles([]string{file1, file2, file3})
191+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2, file3})
168192
assert.NoError(t, err)
169193

170-
hash2, err := hasher.HashFiles([]string{file1, file2})
194+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
171195
assert.NoError(t, err)
172196

173197
assert.NotEqual(t, hash1, hash2)
@@ -176,8 +200,8 @@ func TestFileHasher_NotEqualFileRemoved(t *testing.T) {
176200
}
177201

178202
func TestFileHasher_NotEqualContentChanged(t *testing.T) {
179-
file1 := createTempFile(t, "file1")
180-
file2 := createTempFile(t, "file2")
203+
file1 := createTempFile(t, "", "file1")
204+
file2 := createTempFile(t, "", "file2")
181205

182206
tc := &testCache{
183207
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -187,10 +211,10 @@ func TestFileHasher_NotEqualContentChanged(t *testing.T) {
187211
cache: tc,
188212
}
189213

190-
hash1, err := hasher.HashFiles([]string{file1, file2})
214+
hash1, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
191215
assert.NoError(t, err)
192216

193-
hash2, err := hasher.HashFiles([]string{file1, file2})
217+
hash2, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
194218
assert.NoError(t, err)
195219

196220
assert.Equal(t, hash1, hash2)
@@ -203,26 +227,28 @@ func TestFileHasher_NotEqualContentChanged(t *testing.T) {
203227
t.Fatal(err)
204228
}
205229

206-
hash2Modified, err := hasher.HashFiles([]string{file1, file2})
230+
hash2Modified, _, err := hasher.HashFiles(osutils.GetwdUnsafe(), []string{file1, file2})
207231
assert.NoError(t, err)
208232

209233
assert.NotEqual(t, hash1, hash2Modified)
210234
assert.Len(t, tc.hits, 3)
211235
assert.Len(t, tc.misses, 3)
212236
}
213237

214-
func createTempFile(t *testing.T, content string) string {
215-
tmpfile, err := os.CreateTemp("", "testfile")
216-
if err != nil {
217-
t.Fatal(err)
218-
}
219-
220-
if _, err := tmpfile.Write([]byte(content)); err != nil {
221-
t.Fatal(err)
238+
func createTempFile(t *testing.T, dir, path string) string {
239+
if dir == "" {
240+
dir = t.TempDir()
222241
}
223-
if err := tmpfile.Close(); err != nil {
224-
t.Fatal(err)
242+
if path == "" {
243+
tmpfile, err := os.CreateTemp(dir, "")
244+
if err != nil {
245+
t.Fatal(err)
246+
}
247+
path = tmpfile.Name()
248+
tmpfile.Close()
225249
}
250+
err := fileutils.WriteFile(filepath.Join(dir, path), []byte(path)) // Contents aren't important so long as they're consistent
251+
require.NoError(t, err, errs.JoinMessage(err))
226252

227-
return tmpfile.Name()
253+
return path
228254
}

‎go.mod‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,14 @@ require (
7070

7171
require (
7272
github.com/ActiveState/graphql v0.0.0-20230719154233-6949037a6e48
73+
github.com/bmatcuk/doublestar/v4 v4.7.1
7374
github.com/brunoga/deep v1.2.4
7475
github.com/cespare/xxhash v1.1.0
7576
github.com/charmbracelet/bubbles v0.18.0
7677
github.com/charmbracelet/bubbletea v0.25.0
7778
github.com/charmbracelet/lipgloss v0.9.1
7879
github.com/go-git/go-git/v5 v5.12.0
80+
github.com/gowebpki/jcs v1.0.1
7981
github.com/klauspost/compress v1.11.4
8082
github.com/mholt/archiver/v3 v3.5.1
8183
github.com/zijiren233/yaml-comment v0.2.1
@@ -143,7 +145,7 @@ require (
143145
github.com/josharian/intern v1.0.0 // indirect
144146
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
145147
github.com/kevinburke/ssh_config v1.2.0 // indirect
146-
github.com/labstack/gommon v0.3.1 // indirect
148+
github.com/labstack/gommon v0.3.1
147149
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
148150
github.com/mailru/easyjson v0.7.7 // indirect
149151
github.com/matryer/is v1.2.0 // indirect

0 commit comments

Comments
 (0)