Skip to content

Commit 7097fd8

Browse files
kommendorkaptenjoshuaglethan-lowman-dd
authored
feat: Adds a new raw file metadata storage for clients (#347)
* Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman <[email protected]> * Ignore emacs temporary files Signed-off-by: Fredrik Skogman <[email protected]> * moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman <[email protected]> * Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman <[email protected]> * Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman <[email protected]> * Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman <[email protected]> * More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman <[email protected]> * Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman <[email protected]> * Spelling error. Signed-off-by: Fredrik Skogman <[email protected]> * Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman <[email protected]> * Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman <[email protected]> * Ignore emacs temporary files Signed-off-by: Fredrik Skogman <[email protected]> * moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman <[email protected]> * Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman <[email protected]> * Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman <[email protected]> * Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman <[email protected]> * More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman <[email protected]> * Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman <[email protected]> * Spelling error. Signed-off-by: Fredrik Skogman <[email protected]> * Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman <[email protected]> * Disabled filesystem permission checks for windows. Windows filesystem permission is a bit different from UNIX like systems, and there is no good API in go to manipulate it. Nor is having these checks a requirement by the TUF spec. This commit relies on build tags to inject the correct filesystem permssion check, the Windows version always return "satisfied". Signed-off-by: Fredrik Skogman <[email protected]> * Moved the tests that rely on permission bits to a new test file. The permission bits are not executed during windows builds. Signed-off-by: Fredrik Skogman <[email protected]> * Clarify permissions check naming and comment, add tests Signed-off-by: Ethan Lowman <[email protected]> * Update internal/fsutil/fsutil.go Co-authored-by: Ethan Lowman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> * Updates based on feeback. Mostly minor things like removing sentinel errors, wrapping errors from std library. Signed-off-by: Fredrik Skogman <[email protected]> * Use fs.ErrNotExist instead of os.ErrNotExist as recommended by Go docs Signed-off-by: Ethan Lowman <[email protected]> * Clean up remaining unnecessary usage of filepath.FromSlash Signed-off-by: Ethan Lowman <[email protected]> * Add missing error checks in tests Signed-off-by: Ethan Lowman <[email protected]> * Make sure to test that returned err is nil in the tests. Signed-off-by: Fredrik Skogman <[email protected]> Signed-off-by: Fredrik Skogman <[email protected]> Signed-off-by: Ethan Lowman <[email protected]> Co-authored-by: Joshua Lock <[email protected]> Co-authored-by: Ethan Lowman <[email protected]> Co-authored-by: Ethan Lowman <[email protected]>
1 parent a9ddd89 commit 7097fd8

File tree

9 files changed

+541
-15
lines changed

9 files changed

+541
-15
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
cmd/tuf/tuf
33
cmd/tuf-client/tuf-client
44
.vscode
5+
*~
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package client
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"fmt"
7+
"io/fs"
8+
"os"
9+
"path/filepath"
10+
"sync"
11+
12+
"github.com/theupdateframework/go-tuf/client"
13+
"github.com/theupdateframework/go-tuf/internal/fsutil"
14+
"github.com/theupdateframework/go-tuf/util"
15+
)
16+
17+
const (
18+
// user: rwx
19+
// group: r-x
20+
// other: ---
21+
dirCreateMode = os.FileMode(0750)
22+
// user: rw-
23+
// group: r--
24+
// other: ---
25+
fileCreateMode = os.FileMode(0640)
26+
)
27+
28+
// FileJSONStore represents a local metadata cache relying on raw JSON files
29+
// as retrieved from the remote repository.
30+
type FileJSONStore struct {
31+
mtx sync.RWMutex
32+
baseDir string
33+
}
34+
35+
var _ client.LocalStore = (*FileJSONStore)(nil)
36+
37+
// NewFileJSONStore returns a new metadata cache, implemented using raw JSON
38+
// files, stored in a directory provided by the client.
39+
// If the provided directory does not exist on disk, it will be created.
40+
// The provided metadata cache is safe for concurrent access.
41+
func NewFileJSONStore(baseDir string) (*FileJSONStore, error) {
42+
f := &FileJSONStore{
43+
baseDir: baseDir,
44+
}
45+
46+
// Does the directory exist?
47+
fi, err := os.Stat(baseDir)
48+
if err != nil {
49+
if errors.Is(err, fs.ErrNotExist) {
50+
// Create the directory
51+
if err = os.MkdirAll(baseDir, dirCreateMode); err != nil {
52+
return nil, fmt.Errorf("error creating directory for metadata cache: %w", err)
53+
}
54+
} else {
55+
return nil, fmt.Errorf("error getting FileInfo for %s: %w", baseDir, err)
56+
}
57+
} else {
58+
// Verify that it is a directory
59+
if !fi.IsDir() {
60+
return nil, fmt.Errorf("can not open %s, not a directory", baseDir)
61+
}
62+
// Verify file mode is not too permissive.
63+
if err = fsutil.EnsureMaxPermissions(fi, dirCreateMode); err != nil {
64+
return nil, err
65+
}
66+
}
67+
68+
return f, nil
69+
}
70+
71+
// GetMeta returns the currently cached set of metadata files.
72+
func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) {
73+
f.mtx.RLock()
74+
defer f.mtx.RUnlock()
75+
76+
names, err := os.ReadDir(f.baseDir)
77+
if err != nil {
78+
return nil, fmt.Errorf("error reading directory %s: %w", f.baseDir, err)
79+
}
80+
81+
meta := map[string]json.RawMessage{}
82+
for _, name := range names {
83+
ok, err := fsutil.IsMetaFile(name)
84+
if err != nil {
85+
return nil, err
86+
}
87+
if !ok {
88+
continue
89+
}
90+
91+
// Verify permissions
92+
info, err := name.Info()
93+
if err != nil {
94+
return nil, fmt.Errorf("error retrieving FileInfo for %s: %w", name.Name(), err)
95+
}
96+
if err = fsutil.EnsureMaxPermissions(info, fileCreateMode); err != nil {
97+
return nil, err
98+
}
99+
100+
p := filepath.Join(f.baseDir, name.Name())
101+
b, err := os.ReadFile(p)
102+
if err != nil {
103+
return nil, fmt.Errorf("error reading file %s: %w", name.Name(), err)
104+
}
105+
meta[name.Name()] = b
106+
}
107+
108+
return meta, nil
109+
}
110+
111+
// SetMeta stores a metadata file in the cache. If the metadata file exist,
112+
// it will be overwritten.
113+
func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error {
114+
f.mtx.Lock()
115+
defer f.mtx.Unlock()
116+
117+
if filepath.Ext(name) != ".json" {
118+
return fmt.Errorf("file %s is not a JSON file", name)
119+
}
120+
121+
p := filepath.Join(f.baseDir, name)
122+
err := util.AtomicallyWriteFile(p, meta, fileCreateMode)
123+
return err
124+
}
125+
126+
// DeleteMeta deletes a metadata file from the cache.
127+
// If the file does not exist, an *os.PathError is returned.
128+
func (f *FileJSONStore) DeleteMeta(name string) error {
129+
f.mtx.Lock()
130+
defer f.mtx.Unlock()
131+
132+
if filepath.Ext(name) != ".json" {
133+
return fmt.Errorf("file %s is not a JSON file", name)
134+
}
135+
136+
p := filepath.Join(f.baseDir, name)
137+
err := os.Remove(p)
138+
if err == nil {
139+
return nil
140+
}
141+
142+
return fmt.Errorf("error deleting file %s: %w", name, err)
143+
}
144+
145+
// Close closes the metadata cache. This is a no-op.
146+
func (f *FileJSONStore) Close() error {
147+
return nil
148+
}
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package client
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
"testing"
10+
11+
"gopkg.in/check.v1"
12+
)
13+
14+
type FileJSONStoreSuite struct{}
15+
16+
var _ = check.Suite(&FileJSONStoreSuite{})
17+
18+
// Hook up gocheck into the "go test" runner
19+
func Test(t *testing.T) { check.TestingT(t) }
20+
21+
func (FileJSONStoreSuite) TestNewOk(c *check.C) {
22+
tmp := c.MkDir()
23+
p := filepath.Join(tmp, "tuf_raw.db")
24+
25+
// Assert path does not exist
26+
fi, err := os.Stat(p)
27+
c.Assert(fi, check.IsNil)
28+
c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true)
29+
30+
// Create implementation
31+
s, err := NewFileJSONStore(p)
32+
c.Assert(err, check.IsNil)
33+
c.Assert(s, check.NotNil)
34+
35+
// Assert path does exist and is a directory
36+
fi, err = os.Stat(p)
37+
c.Assert(fi, check.NotNil)
38+
c.Assert(err, check.IsNil)
39+
c.Assert(fi.IsDir(), check.Equals, true)
40+
}
41+
42+
func (FileJSONStoreSuite) TestNewFileExists(c *check.C) {
43+
tmp := c.MkDir()
44+
p := filepath.Join(tmp, "tuf_raw.db")
45+
46+
// Create an empty file
47+
f, err := os.Create(p)
48+
c.Assert(err, check.IsNil)
49+
f.Close()
50+
51+
// Create implementation
52+
s, err := NewFileJSONStore(p)
53+
c.Assert(s, check.IsNil)
54+
c.Assert(err, check.NotNil)
55+
found := strings.Contains(err.Error(), ", not a directory")
56+
c.Assert(found, check.Equals, true)
57+
}
58+
59+
func (FileJSONStoreSuite) TestNewDirectoryExists(c *check.C) {
60+
tmp := c.MkDir()
61+
p := filepath.Join(tmp, "tuf_raw.db")
62+
63+
err := os.Mkdir(p, 0750)
64+
c.Assert(err, check.IsNil)
65+
66+
// Create implementation
67+
s, err := NewFileJSONStore(p)
68+
c.Assert(s, check.NotNil)
69+
c.Assert(err, check.IsNil)
70+
}
71+
72+
func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) {
73+
tmp := c.MkDir()
74+
p := filepath.Join(tmp, "tuf_raw.db")
75+
s, err := NewFileJSONStore(p)
76+
c.Assert(err, check.IsNil)
77+
78+
md, err := s.GetMeta()
79+
c.Assert(err, check.IsNil)
80+
c.Assert(md, check.HasLen, 0)
81+
}
82+
83+
func (FileJSONStoreSuite) TestGetNoDirectory(c *check.C) {
84+
tmp := c.MkDir()
85+
p := filepath.Join(tmp, "tuf_raw.db")
86+
s, err := NewFileJSONStore(p)
87+
c.Assert(err, check.IsNil)
88+
89+
err = os.Remove(p)
90+
c.Assert(err, check.IsNil)
91+
92+
md, err := s.GetMeta()
93+
c.Assert(md, check.IsNil)
94+
c.Assert(err, check.NotNil)
95+
}
96+
97+
func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) {
98+
tmp := c.MkDir()
99+
p := filepath.Join(tmp, "tuf_raw.db")
100+
s, err := NewFileJSONStore(p)
101+
c.Assert(err, check.IsNil)
102+
103+
expected := map[string]json.RawMessage{
104+
"file1.json": []byte{0xf1, 0xe1, 0xd1},
105+
"file2.json": []byte{0xf2, 0xe2, 0xd2},
106+
"file3.json": []byte{0xf3, 0xe3, 0xd3},
107+
}
108+
109+
for k, v := range expected {
110+
err := s.SetMeta(k, v)
111+
c.Assert(err, check.IsNil)
112+
}
113+
114+
md, err := s.GetMeta()
115+
c.Assert(err, check.IsNil)
116+
c.Assert(md, check.HasLen, 3)
117+
c.Assert(md, check.DeepEquals, expected)
118+
119+
// Delete all items
120+
count := 3
121+
for k := range expected {
122+
err = s.DeleteMeta(k)
123+
c.Assert(err, check.IsNil)
124+
125+
md, err := s.GetMeta()
126+
c.Assert(err, check.IsNil)
127+
128+
count--
129+
c.Assert(md, check.HasLen, count)
130+
}
131+
132+
md, err = s.GetMeta()
133+
c.Assert(err, check.IsNil)
134+
c.Assert(md, check.HasLen, 0)
135+
}
136+
137+
func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) {
138+
tmp := c.MkDir()
139+
p := filepath.Join(tmp, "tuf_raw.db")
140+
s, err := NewFileJSONStore(p)
141+
c.Assert(s, check.NotNil)
142+
c.Assert(err, check.IsNil)
143+
144+
// Create a file which does not end with '.json'
145+
fp := filepath.Join(p, "meta.xml")
146+
err = os.WriteFile(fp, []byte{}, 0644)
147+
c.Assert(err, check.IsNil)
148+
149+
md, err := s.GetMeta()
150+
c.Assert(err, check.IsNil)
151+
c.Assert(md, check.HasLen, 0)
152+
}
153+
154+
func (FileJSONStoreSuite) TestNoJSON(c *check.C) {
155+
tmp := c.MkDir()
156+
p := filepath.Join(tmp, "tuf_raw.db")
157+
s, err := NewFileJSONStore(p)
158+
c.Assert(s, check.NotNil)
159+
c.Assert(err, check.IsNil)
160+
161+
files := []string{
162+
"file.xml",
163+
"file",
164+
"",
165+
}
166+
for _, f := range files {
167+
err := s.SetMeta(f, []byte{})
168+
c.Assert(err, check.ErrorMatches, "file.*is not a JSON file")
169+
}
170+
}
171+
172+
func (FileJSONStoreSuite) TestClose(c *check.C) {
173+
tmp := c.MkDir()
174+
p := filepath.Join(tmp, "tuf_raw.db")
175+
s, err := NewFileJSONStore(p)
176+
c.Assert(s, check.NotNil)
177+
c.Assert(err, check.IsNil)
178+
179+
err = s.Close()
180+
c.Assert(err, check.IsNil)
181+
}
182+
183+
func (FileJSONStoreSuite) TestDelete(c *check.C) {
184+
tmp := c.MkDir()
185+
p := filepath.Join(tmp, "tuf_raw.db")
186+
s, err := NewFileJSONStore(p)
187+
c.Assert(s, check.NotNil)
188+
c.Assert(err, check.IsNil)
189+
190+
err = s.DeleteMeta("not_json.yml")
191+
c.Assert(err, check.ErrorMatches, "file not_json\\.yml is not a JSON file")
192+
err = s.DeleteMeta("non_existing.json")
193+
c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true)
194+
}

0 commit comments

Comments
 (0)