Skip to content

Commit 9576041

Browse files
emcfarlaneemcfarlane
authored
Add bufpolicystore and bufpolicycache for Policies (#3796)
1 parent 1aff147 commit 9576041

File tree

6 files changed

+383
-0
lines changed

6 files changed

+383
-0
lines changed

‎private/bufpkg/bufpolicy/bufpolicycache/bufpolicycache.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright 2020-2025 Buf Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package bufpolicycache

‎private/bufpkg/bufpolicy/bufpolicycache/policy_data_provider.go‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright 2020-2025 Buf Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package bufpolicycache
16+
17+
import (
18+
"context"
19+
"log/slog"
20+
"sync/atomic"
21+
22+
"buf.build/go/standard/xslices"
23+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
24+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicystore"
25+
"github.com/bufbuild/buf/private/pkg/syserror"
26+
"github.com/bufbuild/buf/private/pkg/uuidutil"
27+
"github.com/google/uuid"
28+
)
29+
30+
// NewPolicyDataProvider returns a new PolicyDataProvider that caches the results of the delegate.
31+
//
32+
// The PolicyDataStore is used as a cache.
33+
func NewPolicyDataProvider(
34+
logger *slog.Logger,
35+
delegate bufpolicy.PolicyDataProvider,
36+
store bufpolicystore.PolicyDataStore,
37+
) bufpolicy.PolicyDataProvider {
38+
return newPolicyDataProvider(logger, delegate, store)
39+
}
40+
41+
/// *** PRIVATE ***
42+
43+
type policyDataProvider struct {
44+
logger *slog.Logger
45+
delegate bufpolicy.PolicyDataProvider
46+
store bufpolicystore.PolicyDataStore
47+
48+
keysRetrieved atomic.Int64
49+
keysHit atomic.Int64
50+
}
51+
52+
func newPolicyDataProvider(
53+
logger *slog.Logger,
54+
delegate bufpolicy.PolicyDataProvider,
55+
store bufpolicystore.PolicyDataStore,
56+
) *policyDataProvider {
57+
return &policyDataProvider{
58+
logger: logger,
59+
delegate: delegate,
60+
store: store,
61+
}
62+
}
63+
64+
func (p *policyDataProvider) GetPolicyDatasForPolicyKeys(
65+
ctx context.Context,
66+
policyKeys []bufpolicy.PolicyKey,
67+
) ([]bufpolicy.PolicyData, error) {
68+
foundValues, notFoundKeys, err := p.store.GetPolicyDatasForPolicyKeys(ctx, policyKeys)
69+
if err != nil {
70+
return nil, err
71+
}
72+
73+
delegateValues, err := p.delegate.GetPolicyDatasForPolicyKeys(ctx, notFoundKeys)
74+
if err != nil {
75+
return nil, err
76+
}
77+
if err := p.store.PutPolicyDatas(ctx, delegateValues); err != nil {
78+
return nil, err
79+
}
80+
81+
p.keysRetrieved.Add(int64(len(policyKeys)))
82+
p.keysHit.Add(int64(len(foundValues)))
83+
84+
commitIDToIndexedKey, err := xslices.ToUniqueIndexedValuesMap(
85+
policyKeys,
86+
func(policyKey bufpolicy.PolicyKey) uuid.UUID {
87+
return policyKey.CommitID()
88+
},
89+
)
90+
if err != nil {
91+
return nil, err
92+
}
93+
indexedValues, err := xslices.MapError(
94+
append(foundValues, delegateValues...),
95+
func(value bufpolicy.PolicyData) (xslices.Indexed[bufpolicy.PolicyData], error) {
96+
commitID := value.PolicyKey().CommitID()
97+
indexedKey, ok := commitIDToIndexedKey[commitID]
98+
if !ok {
99+
return xslices.Indexed[bufpolicy.PolicyData]{}, syserror.Newf("did not get value from store with commitID %q", uuidutil.ToDashless(commitID))
100+
}
101+
return xslices.Indexed[bufpolicy.PolicyData]{
102+
Value: value,
103+
Index: indexedKey.Index,
104+
}, nil
105+
},
106+
)
107+
if err != nil {
108+
return nil, err
109+
}
110+
return xslices.IndexedToSortedValues(indexedValues), nil
111+
}

‎private/bufpkg/bufpolicy/bufpolicycache/usage.gen.go‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎private/bufpkg/bufpolicy/bufpolicystore/buf_policy_store.go‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// Copyright 2020-2025 Buf Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package bufpolicystore
16+
17+
import (
18+
"bytes"
19+
"context"
20+
"errors"
21+
"io/fs"
22+
"log/slog"
23+
24+
policyv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/policy/v1beta1"
25+
"github.com/bufbuild/buf/private/bufpkg/bufcas"
26+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
27+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicyapi"
28+
"github.com/bufbuild/buf/private/pkg/normalpath"
29+
"github.com/bufbuild/buf/private/pkg/protoencoding"
30+
"github.com/bufbuild/buf/private/pkg/storage"
31+
"github.com/bufbuild/buf/private/pkg/uuidutil"
32+
)
33+
34+
// PolicyDataStore reads and writes PolicysDatas.
35+
type PolicyDataStore interface {
36+
// GetPolicyDatasForPolicyKeys gets the PolicyDatas from the store for the PolicyKeys.
37+
//
38+
// Returns the found PolicyDatas, and the input PolicyKeys that were not found, each
39+
// ordered by the order of the input PolicyKeys.
40+
GetPolicyDatasForPolicyKeys(context.Context, []bufpolicy.PolicyKey) (
41+
foundPolicyDatas []bufpolicy.PolicyData,
42+
notFoundPolicyKeys []bufpolicy.PolicyKey,
43+
err error,
44+
)
45+
// PutPolicyDatas puts the PolicyDatas to the store.
46+
PutPolicyDatas(ctx context.Context, moduleDatas []bufpolicy.PolicyData) error
47+
}
48+
49+
// NewPolicyDataStore returns a new PolicyDataStore for the given bucket.
50+
//
51+
// It is assumed that the PolicyDataStore has complete control of the bucket.
52+
//
53+
// This is typically used to interact with a cache directory.
54+
func NewPolicyDataStore(
55+
logger *slog.Logger,
56+
bucket storage.ReadWriteBucket,
57+
) PolicyDataStore {
58+
return newPolicyDataStore(logger, bucket)
59+
}
60+
61+
/// *** PRIVATE ***
62+
63+
type policyDataStore struct {
64+
logger *slog.Logger
65+
bucket storage.ReadWriteBucket
66+
}
67+
68+
func newPolicyDataStore(
69+
logger *slog.Logger,
70+
bucket storage.ReadWriteBucket,
71+
) *policyDataStore {
72+
return &policyDataStore{
73+
logger: logger,
74+
bucket: bucket,
75+
}
76+
}
77+
78+
func (p *policyDataStore) GetPolicyDatasForPolicyKeys(
79+
ctx context.Context,
80+
policyKeys []bufpolicy.PolicyKey,
81+
) ([]bufpolicy.PolicyData, []bufpolicy.PolicyKey, error) {
82+
var foundPolicyDatas []bufpolicy.PolicyData
83+
var notFoundPolicyKeys []bufpolicy.PolicyKey
84+
for _, policyKey := range policyKeys {
85+
policyData, err := p.getPolicyDataForPolicyKey(ctx, policyKey)
86+
if err != nil {
87+
if !errors.Is(err, fs.ErrNotExist) {
88+
return nil, nil, err
89+
}
90+
notFoundPolicyKeys = append(notFoundPolicyKeys, policyKey)
91+
} else {
92+
foundPolicyDatas = append(foundPolicyDatas, policyData)
93+
}
94+
}
95+
return foundPolicyDatas, notFoundPolicyKeys, nil
96+
}
97+
98+
func (p *policyDataStore) PutPolicyDatas(
99+
ctx context.Context,
100+
policyDatas []bufpolicy.PolicyData,
101+
) error {
102+
for _, policyData := range policyDatas {
103+
if err := p.putPolicyData(ctx, policyData); err != nil {
104+
return err
105+
}
106+
}
107+
return nil
108+
}
109+
110+
// getPolicyDataForPolicyKey reads the policy data for the policy key from the cache.
111+
func (p *policyDataStore) getPolicyDataForPolicyKey(
112+
ctx context.Context,
113+
policyKey bufpolicy.PolicyKey,
114+
) (bufpolicy.PolicyData, error) {
115+
policyDataStorePath, err := getPolicyDataStorePath(policyKey)
116+
if err != nil {
117+
return nil, err
118+
}
119+
if exists, err := storage.Exists(ctx, p.bucket, policyDataStorePath); err != nil {
120+
return nil, err
121+
} else if !exists {
122+
return nil, fs.ErrNotExist
123+
}
124+
getConfig := func() (bufpolicy.PolicyConfig, error) {
125+
data, err := storage.ReadPath(ctx, p.bucket, policyDataStorePath)
126+
if err != nil {
127+
return nil, err
128+
}
129+
// Validate the digest, before parsing the config.
130+
bufcasDigest, err := bufcas.NewDigestForContent(bytes.NewReader(data))
131+
if err != nil {
132+
return nil, err
133+
}
134+
expectedDigest, err := policyKey.Digest()
135+
if err != nil {
136+
return nil, err
137+
}
138+
actualDigest, err := bufpolicy.NewDigest(expectedDigest.Type(), bufcasDigest)
139+
if err != nil {
140+
return nil, err
141+
}
142+
if !bufpolicy.DigestEqual(actualDigest, expectedDigest) {
143+
return nil, &bufpolicy.DigestMismatchError{
144+
FullName: policyKey.FullName(),
145+
CommitID: policyKey.CommitID(),
146+
ExpectedDigest: expectedDigest,
147+
ActualDigest: actualDigest,
148+
}
149+
}
150+
// Create the policy config from the data.
151+
var configProto policyv1beta1.PolicyConfig
152+
if err := protoencoding.NewJSONUnmarshaler(nil).Unmarshal(data, &configProto); err != nil {
153+
return nil, err
154+
}
155+
return bufpolicyapi.V1Beta1ProtoToPolicyConfig(policyKey.FullName().Registry(), &configProto)
156+
}
157+
return bufpolicy.NewPolicyData(
158+
ctx,
159+
policyKey,
160+
getConfig,
161+
)
162+
}
163+
164+
// putPolicyData puts the policy data into the policy cache.
165+
func (p *policyDataStore) putPolicyData(
166+
ctx context.Context,
167+
policyData bufpolicy.PolicyData,
168+
) error {
169+
policyKey := policyData.PolicyKey()
170+
policyDataStorePath, err := getPolicyDataStorePath(policyKey)
171+
if err != nil {
172+
return err
173+
}
174+
config, err := policyData.Config()
175+
if err != nil {
176+
return err
177+
}
178+
data, err := bufpolicy.MarshalPolicyConfigAsJSON(config)
179+
if err != nil {
180+
return err
181+
}
182+
// Data is stored uncompressed.
183+
return storage.PutPath(ctx, p.bucket, policyDataStorePath, data)
184+
}
185+
186+
// getPolicyDataStorePath returns the path for the policy data store for the policy key.
187+
//
188+
// This is "digestType/registry/owner/name/dashlessCommitID", e.g. the policy
189+
// "buf.build/acme/check-policy" with commit "12345-abcde" and digest type "o1"
190+
// will return "o1/buf.build/acme/check-policy/12345abcde.yaml".
191+
func getPolicyDataStorePath(policyKey bufpolicy.PolicyKey) (string, error) {
192+
digest, err := policyKey.Digest()
193+
if err != nil {
194+
return "", err
195+
}
196+
fullName := policyKey.FullName()
197+
return normalpath.Join(
198+
digest.Type().String(),
199+
fullName.Registry(),
200+
fullName.Owner(),
201+
fullName.Name(),
202+
uuidutil.ToDashless(policyKey.CommitID())+".yaml",
203+
), nil
204+
}

‎private/bufpkg/bufpolicy/bufpolicystore/bufpolicystore.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright 2020-2025 Buf Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package bufpolicystore

‎private/bufpkg/bufpolicy/bufpolicystore/usage.gen.go‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)