Skip to content

Commit fff3712

Browse files
emcfarlaneemcfarlane
authored
Add bufcheck support for remote policies (#3934)
1 parent d592e99 commit fff3712

File tree

15 files changed

+720
-236
lines changed

15 files changed

+720
-236
lines changed

‎private/buf/bufcli/cache.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,14 @@ import (
3232
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
3333
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache"
3434
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore"
35+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
36+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicyapi"
37+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicycache"
38+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicystore"
3539
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
3640
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
3741
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
42+
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapipolicy"
3843
"github.com/bufbuild/buf/private/pkg/filelock"
3944
"github.com/bufbuild/buf/private/pkg/normalpath"
4045
"github.com/bufbuild/buf/private/pkg/storage/storageos"
@@ -60,6 +65,7 @@ var (
6065
v3CacheModuleLockRelDirPath,
6166
v3CacheModuleRelDirPath,
6267
v3CachePluginRelDirPath,
68+
v3CachePolicyRelDirPath,
6369
v3CacheWKTRelDirPath,
6470
v3CacheWasmRuntimeRelDirPath,
6571
}
@@ -122,6 +128,10 @@ var (
122128
//
123129
// Normalized.
124130
v3CacheWasmRuntimeRelDirPath = normalpath.Join("v3", "wasmruntime")
131+
// v3CachePolicyRelDirPath is the relative path to the files cache directory in its newest iteration.
132+
//
133+
// Normalized.
134+
v3CachePolicyRelDirPath = normalpath.Join("v3", "policies")
125135
)
126136

127137
// NewModuleDataProvider returns a new ModuleDataProvider while creating the
@@ -175,6 +185,21 @@ func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProv
175185
)
176186
}
177187

188+
// NewPolicyDataProvider returns a new PolicyDataProvider while creating the
189+
// required cache directories.
190+
func NewPolicyDataProvider(container appext.Container) (bufpolicy.PolicyDataProvider, error) {
191+
clientConfig, err := NewConnectClientConfig(container)
192+
if err != nil {
193+
return nil, err
194+
}
195+
return newPolicyDataProvider(
196+
container,
197+
bufregistryapipolicy.NewClientProvider(
198+
clientConfig,
199+
),
200+
)
201+
}
202+
178203
// NewWasmRuntime returns a new Wasm runtime while creating the required cache
179204
// directories.
180205
func NewWasmRuntime(ctx context.Context, container appext.Container) (wasm.Runtime, error) {
@@ -301,6 +326,33 @@ func newPluginDataProvider(
301326
), nil
302327
}
303328

329+
func newPolicyDataProvider(
330+
container appext.Container,
331+
policyClientProvider bufregistryapipolicy.ClientProvider,
332+
) (bufpolicy.PolicyDataProvider, error) {
333+
if err := createCacheDir(container.CacheDirPath(), v3CachePolicyRelDirPath); err != nil {
334+
return nil, err
335+
}
336+
fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePolicyRelDirPath)
337+
storageosProvider := storageos.NewProvider() // No symlinks.
338+
cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath)
339+
if err != nil {
340+
return nil, err
341+
}
342+
delegateModuleDataProvider := bufpolicyapi.NewPolicyDataProvider(
343+
container.Logger(),
344+
policyClientProvider,
345+
)
346+
return bufpolicycache.NewPolicyDataProvider(
347+
container.Logger(),
348+
delegateModuleDataProvider,
349+
bufpolicystore.NewPolicyDataStore(
350+
container.Logger(),
351+
cacheBucket,
352+
),
353+
), nil
354+
}
355+
304356
func createCacheDir(baseCacheDirPath string, relDirPath string) error {
305357
baseCacheDirPath = normalpath.Unnormalize(baseCacheDirPath)
306358
relDirPath = normalpath.Unnormalize(relDirPath)

‎private/buf/bufcli/controller.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ import (
1919
"github.com/bufbuild/buf/private/buf/bufctl"
2020
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
2121
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
22+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicyapi"
2223
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
2324
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
2425
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
26+
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapipolicy"
2527
)
2628

2729
// NewController returns a new Controller.
@@ -42,6 +44,7 @@ func NewController(
4244
moduleClientProvider := bufregistryapimodule.NewClientProvider(clientConfig)
4345
ownerClientProvider := bufregistryapiowner.NewClientProvider(clientConfig)
4446
pluginClientProvider := bufregistryapiplugin.NewClientProvider(clientConfig)
47+
policyClientProvider := bufregistryapipolicy.NewClientProvider(clientConfig)
4548
moduleDataProvider, err := newModuleDataProvider(container, moduleClientProvider, ownerClientProvider)
4649
if err != nil {
4750
return nil, err
@@ -54,6 +57,10 @@ func NewController(
5457
if err != nil {
5558
return nil, err
5659
}
60+
policyDataProvider, err := newPolicyDataProvider(container, policyClientProvider)
61+
if err != nil {
62+
return nil, err
63+
}
5764
wktStore, err := NewWKTStore(container)
5865
if err != nil {
5966
return nil, err
@@ -67,6 +74,8 @@ func NewController(
6774
commitProvider,
6875
bufpluginapi.NewPluginKeyProvider(container.Logger(), pluginClientProvider),
6976
pluginDataProvider,
77+
bufpolicyapi.NewPolicyKeyProvider(container.Logger(), policyClientProvider),
78+
policyDataProvider,
7079
wktStore,
7180
// TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig
7281
defaultHTTPClient,

‎private/buf/bufctl/controller.go‎

Lines changed: 151 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"github.com/bufbuild/buf/private/bufpkg/bufmodule"
4343
"github.com/bufbuild/buf/private/bufpkg/bufparse"
4444
"github.com/bufbuild/buf/private/bufpkg/bufplugin"
45+
"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
4546
"github.com/bufbuild/buf/private/bufpkg/bufreflect"
4647
"github.com/bufbuild/buf/private/gen/data/datawkt"
4748
imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1"
@@ -164,6 +165,8 @@ func NewController(
164165
commitProvider bufmodule.CommitProvider,
165166
pluginKeyProvider bufplugin.PluginKeyProvider,
166167
pluginDataProvider bufplugin.PluginDataProvider,
168+
policyKeyProvider bufpolicy.PolicyKeyProvider,
169+
policyDataProvider bufpolicy.PolicyDataProvider,
167170
wktStore bufwktstore.Store,
168171
httpClient *http.Client,
169172
httpauthAuthenticator httpauth.Authenticator,
@@ -179,6 +182,8 @@ func NewController(
179182
commitProvider,
180183
pluginKeyProvider,
181184
pluginDataProvider,
185+
policyKeyProvider,
186+
policyDataProvider,
182187
wktStore,
183188
httpClient,
184189
httpauthAuthenticator,
@@ -202,6 +207,8 @@ type controller struct {
202207
commitProvider bufmodule.CommitProvider
203208
pluginKeyProvider bufplugin.PluginKeyProvider
204209
pluginDataProvider bufplugin.PluginDataProvider
210+
policyKeyProvider bufpolicy.PolicyKeyProvider
211+
policyDataProvider bufpolicy.PolicyDataProvider
205212
wktStore bufwktstore.Store
206213

207214
disableSymlinks bool
@@ -226,6 +233,8 @@ func newController(
226233
commitProvider bufmodule.CommitProvider,
227234
pluginKeyProvider bufplugin.PluginKeyProvider,
228235
pluginDataProvider bufplugin.PluginDataProvider,
236+
policyKeyProvider bufpolicy.PolicyKeyProvider,
237+
policyDataProvider bufpolicy.PolicyDataProvider,
229238
wktStore bufwktstore.Store,
230239
httpClient *http.Client,
231240
httpauthAuthenticator httpauth.Authenticator,
@@ -240,6 +249,8 @@ func newController(
240249
commitProvider: commitProvider,
241250
pluginKeyProvider: pluginKeyProvider,
242251
pluginDataProvider: pluginDataProvider,
252+
policyKeyProvider: policyKeyProvider,
253+
policyDataProvider: policyDataProvider,
243254
wktStore: wktStore,
244255
}
245256
for _, option := range options {
@@ -406,10 +417,13 @@ func (c *controller) GetTargetImageWithConfigsAndCheckClient(
406417
lintConfig := bufconfig.DefaultLintConfigV1
407418
breakingConfig := bufconfig.DefaultBreakingConfigV1
408419
var (
409-
pluginConfigs []bufconfig.PluginConfig
410-
policyConfigs []bufconfig.PolicyConfig
420+
pluginConfigs []bufconfig.PluginConfig
421+
policyConfigs []bufconfig.PolicyConfig
422+
pluginKeyProvider = bufplugin.NopPluginKeyProvider
423+
policyKeyProvider = bufpolicy.NopPolicyKeyProvider
424+
policyPluginKeyProvider = bufpolicy.NopPolicyPluginKeyProvider
425+
policyPluginDataProvider = bufpolicy.NopPolicyPluginDataProvider
411426
)
412-
pluginKeyProvider := bufplugin.NopPluginKeyProvider
413427
bufYAMLFile, err := bufconfig.GetBufYAMLFileForPrefixOrOverride(
414428
ctx,
415429
bucket,
@@ -457,7 +471,11 @@ func (c *controller) GetTargetImageWithConfigsAndCheckClient(
457471
// We use the BSR to resolve any remote plugin Refs.
458472
pluginKeyProvider = c.pluginKeyProvider
459473
} else if bufYAMLFile.FileVersion() == bufconfig.FileVersionV2 {
460-
var pluginKeys []bufplugin.PluginKey
474+
var (
475+
remotePluginKeys []bufplugin.PluginKey
476+
remotePolicyKeys []bufpolicy.PolicyKey
477+
policyNameToRemotePluginKeys map[string][]bufplugin.PluginKey
478+
)
461479
if bufLockFile, err := bufconfig.GetBufLockFileForPrefix(
462480
ctx,
463481
bucket,
@@ -468,14 +486,36 @@ func (c *controller) GetTargetImageWithConfigsAndCheckClient(
468486
return nil, nil, err
469487
}
470488
// We did not find a buf.lock in our current directory.
471-
// Remote plugins are not available.
472-
pluginKeys = nil
489+
// Remote plugins and policies are not available.
473490
} else {
474-
pluginKeys = bufLockFile.RemotePluginKeys()
491+
remotePluginKeys = bufLockFile.RemotePluginKeys()
492+
remotePolicyKeys = bufLockFile.RemotePolicyKeys()
493+
policyNameToRemotePluginKeys = bufLockFile.PolicyNameToRemotePluginKeys()
475494
}
476495
pluginKeyProvider, err = newStaticPluginKeyProviderForPluginConfigs(
477496
pluginConfigs,
478-
pluginKeys,
497+
remotePluginKeys,
498+
)
499+
if err != nil {
500+
return nil, nil, err
501+
}
502+
policyKeyProvider, err = newStaticPolicyKeyProviderForPolicyConfigs(
503+
policyConfigs,
504+
remotePolicyKeys,
505+
)
506+
if err != nil {
507+
return nil, nil, err
508+
}
509+
policyPluginKeyProvider, err = newStaticPolicyPluginKeyProviderForPolicyConfigs(
510+
policyConfigs,
511+
policyNameToRemotePluginKeys,
512+
)
513+
if err != nil {
514+
return nil, nil, err
515+
}
516+
policyPluginDataProvider, err = newStaticPolicyPluginDataProviderForPolicyConfigs(
517+
c.pluginDataProvider,
518+
policyConfigs,
479519
)
480520
if err != nil {
481521
return nil, nil, err
@@ -502,6 +542,12 @@ func (c *controller) GetTargetImageWithConfigsAndCheckClient(
502542
bufcheck.ClientWithLocalWasmPluginsFromOS(),
503543
bufcheck.ClientWithRemoteWasmPlugins(pluginKeyProvider, c.pluginDataProvider),
504544
bufcheck.ClientWithLocalPoliciesFromOS(),
545+
bufcheck.ClientWithRemotePolicies(
546+
policyKeyProvider,
547+
c.policyDataProvider,
548+
policyPluginKeyProvider,
549+
policyPluginDataProvider,
550+
),
505551
)
506552
if err != nil {
507553
return nil, nil, err
@@ -808,6 +854,27 @@ func (c *controller) GetCheckClientForWorkspace(
808854
if err != nil {
809855
return nil, err
810856
}
857+
policyKeyProvider, err := newStaticPolicyKeyProviderForPolicyConfigs(
858+
workspace.PolicyConfigs(),
859+
workspace.RemotePolicyKeys(),
860+
)
861+
if err != nil {
862+
return nil, err
863+
}
864+
policyPluginKeyProvider, err := newStaticPolicyPluginKeyProviderForPolicyConfigs(
865+
workspace.PolicyConfigs(),
866+
workspace.PolicyNameToRemotePluginKeys(),
867+
)
868+
if err != nil {
869+
return nil, err
870+
}
871+
policyPluginDataProvider, err := newStaticPolicyPluginDataProviderForPolicyConfigs(
872+
c.pluginDataProvider,
873+
workspace.PolicyConfigs(),
874+
)
875+
if err != nil {
876+
return nil, err
877+
}
811878
return bufcheck.NewClient(
812879
c.logger,
813880
bufcheck.ClientWithStderr(c.container.Stderr()),
@@ -820,6 +887,12 @@ func (c *controller) GetCheckClientForWorkspace(
820887
c.pluginDataProvider,
821888
),
822889
bufcheck.ClientWithLocalPoliciesFromOS(),
890+
bufcheck.ClientWithRemotePolicies(
891+
policyKeyProvider,
892+
c.policyDataProvider,
893+
policyPluginKeyProvider,
894+
policyPluginDataProvider,
895+
),
823896
)
824897
}
825898

@@ -1509,3 +1582,73 @@ func newStaticPluginKeyProviderForPluginConfigs(
15091582
}
15101583
return bufplugin.NewStaticPluginKeyProvider(pluginKeys)
15111584
}
1585+
1586+
// newStaticPolicyKeyProvider creates a new PolicyKeyProvider for the set of PolicyKeys.
1587+
//
1588+
// The PolicyKeys come from the buf.lock file. The PolicyKeyProvider is static
1589+
// and does not change. PolicyConfigs are validated to ensure that all remote
1590+
// PolicyConfigs are pinned in the buf.lock file.
1591+
func newStaticPolicyKeyProviderForPolicyConfigs(
1592+
policyConfigs []bufconfig.PolicyConfig,
1593+
policyKeys []bufpolicy.PolicyKey,
1594+
) (_ bufpolicy.PolicyKeyProvider, retErr error) {
1595+
// Validate that all remote PolicyConfigs are present in the buf.lock file.
1596+
policyKeysByFullName, err := xslices.ToUniqueValuesMap(policyKeys, func(policyKey bufpolicy.PolicyKey) string {
1597+
return policyKey.FullName().String()
1598+
})
1599+
if err != nil {
1600+
return nil, fmt.Errorf("failed to validate remote PolicyKeys: %w", err)
1601+
}
1602+
// Remote PolicyConfig Refs are any PolicyConfigs that have a Ref.
1603+
remotePolicyRefs := xslices.Filter(
1604+
xslices.Map(policyConfigs, func(policyConfig bufconfig.PolicyConfig) bufparse.Ref {
1605+
return policyConfig.Ref()
1606+
}),
1607+
func(policyRef bufparse.Ref) bool {
1608+
return policyRef != nil
1609+
},
1610+
)
1611+
for _, remotePolicyRef := range remotePolicyRefs {
1612+
if _, ok := policyKeysByFullName[remotePolicyRef.FullName().String()]; !ok {
1613+
return nil, fmt.Errorf(`remote policy %q is not in the buf.lock file, use "buf policy update" to pin remote refs`, remotePolicyRef)
1614+
}
1615+
}
1616+
return bufpolicy.NewStaticPolicyKeyProvider(policyKeys)
1617+
}
1618+
1619+
// newStaticPolicyPluginKeyProviderForPolicyConfigs creates a new PolicyPluginKeyProvider for the set of PolicyConfigs.
1620+
func newStaticPolicyPluginKeyProviderForPolicyConfigs(
1621+
policyConfigs []bufconfig.PolicyConfig,
1622+
policyNameToRemotePluginKeys map[string][]bufplugin.PluginKey,
1623+
) (bufpolicy.PolicyPluginKeyProvider, error) {
1624+
// We do not validate that all remote PolicyConfig plugins are present in the buf.lock file.
1625+
// This would require loading the PolicyConfig data. Check is defered to the runtime.
1626+
for policyName, remotePluginKeys := range policyNameToRemotePluginKeys {
1627+
_, err := xslices.ToUniqueValuesMap(remotePluginKeys, func(pluginKey bufplugin.PluginKey) string {
1628+
return pluginKey.FullName().String()
1629+
})
1630+
if err != nil {
1631+
return nil, fmt.Errorf("failed to validate remote PluginKeys for Policy %q: %w", policyName, err)
1632+
}
1633+
}
1634+
return bufpolicy.NewStaticPolicyPluginKeyProvider(policyNameToRemotePluginKeys)
1635+
}
1636+
1637+
// newStaticPolicyPluginDataProviderForPolicyConfigs creates a new PolicyPluginKeyProvider for the set of PolicyConfigs.
1638+
//
1639+
// The pluginDataProvider is shared across all policies.
1640+
func newStaticPolicyPluginDataProviderForPolicyConfigs(
1641+
pluginDataProvider bufplugin.PluginDataProvider,
1642+
policyConfigs []bufconfig.PolicyConfig,
1643+
) (bufpolicy.PolicyPluginDataProvider, error) {
1644+
policyNameToPluginDataProvider := make(map[string]bufplugin.PluginDataProvider)
1645+
for _, policyConfig := range policyConfigs {
1646+
policyName := policyConfig.Name()
1647+
if _, ok := policyNameToPluginDataProvider[policyName]; ok {
1648+
return nil, fmt.Errorf("duplicate policy name %q found in policy configs", policyName)
1649+
}
1650+
// We use the same pluginDataProvider for all policies.
1651+
policyNameToPluginDataProvider[policyName] = pluginDataProvider
1652+
}
1653+
return bufpolicy.NewStaticPolicyPluginDataProvider(policyNameToPluginDataProvider)
1654+
}

0 commit comments

Comments
 (0)