Skip to content

Commit 839ed4c

Browse files
emcfarlaneemcfarlane
committed
Add plugin update command
This adds a new command `buf plugin update` enabling the use of remote plugin refs in `buf.yaml` files. Running the command resolves the refs and updates the `buf.lock` file, adding the new section plugins with the resolved digest. Plugins are cached under the `v3/plugins` directory. This allows `buf lint` to invoke remote plugins locally enabling sharing of custom plugins.
1 parent dfeacfa commit 839ed4c

File tree

38 files changed

+1447
-48
lines changed

38 files changed

+1447
-48
lines changed

‎private/buf/bufcli/cache.go‎

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,32 +26,40 @@ import (
2626
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
2727
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulecache"
2828
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulestore"
29+
"github.com/bufbuild/buf/private/bufpkg/bufplugin"
30+
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
31+
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache"
32+
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore"
2933
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
3034
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
35+
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
3136
"github.com/bufbuild/buf/private/pkg/app/appext"
3237
"github.com/bufbuild/buf/private/pkg/filelock"
3338
"github.com/bufbuild/buf/private/pkg/normalpath"
3439
"github.com/bufbuild/buf/private/pkg/storage/storageos"
3540
)
3641

3742
var (
38-
// AllCacheModuleRelDirPaths are all directory paths for all time concerning the module cache.
43+
// AllCacheRelDirPaths are all directory paths for all time
44+
// concerning the module and plugin caches.
3945
//
4046
// These are normalized.
4147
// These are relative to container.CacheDirPath().
4248
//
4349
// This variable is used for clearing the cache.
44-
AllCacheModuleRelDirPaths = []string{
45-
v1beta1CacheModuleDataRelDirPath,
46-
v1beta1CacheModuleLockRelDirPath,
50+
AllCacheRelDirPaths = []string{
4751
v1CacheModuleDataRelDirPath,
4852
v1CacheModuleLockRelDirPath,
4953
v1CacheModuleSumRelDirPath,
54+
v1beta1CacheModuleDataRelDirPath,
55+
v1beta1CacheModuleLockRelDirPath,
5056
v2CacheModuleRelDirPath,
51-
v3CacheModuleRelDirPath,
5257
v3CacheCommitsRelDirPath,
53-
v3CacheWKTRelDirPath,
5458
v3CacheModuleLockRelDirPath,
59+
v3CacheModuleRelDirPath,
60+
v3CachePluginRelDirPath,
61+
v3CacheWKTRelDirPath,
62+
v3CacheWasmRuntimeRelDirPath,
5563
}
5664

5765
// v1CacheModuleDataRelDirPath is the relative path to the cache directory where module data
@@ -103,6 +111,10 @@ var (
103111
//
104112
// Normalized.
105113
v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
114+
// v3CachePluginRelDirPath is the relative path to the files cache directory in its newest iteration.
115+
//
116+
// Normalized.
117+
v3CachePluginRelDirPath = normalpath.Join("v3", "plugins")
106118
// v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration.
107119
// This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime.
108120
//
@@ -146,6 +158,21 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
146158
)
147159
}
148160

161+
// NewPluginDataProvider returns a new PluginDataProvider while creating the
162+
// required cache directories.
163+
func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProvider, error) {
164+
clientConfig, err := NewConnectClientConfig(container)
165+
if err != nil {
166+
return nil, err
167+
}
168+
return newPluginDataProvider(
169+
container,
170+
bufregistryapiplugin.NewClientProvider(
171+
clientConfig,
172+
),
173+
)
174+
}
175+
149176
// CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime.
150177
//
151178
// This is used by the Wasm runtime to cache compiled Wasm plugins. This is an
@@ -241,6 +268,33 @@ func newCommitProvider(
241268
), nil
242269
}
243270

271+
func newPluginDataProvider(
272+
container appext.Container,
273+
pluginClientProvider bufregistryapiplugin.ClientProvider,
274+
) (bufplugin.PluginDataProvider, error) {
275+
if err := createCacheDir(container.CacheDirPath(), v3CachePluginRelDirPath); err != nil {
276+
return nil, err
277+
}
278+
fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginRelDirPath)
279+
storageosProvider := storageos.NewProvider() // No symlinks.
280+
cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath)
281+
if err != nil {
282+
return nil, err
283+
}
284+
delegateModuleDataProvider := bufpluginapi.NewPluingDataProvider(
285+
container.Logger(),
286+
pluginClientProvider,
287+
)
288+
return bufplugincache.NewPluginDataProvider(
289+
container.Logger(),
290+
delegateModuleDataProvider,
291+
bufpluginstore.NewPluginDataStore(
292+
container.Logger(),
293+
cacheBucket,
294+
),
295+
), nil
296+
}
297+
244298
func createCacheDir(baseCacheDirPath string, relDirPath string) error {
245299
baseCacheDirPath = normalpath.Unnormalize(baseCacheDirPath)
246300
relDirPath = normalpath.Unnormalize(relDirPath)

‎private/buf/bufcli/controller.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ package bufcli
1717
import (
1818
"github.com/bufbuild/buf/private/buf/bufctl"
1919
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
20+
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
2021
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
2122
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
23+
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
2224
"github.com/bufbuild/buf/private/pkg/app/appext"
2325
)
2426

@@ -39,6 +41,7 @@ func NewController(
3941
}
4042
moduleClientProvider := bufregistryapimodule.NewClientProvider(clientConfig)
4143
ownerClientProvider := bufregistryapiowner.NewClientProvider(clientConfig)
44+
pluginClientProvider := bufregistryapiplugin.NewClientProvider(clientConfig)
4245
moduleDataProvider, err := newModuleDataProvider(container, moduleClientProvider, ownerClientProvider)
4346
if err != nil {
4447
return nil, err
@@ -47,6 +50,10 @@ func NewController(
4750
if err != nil {
4851
return nil, err
4952
}
53+
pluginDataProvider, err := newPluginDataProvider(container, pluginClientProvider)
54+
if err != nil {
55+
return nil, err
56+
}
5057
wktStore, err := NewWKTStore(container)
5158
if err != nil {
5259
return nil, err
@@ -58,6 +65,8 @@ func NewController(
5865
bufmoduleapi.NewModuleKeyProvider(container.Logger(), moduleClientProvider),
5966
moduleDataProvider,
6067
commitProvider,
68+
bufpluginapi.NewPluginKeyProvider(container.Logger(), pluginClientProvider),
69+
pluginDataProvider,
6170
wktStore,
6271
// TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig
6372
defaultHTTPClient,

‎private/buf/bufcli/plugin_key_provider.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright 2020-2024 Buf Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package bufcli
16+
17+
import (
18+
"github.com/bufbuild/buf/private/bufpkg/bufplugin"
19+
"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
20+
"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
21+
"github.com/bufbuild/buf/private/pkg/app/appext"
22+
)
23+
24+
// NewPluginKeyProvider returns a new PluginKeyProvider.
25+
func NewPluginKeyProvider(container appext.Container) (bufplugin.PluginKeyProvider, error) {
26+
clientConfig, err := NewConnectClientConfig(container)
27+
if err != nil {
28+
return nil, err
29+
}
30+
return bufpluginapi.NewPluginKeyProvider(
31+
container.Logger(),
32+
bufregistryapiplugin.NewClientProvider(
33+
clientConfig,
34+
),
35+
), nil
36+
}

‎private/buf/bufctl/controller.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil"
3535
"github.com/bufbuild/buf/private/bufpkg/bufmodule"
3636
"github.com/bufbuild/buf/private/bufpkg/bufparse"
37+
"github.com/bufbuild/buf/private/bufpkg/bufplugin"
3738
"github.com/bufbuild/buf/private/bufpkg/bufreflect"
3839
"github.com/bufbuild/buf/private/gen/data/datawkt"
3940
imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1"
@@ -127,6 +128,8 @@ type Controller interface {
127128
defaultMessageEncoding buffetch.MessageEncoding,
128129
options ...FunctionOption,
129130
) error
131+
PluginKeyProvider() bufplugin.PluginKeyProvider
132+
PluginDataProvider() bufplugin.PluginDataProvider
130133
}
131134

132135
func NewController(
@@ -136,6 +139,8 @@ func NewController(
136139
moduleKeyProvider bufmodule.ModuleKeyProvider,
137140
moduleDataProvider bufmodule.ModuleDataProvider,
138141
commitProvider bufmodule.CommitProvider,
142+
pluginKeyProvider bufplugin.PluginKeyProvider,
143+
pluginDataProvider bufplugin.PluginDataProvider,
139144
wktStore bufwktstore.Store,
140145
httpClient *http.Client,
141146
httpauthAuthenticator httpauth.Authenticator,
@@ -149,6 +154,8 @@ func NewController(
149154
moduleKeyProvider,
150155
moduleDataProvider,
151156
commitProvider,
157+
pluginKeyProvider,
158+
pluginDataProvider,
152159
wktStore,
153160
httpClient,
154161
httpauthAuthenticator,
@@ -170,6 +177,8 @@ type controller struct {
170177
moduleDataProvider bufmodule.ModuleDataProvider
171178
graphProvider bufmodule.GraphProvider
172179
commitProvider bufmodule.CommitProvider
180+
pluginKeyProvider bufplugin.PluginKeyProvider
181+
pluginDataProvider bufplugin.PluginDataProvider
173182
wktStore bufwktstore.Store
174183

175184
disableSymlinks bool
@@ -192,6 +201,8 @@ func newController(
192201
moduleKeyProvider bufmodule.ModuleKeyProvider,
193202
moduleDataProvider bufmodule.ModuleDataProvider,
194203
commitProvider bufmodule.CommitProvider,
204+
pluginKeyProvider bufplugin.PluginKeyProvider,
205+
pluginDataProvider bufplugin.PluginDataProvider,
195206
wktStore bufwktstore.Store,
196207
httpClient *http.Client,
197208
httpauthAuthenticator httpauth.Authenticator,
@@ -204,6 +215,8 @@ func newController(
204215
graphProvider: graphProvider,
205216
moduleDataProvider: moduleDataProvider,
206217
commitProvider: commitProvider,
218+
pluginKeyProvider: pluginKeyProvider,
219+
pluginDataProvider: pluginDataProvider,
207220
wktStore: wktStore,
208221
}
209222
for _, option := range options {
@@ -695,6 +708,14 @@ func (c *controller) PutMessage(
695708
return errors.Join(err, writeCloser.Close())
696709
}
697710

711+
func (c *controller) PluginKeyProvider() bufplugin.PluginKeyProvider {
712+
return c.pluginKeyProvider
713+
}
714+
715+
func (c *controller) PluginDataProvider() bufplugin.PluginDataProvider {
716+
return c.pluginDataProvider
717+
}
718+
698719
func (c *controller) getImage(
699720
ctx context.Context,
700721
input string,

‎private/buf/bufmigrate/migrator.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/bufbuild/buf/private/bufpkg/bufconfig"
3030
"github.com/bufbuild/buf/private/bufpkg/bufmodule"
3131
"github.com/bufbuild/buf/private/bufpkg/bufparse"
32+
"github.com/bufbuild/buf/private/bufpkg/bufplugin"
3233
"github.com/bufbuild/buf/private/pkg/normalpath"
3334
"github.com/bufbuild/buf/private/pkg/slicesext"
3435
"github.com/bufbuild/buf/private/pkg/storage"
@@ -400,6 +401,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
400401
bufLock, err = bufconfig.NewBufLockFile(
401402
bufconfig.FileVersionV2,
402403
resolvedLockEntries,
404+
nil,
403405
)
404406
if err != nil {
405407
return nil, nil, err
@@ -444,6 +446,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
444446
bufLock, err = bufconfig.NewBufLockFile(
445447
bufconfig.FileVersionV2,
446448
resolvedDepModuleKeys,
449+
nil, // Plugins are not supported in v1.
447450
)
448451
if err != nil {
449452
return nil, nil, err
@@ -693,7 +696,7 @@ func equivalentCheckConfigInV2(
693696
) (bufconfig.CheckConfig, error) {
694697
// No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
695698
// TODO: If we ever need v3, then we will have to deal with this.
696-
client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime))
699+
client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider))
697700
if err != nil {
698701
return nil, err
699702
}

0 commit comments

Comments
 (0)