Merge pull request #1511 from googleapis/feat/migrate-generator-no-dart

feat: migrate generator (excluding dart) from google-cloud-rust to internal/sidekick

Author: julieqiu

all_test.go

@@ -72,6 +72,10 @@ func TestHeaders(t *testing.T) {
 			return nil
 		case strings.HasSuffix(path, ".excalidraw"):
 			return nil
+		case strings.HasPrefix(path, "internal/sidekick"):
+			// TODO(https://github.com/googleapis/librarian/issues/1510): fix
+			// tests for sidekick and remove this case statement.
+			return nil
 		case slices.Contains(noHeaderRequiredFiles, path):
 			return nil
 		default:
@@ -154,6 +158,11 @@ func TestExportedSymbolsHaveDocs(t *testing.T) {
 			strings.HasSuffix(path, "_test.go") || strings.HasSuffix(path, ".pb.go") {
 			return nil
 		}
+		// TODO(https://github.com/googleapis/librarian/issues/1510): fix docs
+		// for sidekick and remove
+		if strings.HasPrefix(path, "internal/sidekick") {
+			return nil
+		}
 
 		fset := token.NewFileSet()
 		node, err := parser.ParseFile(fset, path, nil, parser.ParseComments)

internal/sidekick/README.md

@@ -0,0 +1,76 @@
+# sidekick: a tool to generate and maintain Google Cloud SDKs
+
+`sidekick` automates (or will soon automate) most activities around generating
+and maintaining SDKs for Google Cloud.
+
+## Example Run with Protobuf
+
+You need to have `protoc` installed in your path. You can find useful links
+below.
+
+This will generate the client library for [Secret Manager] in the
+`generator/testdata/rust/openapi/golden` directory. In future releases most
+options should be already configured in a `.sidekick.toml` file.
+
+```bash
+cd generator
+go run cmd/sidekick/main.go generate -project-root=.. \
+  -specification-format protobuf \
+  -specification-source generator/testdata/googleapis/google/cloud/secretmanager/v1 \
+  -service-config generator/testdata/googleapis/google/cloud/secretmanager/v1/secretmanager_v1.yaml \
+  -source-option googleapis-root=generator/testdata/googleapis \
+  -language rust \
+  -output generator/testdata/rust/protobuf/golden/secretmanager \
+  -codec-option package-name-override=secretmanager-golden-protobuf \
+  -codec-option package:wkt=package=types,path=types,source=google.protobuf \
+  -codec-option package:gax=package=gax,path=gax,feature=unstable-sdk-client \
+  -codec-option package:iam=package=iam-v1-golden-protobuf,path=generator/testdata/rust/protobuf/golden/iam/v1,source=google.iam.v1
+```
+
+## Example Run with OpenAPI
+
+This will generate the client library for [Secret Manager] in the
+`generator/testdata/rust/openapi/golden` directory. In future releases most
+options should be already configured in a `.sidekick.toml` file.
+
+```bash
+cd generator
+go run cmd/sidekick/main.go generate -project-root=.. \
+  -specification-format openapi \
+  -specification-source generator/testdata/openapi/secretmanager_openapi_v1.json \
+  -service-config generator/testdata/googleapis/google/cloud/secretmanager/v1/secretmanager_v1.yaml \
+  -language rust \
+  -output generator/testdata/rust/openapi/golden \
+  -codec-option package-name-override=secretmanager-golden-openapi \
+  -codec-option package:wkt=package=types,path=types,source=google.protobuf \
+  -codec-option package:gax=package=gax,path=gax,feature=unstable-sdk-client
+```
+
+## Testing
+
+From the repo root: `go -C generator/ test ./...`
+
+Or from `generator/`: `go test ./...`
+
+## Prerequisites
+
+### Installing `protoc`: the Protobuf Compiler
+
+The generator and its unit tests use `protoc`, the Protobuf compiler. Ensure you
+have `protoc >= v23.0` installed and it is found via your `$PATH`.
+
+```bash
+protoc --version
+```
+
+If not, follow the steps in [Protocol Buffer Compiler Installation] to download
+a suitable version.
+
+### Install goimports
+
+```shell
+go install golang.org/x/tools/cmd/goimports@latest
+```
+
+[protocol buffer compiler installation]: https://protobuf.dev/installation/
+[secret manager]: https://cloud.google.com/secret-manager/

internal/sidekick/cmd/sidekick/main.go

@@ -0,0 +1,29 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/googleapis/google-cloud-rust/generator/internal/sidekick"
+)
+
+func main() {
+	if err := sidekick.Run(os.Args[1:]); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

internal/sidekick/cmd/sidekick/main_test.go

@@ -0,0 +1,26 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Exit(m.Run())
+}

internal/sidekick/doc/contributor/howto-guide-new-sidekick-codec.md

@@ -0,0 +1,131 @@
+# How-To Guide: Implementing a new Codec
+
+This guide is intended for contributors to the `sidekick` project. It will walk
+you through the steps necessary to add a new `codec`. If you are just looking
+for instructions on how to use the `sidekick` consult the top-level README or
+the specific guide for your SDK.
+
+## What is a `codec`
+
+In the context of `sidekick`: a golang package that outputs the syntax tree for
+one specific "API". The most common case is to generate a client library (a
+GAPIC) for a specific SDK. As usual with software, the same abstraction can be
+used to output other things, like a text representation of the API syntax tree,
+or a CLI, or maybe a partial client library that is wrapped by a veneer.
+
+The basic data flows is:
+
+- Sidekick parses the source specification of an API (Protobuf or OpenAPIv3) and
+  converts this into a language-neutral, source-neutral abstract syntax tree. We
+  often call this syntax tree "the model".
+- Sidekick then calls the `Generate()` function in a codec, and passes the model
+  (as a `api.API` data structure, and some configuration options).
+- The codec annotates the syntax tree. Each node in the tree has a `Codec`
+  field. The codec is expected to put its own data structure in that node.
+- The code then loads a number of mustache templates and calls these templates
+  to output the `api.API` data structure as described by these templates.
+
+## What are the contents of an `api.API`?
+
+The elements to be generated are in `api.API.Messages`, `api.API.Enums` and
+`api.API.Services`. Note that `Services` is a list, and typically there may be
+more than one service (in the "gRPC service" sense in the same `api.API`
+invocation).
+
+Sidekick requires that all the elements in an `api.API` are in the same
+namespace. That is, all the elements in the generation lists must have the same
+`.Package` value in their field, and none may have the same `.ID` field.
+
+This is trivial for OpenAPI. For Protobuf this implies all the root messages,
+root enums, and services must be part of the same package. If you need to
+generate multiple packages that will require different calls to `Generate()`.
+
+To support mixins, `api.API` may reference messages, enums and and services.
+These are found in the `api.API.State.*ByID` maps.
+
+## How does sidekick gets its configuration?
+
+In a SDK repository the client libraries are generated in separate directories.
+Each directory contains a `.sidekick.toml` file. This file describes where to
+find the source for the client library, and may include some extra configuration
+for the codec. The top-level directory contains a `.sidekick.toml` configuration
+file which applies to all the generated client libraries.
+
+## How is `api.API` built?
+
+The `.sidekick.toml` file describes what parser to use for that directory, and
+where to find the source. The most common parser is `protobuf`. This parser
+typically receives a directory as the source, the parser uses all the `.proto`
+files in that directory (without recursion). It is also possible to list
+specific files, or to exclude some files.
+
+The parser calls `protoc`, reads the resulting proto descriptors and the service
+config YAML, and then converts that into a `api.API` instance. This is the input
+into your codec.
+
+## Implementing a `codec`
+
+First, add a module, something like this would do:
+
+```shell
+ls -l generator/internal/codec_sample
+```
+
+The main entry point in that module is the `Generate()` function, found in
+`generate.go`:
+
+```shell
+cat generator/internal/codec_sample/generate.go
+```
+
+This module has a single (and relatively simple) template file:
+
+```shell
+cat generator/internal/codec_sample/templates/readme/README.md.mustache
+```
+
+The codec is invoked from a single point in sidekick:
+
+```shell
+git grep -A 2 -B 2 codec_sample.Generate
+```
+
+## Basic integration tests
+
+A simple integration test for this module is found in:
+
+```shell
+cat generator/internal/sidekick/sidekick_sample_test.go
+```
+
+## Unit tests
+
+You can write tests for the codec as usual. There are some helpers to initialize
+the `api.API` data structure. Look at the other codecs for examples.
+
+## Annotations
+
+Eventually your codec will need to add annotations to the `api.API` structure. A
+simple annotation may be a boolean indicating if an API has any services. If you
+needed such an annotation you would write your own `annotate()` function:
+
+```go
+type modelAnnotation {
+  HasServices bool
+}
+
+func annotate(model *api.API) {
+  model.Codec = &modelAnnotation{
+    HasServices: len(model.Services) > 0
+  }
+}
+```
+
+You would need to call this function from your `Generate()` function. Note that
+only the `.Codec` field is modified, and that the type is of your own choosing.
+
+In a more interesting codec you would iterate over the `model` data structure
+and set the `.Codec` field with annotations about methods, messages, enums, and
+so forth. Look at the other codecs for examples. The most common annotations are
+the names of the generated elements, as these often differ in non-trivial ways
+from the source name.

internal/sidekick/go.mod

@@ -0,0 +1,38 @@
+module github.com/googleapis/google-cloud-rust/generator
+
+go 1.23.6
+
+require (
+	cloud.google.com/go/iam v1.5.2
+	cloud.google.com/go/longrunning v0.6.7
+	github.com/cbroglie/mustache v1.4.0
+	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-cmp v0.7.0
+	github.com/iancoleman/strcase v0.3.0
+	github.com/pb33f/libopenapi v0.23.0
+	github.com/pelletier/go-toml/v2 v2.2.4
+	github.com/walle/targz v0.0.0-20140417120357-57fe4206da5a
+	github.com/yuin/goldmark v1.7.12
+	google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb
+	google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e
+	google.golang.org/protobuf v1.36.6
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/speakeasy-api/jsonpath v0.6.2 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.9-0.20240815153524-6ea36470d1bd // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	google.golang.org/grpc v1.71.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)