Skip to content

Commit e1633ef

Browse files
authored
Release Automation Tooling (#1226)
* Go Module Generation for Smithy Artifacts * Tooling for updating module dependency requirements * Tooling for calculating releases * Changelog tool for annotating module changes
1 parent ea88d01 commit e1633ef

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+6099
-153
lines changed

Makefile

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,24 @@ ENDPOINT_PREFIX_JSON=${CODEGEN_RESOURCES_PATH}/endpoint-prefix.json
3838

3939
LICENSE_FILE=$(shell pwd)/LICENSE.txt
4040

41+
RELEASE_MANIFEST_FILE ?=
42+
RELEASE_CHGLOG_DESC_FILE ?=
43+
44+
.PHONY: all
4145
all: generate unit
4246

4347
###################
4448
# Code Generation #
4549
###################
46-
generate: smithy-generate gen-config-asserts copy-attributevalue-feature gen-repo-mod-replace gen-mod-dropreplace-smithy min-go-version-. tidy-modules-. add-module-license-files gen-aws-ptrs
50+
.PHONY: generate smithy-generate smithy-build smithy-build-% smithy-clean smithy-go-publish-local format \
51+
gen-config-asserts gen-repo-mod-replace gen-mod-replace-smithy gen-mod-dropreplace-smithy gen-aws-ptrs tidy-modules-% \
52+
add-module-license-files sync-models sync-endpoints-model sync-endpoints.json clone-v1-models gen-endpoint-prefix.json \
53+
sync-api-models copy-attributevalue-feature min-go-version-% update-requires smithy-annotate-stable \
54+
update-module-metadata
55+
56+
generate: smithy-generate update-requires update-module-metadata smithy-annotate-stable gen-config-asserts \
57+
copy-attributevalue-feature gen-repo-mod-replace gen-mod-dropreplace-smithy min-go-version-. tidy-modules-. \
58+
add-module-license-files gen-aws-ptrs format
4759

4860
smithy-generate:
4961
cd codegen && ./gradlew clean build -Plog-tests && ./gradlew clean
@@ -61,6 +73,10 @@ smithy-build-%: gen-repo-mod-replace
6173
cd codegen && \
6274
SMITHY_GO_BUILD_API="$(subst smithy-build-,,$@)" ./gradlew clean build -Plog-tests
6375

76+
smithy-annotate-stable:
77+
cd ./internal/repotools && \
78+
go run ./cmd/annotatestablegen
79+
6480
smithy-clean:
6581
cd codegen && ./gradlew clean
6682

@@ -69,6 +85,9 @@ smithy-go-publish-local:
6985
git clone https://github.com/aws/smithy-go /tmp/smithy-go-local
7086
make -C /tmp/smithy-go-local smithy-clean smithy-publish-local
7187

88+
format:
89+
gofmt -w -s .
90+
7291
gen-config-asserts:
7392
@echo "Generating SDK config package implementor assertions"
7493
cd config \
@@ -152,10 +171,19 @@ min-go-version-%:
152171
&& go run . -p $(subst _,/,$(subst min-go-version-,,$@)) ${EACHMODULE_FLAGS} \
153172
"go mod edit -go=${SDK_MIN_GO_VERSION}"
154173

174+
update-requires:
175+
cd ./internal/repotools && \
176+
go run ./cmd/updaterequires
177+
178+
update-module-metadata:
179+
cd ./internal/repotools && \
180+
go run ./cmd/updatemodulemeta
155181

156182
################
157183
# Unit Testing #
158184
################
185+
.PHONY: unit unit-race unit-test unit-race-test unit-race-modules-% unit-modules-% build build-modules-% \
186+
go-build-modules-% test test-race-modules-% test-modules-% cachedep cachedep-modules-% api-diff-modules-%
159187

160188
unit: lint unit-modules-.
161189
unit-race: lint unit-race-modules-.
@@ -264,6 +292,8 @@ api-diff-modules-%:
264292
##############
265293
# CI Testing #
266294
##############
295+
.PHONY: ci-test ci-test-no-generate ci-test-generate-validate
296+
267297
ci-test: generate unit-race ci-test-generate-validate
268298
ci-test-no-generate: unit-race
269299

@@ -279,6 +309,8 @@ ci-test-generate-validate:
279309
#######################
280310
# Integration Testing #
281311
#######################
312+
.PHONY: integration integ-modules-% cleanup-integ-buckets
313+
282314
integration: integ-modules-service
283315

284316
integ-modules-%:
@@ -298,6 +330,8 @@ cleanup-integ-buckets:
298330
##############
299331
# Benchmarks #
300332
##############
333+
.PHONY: bench bench-modules-%
334+
301335
bench: bench-modules-.
302336

303337
bench-modules-%:
@@ -310,9 +344,47 @@ bench-modules-%:
310344
&& go run . -p $(subst _,/,$(subst bench-modules-,,$@)) ${EACHMODULE_FLAGS} \
311345
"go test -timeout=10m -bench . --benchmem ${BUILD_TAGS} ${RUN_NONE} ./..."
312346

347+
348+
#####################
349+
# Release Process #
350+
#####################
351+
.PHONY: preview-release pre-release-validation release
352+
353+
preview-release:
354+
@cd ./internal/repotools && \
355+
go run ./cmd/calculaterelease
356+
357+
pre-release-validation:
358+
@if [[ -z "${RELEASE_MANIFEST_FILE}" ]]; then \
359+
echo "RELEASE_MANIFEST_FILE is required to specify the file to write the release manifest" && false; \
360+
fi
361+
@if [[ -z "${RELEASE_CHGLOG_DESC_FILE}" ]]; then \
362+
echo "RELEASE_CHGLOG_DESC_FILE is required to specify the file to write the release notes" && false; \
363+
fi
364+
365+
release: pre-release-validation
366+
cd ./internal/repotools && \
367+
go run ./cmd/calculaterelease -o ${RELEASE_MANIFEST_FILE} && \
368+
go run ./cmd/updaterequires -release ${RELEASE_MANIFEST_FILE} && \
369+
go run ./cmd/updatemodulemeta -release ${RELEASE_MANIFEST_FILE} && \
370+
go run ./cmd/generatechangelog -release ${RELEASE_MANIFEST_FILE} -o ${RELEASE_CHGLOG_DESC_FILE} && \
371+
go run ./cmd/changelog rm -all && \
372+
go run ./cmd/tagrelease -release ${RELEASE_MANIFEST_FILE}
373+
374+
##############
375+
# Repo Tools #
376+
##############
377+
.PHONY: install-repotools
378+
379+
install-repotools:
380+
cd ./internal/repotools && \
381+
go install ./cmd/changelog
382+
313383
##################
314384
# Linting/Verify #
315385
##################
386+
.PHONY: verify lint vet vet-modules-% sdkv1check
387+
316388
verify: lint vet sdkv1check
317389

318390
lint:
@@ -339,6 +411,9 @@ sdkv1check:
339411
###################
340412
# Sandbox Testing #
341413
###################
414+
.PHONY: sandbox-tests sandbox-build-go1.15 sandbox-go1.15 sandbox-test-go1.15 sandbox-build-go1.16 \
415+
sandbox-go1.16 sandbox-test-go1.16 sandbox-build-gotip sandbox-gotip sandbox-test-gotip update-aws-golang-tip
416+
342417
sandbox-tests: sandbox-test-go1.15 sandbox-test-go1.16 sandbox-test-gotip
343418

344419
sandbox-build-go1.15:

codegen/copy_go_codegen.sh

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,9 @@
1-
#!/bin/sh
1+
#!/usr/bin/env bash
2+
3+
set -xe
24

35
SDK_ROOT=$1
46
CODGEN_ROOT=$2
5-
SDK_PREFIX=$3
6-
7-
for MOD_PATH in $(find ${CODGEN_ROOT} -type f -name "go.mod" | xargs -I {} dirname {})
8-
do
9-
cd ${MOD_PATH}
10-
MOD=$(grep -e "^module" go.mod | sed -e 's/^module *//')
11-
12-
DST=${SDK_ROOT}/${MOD#"$SDK_PREFIX"}
13-
14-
echo "Copying ${MOD} to ${DST}"
157

16-
rm ${DST}/*.go
17-
rm -rf ${DST}/types
18-
rm -rf ${DST}/internal/endpoints
19-
mkdir -p ${DST} 2>/dev/null
20-
cp -r . ${DST}
21-
done
8+
cd "$1"/internal/repotools
9+
go run ./cmd/gomodgen -build "$CODGEN_ROOT"

codegen/protocol-test-codegen/build.gradle.kts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ tasks["build"].finalizedBy(tasks["buildSdk"])
4040

4141
// ensure built artifacts are put into the SDK's folders
4242
tasks.create<Exec>("copyGoCodegen") {
43-
dependsOn ("buildSdk")
44-
commandLine ("$rootDir/copy_go_codegen.sh", "$rootDir/../", "$buildDir", "github.com/aws/aws-sdk-go-v2/")
43+
dependsOn("buildSdk")
44+
commandLine("$rootDir/copy_go_codegen.sh", "$rootDir/..", (tasks["buildSdk"] as SmithyBuild).outputDirectory.absolutePath)
4545
}
4646
tasks["buildSdk"].finalizedBy(tasks["copyGoCodegen"])

codegen/sdk-codegen/build.gradle.kts

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,15 @@ import software.amazon.smithy.model.Model
1717
import software.amazon.smithy.model.node.Node
1818
import software.amazon.smithy.model.shapes.ServiceShape
1919
import software.amazon.smithy.gradle.tasks.SmithyBuild
20+
import software.amazon.smithy.aws.traits.ServiceTrait
2021
import kotlin.streams.toList
2122

23+
buildscript {
24+
dependencies {
25+
"classpath"("software.amazon.smithy:smithy-aws-traits:[1.5.1,2.0.0[")
26+
}
27+
}
28+
2229
plugins {
2330
id("software.amazon.smithy") version "0.5.3"
2431
}
@@ -70,18 +77,21 @@ tasks.register("generate-smithy-build") {
7077
}
7178
}
7279

73-
var (sdkId, version, remaining) = file.name.split(".")
74-
sdkId = sdkId.replace("-", "").toLowerCase();
80+
val serviceTrait = service.getTrait(ServiceTrait::class.javaObjectType).get();
81+
82+
val sdkId = serviceTrait.sdkId
83+
.replace("-", "")
84+
.replace(" ", "")
85+
.toLowerCase();
7586
val projectionContents = Node.objectNodeBuilder()
76-
.withMember("imports", Node.fromStrings("${models.getAbsolutePath()}${File.separator}${file.name}"))
87+
.withMember("imports", Node.fromStrings("${models.absolutePath}${File.separator}${file.name}"))
7788
.withMember("plugins", Node.objectNode()
7889
.withMember("go-codegen", Node.objectNodeBuilder()
7990
.withMember("service", service.id.toString())
80-
.withMember("module", "github.com/aws/aws-sdk-go-v2/service/" + sdkId.toLowerCase())
81-
.withMember("moduleVersion", "1.0")
91+
.withMember("module", "github.com/aws/aws-sdk-go-v2/service/$sdkId")
8292
.build()))
8393
.build()
84-
projectionsBuilder.withMember(sdkId + "." + version.toLowerCase(), projectionContents)
94+
projectionsBuilder.withMember(sdkId + "." + service.version.toLowerCase(), projectionContents)
8595
}
8696

8797
file("smithy-build.json").writeText(Node.prettyPrintJson(Node.objectNodeBuilder()
@@ -99,6 +109,6 @@ tasks["build"]
99109
// ensure built artifacts are put into the SDK's folders
100110
tasks.create<Exec>("copyGoCodegen") {
101111
dependsOn ("buildSdk")
102-
commandLine ("$rootDir/copy_go_codegen.sh", "$rootDir/../", "$buildDir", "github.com/aws/aws-sdk-go-v2/")
112+
commandLine ("$rootDir/copy_go_codegen.sh", "$rootDir/..", (tasks["buildSdk"] as SmithyBuild).outputDirectory.absolutePath)
103113
}
104114
tasks["buildSdk"].finalizedBy(tasks["copyGoCodegen"])

internal/repotools/README.md

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Repository Management Tools
2+
3+
This Go module is a collection of tools that have been written for managing the AWS SDK for Go V2 source repository.
4+
At the time these were written, the Go ecosystem lacked tooling for managing repositories containing multiple Go modules
5+
with at the size and scale of the AWS SDK. With over 274 Go modules in the repository the tooling found here has been
6+
made to manage the lifecycle of managing dependencies (iter-repository and external), managing the releases of new
7+
changes, tag management following Go semver requirements, and production of changelogs.
8+
9+
## Utilities
10+
The following is a breakdown of some the key utilities found in this module that are used to manage the SDK and handle
11+
the complex release process.
12+
13+
Commands | Description | README
14+
--- | --- | ---
15+
`changelog` | Create and manage changelog annotations. Annotations are used to document module changes and refining of the next semver version. | [Link][changelog]
16+
`updaterequires` | Manages `go.mod` require entries, allows for easily updating inter-repository module dependencies to their latest tag, and the ability to quickly manage external dependency requirements. | N/A
17+
`updatemodulemeta` | Generates a `go_module_metadata.go` file in each module containing useful runtime metadata like the modules tagged version. | N/A
18+
`generatechangelog` | Uses a release description and associated changelog annotations to produce `CHANGELOG.md` entries for the release in each repository module. In addition, a summarized release statement will be created at the root of the repository. | N/A
19+
`gomodgen` | Copies [smithy-go] codegen build artifacts into the SDK repository and generates a `go.mod` file using the build artifacts `generated.json` description. | N/A
20+
`annotatestablegen` | Generates a release changelog annotation type for **new** [smithy-go] generated modules that are not marked as unstable. | N/A
21+
`calculaterelease` | Detects new and changed Go modules in the repository, associates changelog annotations, and computes the next semver version tag for each module. Produces a release manifest that is used with other utilities to orchestrate a release. | [Link][calculaterelease]
22+
`tagrelease` | Commits pending changes to the working directory, reads the release manifest, and creates the computed tags | N/A
23+
`makerelative` | Used to generate `go.mod` `replace` statements for inter-repository module dependencies. This ensures that when developing on a given Go module it's iter-repository dependencies refer to the cloned repository. | N/A
24+
`eachmodule` | Utility for quickly scripting execution of commands in each module of a repository. | N/A
25+
26+
# Configuration
27+
28+
A number of the repository tools, specifically those involved with the dependency management and release have specific
29+
behavior that is driven by the `modman.toml` file found at the root of the git repository. This configuration file is
30+
a [TOML] configuration file.
31+
32+
## Dependencies
33+
The `dependencies` is a dictionary of key-value pairs that describe **external** dependencies that one or more modules
34+
within the repository may depend on. (External dependencies is defined as the set of Go modules that are not found
35+
within the project git repository.) This section is used to quickly set the version of a dependency modules in the
36+
repository should use. The `updaterequires` tool can be used to update all Go modules require statements for each module
37+
in the repository and update them to the indicated version if they depend on the given external module.
38+
39+
### Example
40+
```toml
41+
[dependencies]
42+
"github.com/aws/smithy-go" = "v1.4.0"
43+
```
44+
45+
This example indicates that repository modules that depend on `github.com/aws/smithy-go` should depend on `v1.4.0`
46+
version of the library. After updating the value in the configuration file, `updaterequires` can be used to update
47+
modules with this information.
48+
49+
## Modules
50+
51+
`modules` is a dictionary where the keys are module directories relative to the repository root. Each key maps to a
52+
dictionary of key-value pairs that can configure several properties of a module that affect how or
53+
if a module is handled when performing a release.
54+
55+
### Example
56+
To configure the module `feature/service/shinything` to not be tagged by the release process:
57+
58+
```toml
59+
[modules."feature/service/shinything"]
60+
no_tag = true
61+
```
62+
63+
For more information on how to configure how modules are released see the [calculaterelease README][calculaterelease].
64+
65+
#### Mark a module to be
66+
67+
**NOTE**: If you wish to create a configuration item for a module located at the root of the repository use
68+
`.` as the key name.
69+
70+
[calculaterelease]: cmd/calculaterelease/README.md
71+
[changelog]: cmd/changelog/README.md
72+
[smithy-go]: https://github.com/aws/smithy-go
73+
[TOML]: https://toml.io

0 commit comments

Comments
 (0)