Skip to content

Commit cb11d40

Browse files
iwahbeVenelinMartinov
iwahbe
and
VenelinMartinov
authored
Test sharding - take 2 (#2832)
Follow-up work on #2810. This PR aimed to validate the more complex test optimization strategy of #2810 (pre-calculated perfect allocation; implemented by github.com/pulumi/sharder) by demonstrating the effects of a simpler strategy (random allocation; implemented by github.com/pulumi/shard). Initial testing showed that both strategies were bounded by the 15 minute job [`Test and Lint / test (1.22.x, windows-latest, DEFAULT, 0)`](https://github.com/pulumi/pulumi-terraform-bridge/actions/runs/12770018688/job/35593903461?pr=2832#logs), which is not effected by sharding. Since both approaches appear identical, we will merge the simpler strategy. --------- Co-authored-by: Venelin <[email protected]>
1 parent 1bec71d commit cb11d40

File tree

6 files changed

+179
-996
lines changed

6 files changed

+179
-996
lines changed

‎.github/workflows/build-and-test.yml‎

Lines changed: 97 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ on:
88
required: false
99
env:
1010
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
# Note this needs to match the shard input to the test matrix below as well as pattern in exclude.
12+
# see jobs.test.strategy.matrix.{shard,exclude}
13+
TOTAL_SHARDS: 15
1114

1215
jobs:
1316
test:
@@ -21,6 +24,72 @@ jobs:
2124
go-version: [1.22.x, 1.23.x]
2225
platform: [ubuntu-latest, macos-latest, windows-latest]
2326
feature-flags: ["DEFAULT", "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"]
27+
# Needs to match TOTAL_SHARDS
28+
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
29+
exclude:
30+
# We do not run non-default feature flags on non-ubuntu.
31+
- platform: windows-latest
32+
feature-flags: "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"
33+
- platform: macos-latest
34+
feature-flags: "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"
35+
# Windows and mac test runs do not need to be sharded as they are fast enough.
36+
# In order to do that we will skip all except the 0-th shard.
37+
- platform: windows-latest
38+
shard: 1
39+
- platform: windows-latest
40+
shard: 2
41+
- platform: windows-latest
42+
shard: 3
43+
- platform: windows-latest
44+
shard: 4
45+
- platform: windows-latest
46+
shard: 5
47+
- platform: windows-latest
48+
shard: 6
49+
- platform: windows-latest
50+
shard: 7
51+
- platform: windows-latest
52+
shard: 8
53+
- platform: windows-latest
54+
shard: 9
55+
- platform: windows-latest
56+
shard: 10
57+
- platform: windows-latest
58+
shard: 11
59+
- platform: windows-latest
60+
shard: 12
61+
- platform: windows-latest
62+
shard: 13
63+
- platform: windows-latest
64+
shard: 14
65+
- platform: macos-latest
66+
shard: 1
67+
- platform: macos-latest
68+
shard: 2
69+
- platform: macos-latest
70+
shard: 3
71+
- platform: macos-latest
72+
shard: 4
73+
- platform: macos-latest
74+
shard: 5
75+
- platform: macos-latest
76+
shard: 6
77+
- platform: macos-latest
78+
shard: 7
79+
- platform: macos-latest
80+
shard: 8
81+
- platform: macos-latest
82+
shard: 9
83+
- platform: macos-latest
84+
shard: 10
85+
- platform: macos-latest
86+
shard: 11
87+
- platform: macos-latest
88+
shard: 12
89+
- platform: macos-latest
90+
shard: 13
91+
- platform: macos-latest
92+
shard: 14
2493
runs-on: ${{ matrix.platform }}
2594
steps:
2695
- name: Install pulumi
@@ -39,16 +108,19 @@ jobs:
39108
go-version: ${{ matrix.go-version }}
40109
cache-dependency-path: |
41110
**/go.sum
111+
# disable caching on windows because it's very slow
112+
# see https://github.com/actions/setup-go/issues/495
113+
cache: ${{ matrix.platform != 'windows-latest' }}
42114
- name: export feature flags
43115
run: echo ${{ matrix.feature-flags }}=true >> $GITHUB_ENV
44116
if: ${{ matrix.platform != 'windows-latest' && matrix.feature-flags != 'DEFAULT' }}
45-
- name: export feature flags
46-
run: echo ${{ matrix.feature-flags }}=true >> $env:GITHUB_ENV
47-
if: ${{ matrix.platform == 'windows-latest' && matrix.feature-flags != 'DEFAULT' }}
48117
- name: Build
49118
run: make build
50119
- name: Build PF
51120
run: cd pkg/pf && make build
121+
- name: Shard tests
122+
run: echo "RUN_TEST_CMD=$(go run github.com/pulumi/shard@5b6297aaffa0c06291fb8231968d7a9f4e6832e6 --total ${{ env.TOTAL_SHARDS }} --index ${{ matrix.shard }} --seed 314)" >> $GITHUB_ENV
123+
if: ${{ matrix.platform == 'ubuntu-latest' }}
52124
- name: Test
53125
run: make test
54126
- name: Upload coverage reports to Codecov
@@ -78,3 +150,25 @@ jobs:
78150
version: v1.62
79151
- name: Lint
80152
run: make lint
153+
sentinel:
154+
name: sentinel
155+
if: github.event_name == 'repository_dispatch' ||
156+
github.event.pull_request.head.repo.full_name == github.repository
157+
permissions:
158+
statuses: write
159+
needs:
160+
- test
161+
- lint
162+
runs-on: ubuntu-latest
163+
steps:
164+
- uses: guibranco/github-status-action-v2@0849440ec82c5fa69b2377725b9b7852a3977e76 # v1.1.13
165+
with:
166+
authToken: ${{secrets.GITHUB_TOKEN}}
167+
# Write an explicit status check called "Sentinel" which will only pass if this code really runs.
168+
# This should always be a required check for PRs.
169+
context: 'Sentinel'
170+
description: 'All required checks passed'
171+
state: 'success'
172+
# Write to the PR commit SHA if it's available as we don't want the merge commit sha,
173+
# otherwise use the current SHA for any other type of build.
174+
sha: ${{ github.event.pull_request.head.sha || github.sha }}

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,12 @@ lint:
2727
lint_fix:
2828
go run scripts/build.go fix-lint
2929

30+
RUN_TEST_CMD ?= ./...
3031
test:: install_plugins
3132
@mkdir -p bin
3233
go build -o bin ./internal/testing/pulumi-terraform-bridge-test-provider
3334
PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \
34-
go test -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 2h -parallel ${TESTPARALLELISM} ./...
35+
go test -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 2h -parallel ${TESTPARALLELISM} $(value RUN_TEST_CMD)
3536

3637
# Run tests while accepting current output as expected output "golden"
3738
# tests. In case where system behavior changes intentionally this can

‎pkg/internal/tests/pulcheck/pulcheck.go‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,39 @@ func resourceNeedsUpdate(res *schema.Resource) bool {
5252
return false
5353
}
5454

55+
func copyMap[K comparable, V any](m map[K]V, cp func(V) V) map[K]V {
56+
dst := make(map[K]V, len(m))
57+
for k, v := range m {
58+
dst[k] = cp(v)
59+
}
60+
return dst
61+
}
62+
63+
// WithValidProvider returns a copy of tfp, with all required fields filled with default
64+
// values.
65+
//
5566
// This is an experimental API.
56-
func EnsureProviderValid(t T, tfp *schema.Provider) {
67+
func WithValidProvider(t T, tfp *schema.Provider) *schema.Provider {
68+
if tfp == nil {
69+
return nil
70+
}
71+
72+
// Copy tfp as deep as we will mutate.
73+
{
74+
dst := *tfp // memcopy
75+
dst.ResourcesMap = copyMap(tfp.ResourcesMap, func(v *schema.Resource) *schema.Resource {
76+
cp := *v // memcopy
77+
cp.Schema = copyMap(cp.Schema, func(s *schema.Schema) *schema.Schema {
78+
cp := *s
79+
return &cp
80+
})
81+
return &cp
82+
})
83+
tfp = &dst
84+
}
85+
86+
// Now ensure that tfp is valid
87+
5788
for _, r := range tfp.ResourcesMap {
5889
if r.Schema["id"] == nil {
5990
r.Schema["id"] = &schema.Schema{
@@ -108,6 +139,8 @@ func EnsureProviderValid(t T, tfp *schema.Provider) {
108139
}
109140
}
110141
require.NoError(t, tfp.InternalValidate())
142+
143+
return tfp
111144
}
112145

113146
func ProviderServerFromInfo(
@@ -206,7 +239,7 @@ func BridgedProvider(t T, providerName string, tfp *schema.Provider, opts ...Bri
206239
opt(&options)
207240
}
208241

209-
EnsureProviderValid(t, tfp)
242+
tfp = WithValidProvider(t, tfp)
210243

211244
// If the PULUMI_ACCURATE_BRIDGE_PREVIEWS environment variable is set, use it to enable
212245
// accurate bridge previews.
@@ -230,10 +263,8 @@ func BridgedProvider(t T, providerName string, tfp *schema.Provider, opts ...Bri
230263
EnableAccurateBridgePreview: accurateBridgePreviews,
231264
Config: options.configInfo,
232265
}
233-
makeToken := func(module, name string) (string, error) {
234-
return tokens.MakeStandard(providerName)(module, name)
235-
}
236-
provider.MustComputeTokens(tokens.SingleModule(providerName, "index", makeToken))
266+
provider.MustComputeTokens(tokens.SingleModule(providerName,
267+
"index", tokens.MakeStandard(providerName)))
237268

238269
return provider
239270
}

‎pkg/tests/import_test.go‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,54 +16,57 @@ import (
1616
"gopkg.in/yaml.v3"
1717

1818
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/tests/pulcheck"
19+
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
1920
)
2021

2122
func TestFullyComputedNestedAttribute(t *testing.T) {
2223
t.Parallel()
23-
resMap := map[string]*schema.Resource{
24-
"prov_test": {
25-
Schema: map[string]*schema.Schema{
26-
"attached_disks": {
27-
Type: schema.TypeList,
28-
Optional: true,
29-
Elem: &schema.Resource{
30-
Schema: map[string]*schema.Schema{
31-
"name": {
32-
Optional: true,
33-
Type: schema.TypeString,
34-
},
35-
"key256": {
36-
Computed: true,
37-
Type: schema.TypeString,
24+
25+
bridgedProvider := func(importVal any) info.Provider {
26+
return pulcheck.BridgedProvider(t, "prov", &schema.Provider{
27+
ResourcesMap: map[string]*schema.Resource{
28+
"prov_test": {
29+
Schema: map[string]*schema.Schema{
30+
"attached_disks": {
31+
Type: schema.TypeList,
32+
Optional: true,
33+
Elem: &schema.Resource{
34+
Schema: map[string]*schema.Schema{
35+
"name": {
36+
Optional: true,
37+
Type: schema.TypeString,
38+
},
39+
"key256": {
40+
Computed: true,
41+
Type: schema.TypeString,
42+
},
43+
},
3844
},
3945
},
46+
"top_level_computed": {
47+
Type: schema.TypeString,
48+
Computed: true,
49+
},
4050
},
41-
},
42-
"top_level_computed": {
43-
Type: schema.TypeString,
44-
Computed: true,
45-
},
46-
},
47-
},
48-
}
49-
50-
importer := func(val any) func(context.Context, *schema.ResourceData, interface{}) ([]*schema.ResourceData, error) {
51-
return func(ctx context.Context, rd *schema.ResourceData, i interface{}) ([]*schema.ResourceData, error) {
52-
elMap := map[string]any{
53-
"name": "disk1",
54-
"key256": val,
55-
}
56-
err := rd.Set("attached_disks", []map[string]any{elMap})
57-
require.NoError(t, err)
51+
Importer: &schema.ResourceImporter{
52+
StateContext: func(ctx context.Context, rd *schema.ResourceData, i interface{}) ([]*schema.ResourceData, error) {
53+
elMap := map[string]any{
54+
"name": "disk1",
55+
"key256": importVal,
56+
}
57+
err := rd.Set("attached_disks", []map[string]any{elMap})
58+
require.NoError(t, err)
5859

59-
err = rd.Set("top_level_computed", "computed_val")
60-
require.NoError(t, err)
60+
err = rd.Set("top_level_computed", "computed_val")
61+
require.NoError(t, err)
6162

62-
return []*schema.ResourceData{rd}, nil
63-
}
63+
return []*schema.ResourceData{rd}, nil
64+
},
65+
},
66+
},
67+
},
68+
})
6469
}
65-
tfp := &schema.Provider{ResourcesMap: resMap}
66-
bridgedProvider := pulcheck.BridgedProvider(t, "prov", tfp)
6770

6871
program := `
6972
name: test
@@ -83,9 +86,7 @@ runtime: yaml
8386
},
8487
} {
8588
t.Run(tc.name, func(t *testing.T) {
86-
resMap["prov_test"].Importer = &schema.ResourceImporter{
87-
StateContext: importer(tc.importVal),
88-
}
89+
bridgedProvider := bridgedProvider(tc.importVal)
8990

9091
pt := pulcheck.PulCheck(t, bridgedProvider, program)
9192

‎pkg/tests/tfcheck/tfcheck.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func NewTfDriver(t pulcheck.T, dir, providerName string, prov any) *TFDriver {
7676
}
7777

7878
func newTfDriverSDK(t pulcheck.T, dir, providerName string, prov *schema.Provider) *TFDriver {
79-
pulcheck.EnsureProviderValid(t, prov)
79+
prov = pulcheck.WithValidProvider(t, prov)
8080
v6server, err := tf5to6server.UpgradeServer(context.Background(),
8181
func() tfprotov5.ProviderServer { return prov.GRPCProvider() })
8282
require.NoError(t, err)

0 commit comments

Comments
 (0)