Skip to content

Commit 1029bcb

Browse files
BBBmauBBBmau
committed
Merge branch 'sync-FEATURE-BRANCH-resource-identity' into FEATURE-BRANCH-resource-identity
2 parents f32819a + 6d9dd95 commit 1029bcb

File tree

585 files changed

+35429
-4877
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

585 files changed

+35429
-4877
lines changed

‎.ci/changelog2.tmpl‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{{- if .NotesByType.unknown -}}
2+
UNKNOWN CHANGELOG TYPE:
3+
{{range .NotesByType.unknown -}}
4+
* {{ template "note" .}}
5+
{{ end -}}
6+
{{- end -}}
7+
8+
{{- if .NotesByType.note -}}
9+
NOTES:
10+
{{range .NotesByType.note -}}
11+
* {{ template "note" .}}
12+
{{ end -}}
13+
{{- end -}}
14+
15+
{{- if .NotesByType.deprecation -}}
16+
DEPRECATIONS:
17+
{{range .NotesByType.deprecation -}}
18+
* {{ template "note" .}}
19+
{{ end -}}
20+
{{- end -}}
21+
22+
{{- if index .NotesByType "breaking-change" -}}
23+
BREAKING CHANGES:
24+
{{range index .NotesByType "breaking-change" -}}
25+
* {{ template "note" .}}
26+
{{ end -}}
27+
{{- end -}}
28+
29+
{{- $features := combineTypes .NotesByType.feature (index .NotesByType "new-resource" ) (index .NotesByType "new-datasource") (index .NotesByType "new-data-source") (index .NotesByType "new-function" ) (index .NotesByType "new-ephemeral" ) -}}
30+
{{- if $features }}
31+
FEATURES:
32+
{{range $features | sort -}}
33+
* {{ template "note" . }}
34+
{{ end -}}
35+
{{- end -}}
36+
37+
{{- $improvements := combineTypes .NotesByType.improvement .NotesByType.enhancement -}}
38+
{{- if $improvements }}
39+
IMPROVEMENTS:
40+
{{range $improvements | sort -}}
41+
* {{ template "note" . }}
42+
{{ end -}}
43+
{{- end -}}
44+
45+
{{- if .NotesByType.bug }}
46+
BUG FIXES:
47+
{{range .NotesByType.bug -}}
48+
* {{ template "note" . }}
49+
{{ end -}}
50+
{{- end -}}

‎.ci/containers/build-environment/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ ENV GO111MODULE "on"
2525
ENV LC_ALL=C.UTF-8
2626
ENV LANG=C.UTF-8
2727

28-
RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client apt-transport-https ca-certificates curl netbase wget gcc make jq libjq1
28+
RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client apt-transport-https ca-certificates curl netbase wget gcc make jq libjq1 unzip zip
2929

3030
RUN git config --global user.name "Modular Magician"
3131
RUN git config --global user.email "magic-modules@google.com"

‎.ci/containers/go-plus/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ COPY --from=builder /go/pkg/mod /go/pkg/mod
2121
COPY --from=builder /go/cache /go/cache
2222

2323
RUN apt-get update && \
24-
apt-get install -y git jq unzip parallel curl && \
24+
apt-get install -y git jq unzip zip parallel curl && \
2525
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
2626
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
2727
apt-get update -y && \
@@ -32,4 +32,4 @@ RUN apt-get update && \
3232
RUN wget https://releases.hashicorp.com/terraform/1.11.0/terraform_1.11.0_linux_amd64.zip \
3333
&& unzip terraform_1.11.0_linux_amd64.zip \
3434
&& rm terraform_1.11.0_linux_amd64.zip \
35-
&& mv ./terraform /bin/terraform
35+
&& mv ./terraform /bin/terraform

‎.ci/infra/terraform/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ module "project-services" {
306306
"migrationcenter.googleapis.com",
307307
"ml.googleapis.com",
308308
"mobilecrashreporting.googleapis.com",
309+
"modelarmor.googleapis.com",
309310
"monitoring.googleapis.com",
310311
"multiclustermetering.googleapis.com",
311312
"netapp.googleapis.com",

‎.ci/magician/cmd/generate_comment.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package cmd
1717

1818
import (
19+
"bytes"
1920
"encoding/json"
2021
"fmt"
2122
"os"
@@ -34,6 +35,9 @@ import (
3435
"github.com/GoogleCloudPlatform/magic-modules/tools/issue-labeler/labeler"
3536

3637
"github.com/spf13/cobra"
38+
"github.com/yuin/goldmark"
39+
"github.com/yuin/goldmark/parser"
40+
"go.abhg.dev/goldmark/frontmatter"
3741
"golang.org/x/exp/maps"
3842

3943
_ "embed"
@@ -329,6 +333,11 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
329333
errors[repo.Title] = append(errors[repo.Title], "The missing doc detector failed to run.")
330334
}
331335
data.MissingDocs = missingDocs
336+
337+
errStrs := checkDocumentFrontmatter(repo)
338+
if len(errStrs) > 0 {
339+
errors[repo.Title] = append(errors[repo.Title], errStrs...)
340+
}
332341
}
333342

334343
simpleDiff, err := computeAffectedResources(diffProcessorPath, rnr, repo)
@@ -636,3 +645,49 @@ func pathChanged(path string, changedFiles []string) bool {
636645
func init() {
637646
rootCmd.AddCommand(generateCommentCmd)
638647
}
648+
649+
// checkDocumentFrontmatter checks changed markdown files' frontmatter
650+
// structure in the repo and returns error strings when applicable.
651+
func checkDocumentFrontmatter(repo source.Repo) []string {
652+
var errs []string
653+
for _, f := range repo.ChangedFiles {
654+
if !strings.HasSuffix(f, ".markdown") {
655+
continue
656+
}
657+
src, err := os.ReadFile(filepath.Join(repo.Path, f))
658+
if err != nil {
659+
errs = append(errs, "Error reading file "+f)
660+
continue
661+
}
662+
663+
md := goldmark.New(
664+
goldmark.WithExtensions(&frontmatter.Extender{}),
665+
)
666+
667+
ctx := parser.NewContext()
668+
var buff bytes.Buffer
669+
670+
err = md.Convert(src, &buff, parser.WithContext(ctx))
671+
if err != nil {
672+
errs = append(errs, "Error parsing file "+f)
673+
continue
674+
}
675+
data := frontmatter.Get(ctx)
676+
if data == nil {
677+
errs = append(errs, fmt.Sprintf("No frontmatter found in file %s. This is usually due to an incorrect structure in the frontmatter.", f))
678+
continue
679+
}
680+
681+
var metadata struct {
682+
Subcategory string
683+
}
684+
if err := data.Decode(&metadata); err != nil {
685+
errs = append(errs, fmt.Sprintf("Failed to decode frontmatter in file %s. This is usually due to an incorrect structure in the frontmatter.", f))
686+
continue
687+
}
688+
if metadata.Subcategory == "" {
689+
errs = append(errs, fmt.Sprintf("Failed to detect subcategory in the frontmatter in file %s.", f))
690+
}
691+
}
692+
return errs
693+
}

‎.ci/magician/cmd/generate_comment_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package cmd
1717

1818
import (
1919
"os"
20+
"path/filepath"
2021
"reflect"
2122
"testing"
2223

@@ -510,3 +511,78 @@ func TestPathChanged(t *testing.T) {
510511
})
511512
}
512513
}
514+
515+
func TestCheckDocumentFrontmatter(t *testing.T) {
516+
tmpDir := t.TempDir()
517+
files := map[string]string{
518+
"malformed.markdown": `
519+
subcategory: Example Subcategory
520+
---
521+
`,
522+
"sample.markdown": `
523+
---
524+
subcategory: Example Subcategory
525+
---
526+
`,
527+
"missingsubcategory.markdown": `
528+
---
529+
random: Example Subcategory
530+
---
531+
`,
532+
}
533+
for name, content := range files {
534+
fullPath := filepath.Join(tmpDir, name)
535+
err := os.WriteFile(fullPath, []byte(content), 0644)
536+
if err != nil {
537+
t.Fatalf("Failed to create file %s: %v", name, err)
538+
}
539+
}
540+
541+
tests := []struct {
542+
name string
543+
changedFiles []string
544+
wantErr bool
545+
}{
546+
{
547+
name: "No changed markdown files",
548+
changedFiles: []string{"abc.txt"},
549+
wantErr: false,
550+
},
551+
{
552+
name: "malformed markdown",
553+
changedFiles: []string{"malformed.markdown"},
554+
wantErr: true,
555+
},
556+
{
557+
name: "not exist markdown",
558+
changedFiles: []string{"abc.markdown"},
559+
wantErr: true,
560+
},
561+
{
562+
name: "Changed files with no frontmatter",
563+
changedFiles: []string{"sample.markdown"},
564+
wantErr: false,
565+
},
566+
{
567+
name: "Missing subcategory in frontmatter",
568+
changedFiles: []string{"missingsubcategory.markdown"},
569+
wantErr: true,
570+
},
571+
}
572+
573+
for _, tc := range tests {
574+
t.Run(tc.name, func(t *testing.T) {
575+
repo := source.Repo{
576+
Path: tmpDir,
577+
ChangedFiles: tc.changedFiles,
578+
}
579+
got := checkDocumentFrontmatter(repo)
580+
if tc.wantErr && len(got) == 0 {
581+
t.Errorf("checkDocumentFrontmatter() = %v, want error", got)
582+
}
583+
if !tc.wantErr && len(got) > 0 {
584+
t.Errorf("checkDocumentFrontmatter() = %v, want no error", got)
585+
}
586+
})
587+
}
588+
}

‎.ci/magician/cmd/generate_downstream.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,12 @@ func addChangelogEntry(downstreamRepo *source.Repo, pullRequest *github.PullRequ
375375
return err
376376
}
377377
}
378+
// If changelog entry is missing, add an entry "unknown: <PR title>".
379+
if matches == nil {
380+
if err := rnr.WriteFile(filepath.Join(".changelog", fmt.Sprintf("%d.txt", pullRequest.Number)), "unknown: "+pullRequest.Title); err != nil {
381+
return err
382+
}
383+
}
378384
return rnr.PopDir()
379385
}
380386

‎.ci/magician/cmd/templates/vcr/non_exercised_tests.tmpl‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎.ci/magician/cmd/templates/vcr/post_replay.tmpl‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{{- if or (gt (len .NotRunBetaTests) 0) (gt (len .NotRunGATests) 0)}}
2+
#### Non-exercised tests
3+
4+
{{if gt (len .NotRunBetaTests) 0 -}}
5+
{{color "red" "Tests were added that are skipped in VCR:"}}
6+
{{range .NotRunBetaTests}}{{. | printf "- %s\n"}}{{end}}
7+
{{end}}
8+
9+
{{if gt (len .NotRunGATests) 0 -}}
10+
{{color "red" "Tests were added that are GA-only additions and require manual runs:"}}
11+
{{range .NotRunGATests}}{{. | printf "- %s\n"}}{{end}}
12+
{{end}}
13+
{{end}}
14+
#### Tests analytics
15+
Total tests: {{add (add (len .ReplayingResult.PassedTests) (len .ReplayingResult.SkippedTests)) (len .ReplayingResult.FailedTests) }}
16+
Passed tests: {{len .ReplayingResult.PassedTests}}
17+
Skipped tests: {{len .ReplayingResult.SkippedTests}}
18+
Affected tests: {{len .ReplayingResult.FailedTests}}
19+
20+
<details>
21+
<summary>Click here to see the affected service packages</summary>
22+
<blockquote>
23+
{{if .RunFullVCR}}
24+
All service packages are affected
25+
{{else if gt (len .AffectedServices) 0}}
26+
<ul>
27+
{{range .AffectedServices}}{{. | printf "<li>%s</li>\n"}}{{end}}
28+
</ul>
29+
{{else}}
30+
None
31+
{{end}}
32+
</blockquote>
33+
</details>
34+
{{ if gt (len .ReplayingResult.FailedTests) 0 -}}
35+
#### Action taken
36+
<details>
37+
<summary>Found {{len .ReplayingResult.FailedTests}} affected test(s) by replaying old test recordings. Starting RECORDING based on the most recent commit. Click here to see the affected tests
38+
</summary>
39+
<blockquote>
40+
<ul>
41+
{{range .ReplayingResult.FailedTests}}{{. | printf "<li>%s</li>\n"}}{{end}}
42+
</ul>
43+
</blockquote>
44+
</details>
45+
46+
[Get to know how VCR tests work](https://googlecloudplatform.github.io/magic-modules/develop/test/test/)
47+
{{ else -}}
48+
{{- if .ReplayingErr -}}
49+
{{color "red" "Errors occurred during REPLAYING mode. Please fix them to complete your PR."}}
50+
{{- else -}}
51+
{{color "green" "All tests passed!"}}
52+
{{- end}}
53+
54+
View the [build log](https://storage.cloud.google.com/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/build-log/replaying_test.log)
55+
{{- end}}

‎.ci/magician/cmd/templates/vcr/record_replay.tmpl‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
{{color "green" "Tests passed during RECORDING mode:"}}
33
{{range .RecordingResult.PassedTests -}}
44
`{{.}}` {{/* remove trailing whitespace */ -}}
5-
[[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
5+
[[Debug log]({{$.LogBaseUrl}}/recording/{{.}}.log)]
66
{{/* remove trailing whitespace */ -}}
77
{{end}}
88

@@ -11,8 +11,8 @@
1111
{{color "red" "Tests failed when rerunning REPLAYING mode:"}}
1212
{{range .ReplayingAfterRecordingResult.FailedTests -}}
1313
`{{.}}` {{/* remove trailing whitespace */ -}}
14-
[[Error message](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/build-log/replaying_build_after_recording/{{.}}_replaying_test.log)] {{/* remove trailing whitespace */ -}}
15-
[[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/replaying_after_recording/{{.}}.log)]
14+
[[Error message]({{$.LogBaseUrl}}/build-log/replaying_build_after_recording/{{compoundTest .}}_replaying_test.log)] {{/* remove trailing whitespace */ -}}
15+
[[Debug log]({{$.LogBaseUrl}}/replaying_after_recording/{{.}}.log)]
1616
{{/* remove trailing whitespace */ -}}
1717
{{end}}
1818

@@ -30,8 +30,8 @@ Please fix these to complete your PR. If you believe these test failures to be i
3030
{{color "red" "Tests failed during RECORDING mode:"}}
3131
{{range .RecordingResult.FailedTests -}}
3232
`{{.}}` {{/* remove trailing whitespace */ -}}
33-
[[Error message](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/build-log/recording_build/{{.}}_recording_test.log)] {{/* remove trailing whitespace */ -}}
34-
[[Debug log](https://storage.cloud.google.com/{{$.LogBucket}}/{{$.Version}}/refs/heads/{{$.Head}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
33+
[[Error message]({{$.LogBaseUrl}}/build-log/recording_build/{{compoundTest .}}_recording_test.log)] {{/* remove trailing whitespace */ -}}
34+
[[Debug log]({{$.LogBaseUrl}}/recording/{{.}}.log)]
3535
{{/* remove trailing whitespace */ -}}
3636
{{end}}
3737
{{end}} {{- /* end of if gt (len .RecordingResult.FailedTests) 0 */ -}}
@@ -42,5 +42,5 @@ Please fix these to complete your PR. If you believe these test failures to be i
4242

4343
{{if .AllRecordingPassed}}{{color "green" "All tests passed!"}}{{end}}
4444

45-
View the [build log](https://storage.cloud.google.com/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/build-log/recording_test.log) {{/* remove trailing whitespace */ -}}
46-
or the [debug log](https://console.cloud.google.com/storage/browser/{{.LogBucket}}/{{.Version}}/refs/heads/{{.Head}}/artifacts/{{.BuildID}}/recording) for each test
45+
View the [build log]({{.LogBaseUrl}}/build-log/recording_test.log) {{/* remove trailing whitespace */ -}}
46+
or the [debug log]({{.BrowseLogBaseUrl}}/recording) for each test

0 commit comments

Comments
 (0)