Skip to content

Commit a8bb914

Browse files
julienduchesnejulienduchesne
authored
Config Generation: Structured Result (#1677)
This result is something we can build upon and it allows the following: - Bubbling up multiple errors: Resource A and Resource B failed, but resource C worked - Ignoring or treating differently some errors (by casting). For example, ResourceErrors are not critical because they only mean that a single resource failed - Building up a single result object as we go along. Collecting success or errors from multiple parts of the generation process
1 parent f39fd6f commit a8bb914

File tree

6 files changed

+162
-75
lines changed

6 files changed

+162
-75
lines changed

‎cmd/generate/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"errors"
45
"fmt"
56
"log"
67
"os"
@@ -167,7 +168,8 @@ This supports a glob format. Examples:
167168
if err != nil {
168169
return fmt.Errorf("failed to parse flags: %w", err)
169170
}
170-
return generate.Generate(ctx.Context, cfg)
171+
result := generate.Generate(ctx.Context, cfg)
172+
return errors.Join(result.Errors...)
171173
},
172174
}
173175

‎internal/resources/slo/resource_slo.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -272,13 +272,7 @@ func listSlos(ctx context.Context, client *common.Client, data any) ([]string, e
272272

273273
slolist, _, err := sloClient.DefaultAPI.V1SloGet(ctx).Execute()
274274
if err != nil {
275-
// // TODO: Uninitialized SLO plugin. This should be handled better
276-
// cast, ok := err.(*slo.GenericOpenAPIError)
277-
// if ok && strings.Contains(cast.Error(), "status: 500") {
278-
// return nil, nil
279-
// }
280-
281-
return nil, nil
275+
return nil, err
282276
}
283277

284278
var ids []string

‎pkg/generate/cloud.go‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -33,32 +33,32 @@ type stack struct {
3333
onCallToken string
3434
}
3535

36-
func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
36+
func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, GenerationResult) {
3737
// Gen provider
3838
providerBlock := hclwrite.NewBlock("provider", []string{"grafana"})
3939
providerBlock.Body().SetAttributeValue("alias", cty.StringVal("cloud"))
4040
providerBlock.Body().SetAttributeValue("cloud_access_policy_token", cty.StringVal(cfg.Cloud.AccessPolicyToken))
4141
if err := writeBlocks(filepath.Join(cfg.OutputDir, "cloud-provider.tf"), providerBlock); err != nil {
42-
return nil, err
42+
return nil, failure(err)
4343
}
4444

4545
// Generate imports
4646
config := provider.ProviderConfig{
4747
CloudAccessPolicyToken: types.StringValue(cfg.Cloud.AccessPolicyToken),
4848
}
4949
if err := config.SetDefaults(); err != nil {
50-
return nil, err
50+
return nil, failure(err)
5151
}
5252

5353
client, err := provider.CreateClients(config)
5454
if err != nil {
55-
return nil, err
55+
return nil, failure(err)
5656
}
5757
cloudClient := client.GrafanaCloudAPI
5858

5959
stacks, _, err := cloudClient.InstancesAPI.GetInstances(ctx).Execute()
6060
if err != nil {
61-
return nil, err
61+
return nil, failure(err)
6262
}
6363

6464
// Cleanup SAs
@@ -67,32 +67,33 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
6767
if cfg.Cloud.CreateStackServiceAccount {
6868
for _, stack := range stacks.Items {
6969
if err := createManagementStackServiceAccount(ctx, cloudClient, stack, managementServiceAccountName); err != nil {
70-
return nil, err
70+
return nil, failure(err)
7171
}
7272
}
7373
}
7474

7575
data := cloud.NewListerData(cfg.Cloud.Org)
76-
if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg, "cloud"); err != nil {
77-
return nil, err
76+
returnResult := generateImportBlocks(ctx, client, data, cloud.Resources, cfg, "cloud")
77+
if returnResult.Blocks() == 0 { // Skip if no resources were found
78+
return nil, returnResult
7879
}
7980

8081
plannedState, err := getPlannedState(ctx, cfg)
8182
if err != nil {
82-
return nil, err
83+
return nil, failure(err)
8384
}
8485
if err := postprocessing.StripDefaults(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
85-
return nil, err
86+
return nil, failure(err)
8687
}
8788
if err := postprocessing.WrapJSONFieldsInFunction(filepath.Join(cfg.OutputDir, "cloud-resources.tf")); err != nil {
88-
return nil, err
89+
return nil, failure(err)
8990
}
9091
if err := postprocessing.ReplaceReferences(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), plannedState, nil); err != nil {
91-
return nil, err
92+
return nil, failure(err)
9293
}
9394

9495
if !cfg.Cloud.CreateStackServiceAccount {
95-
return nil, nil
96+
return nil, returnResult
9697
}
9798

9899
// Add management service account (grafana_cloud_stack_service_account)
@@ -149,7 +150,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
149150
providerBlock.Body().SetAttributeTraversal("sm_url", traversal("grafana_synthetic_monitoring_installation", stack.Slug, "stack_sm_api_url"))
150151

151152
if err := writeBlocks(filepath.Join(cfg.OutputDir, fmt.Sprintf("stack-%s-provider.tf", stack.Slug)), saBlock, saTokenBlock, smInstallationMetricsPublishBlock, smInstallationTokenBlock, smInstallationBlock, providerBlock); err != nil {
152-
return nil, fmt.Errorf("failed to write management service account blocks for stack %q: %w", stack.Slug, err)
153+
return nil, failuref("failed to write management service account blocks for stack %q: %w", stack.Slug, err)
153154
}
154155

155156
// Apply then go into the state and find the management key
@@ -161,14 +162,14 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
161162
tfexec.Target("grafana_synthetic_monitoring_installation."+stack.Slug),
162163
)
163164
if err != nil {
164-
return nil, fmt.Errorf("failed to apply management service account blocks for stack %q: %w", stack.Slug, err)
165+
return nil, failuref("failed to apply management service account blocks for stack %q: %w", stack.Slug, err)
165166
}
166167
}
167168

168169
managedStacks := []stack{}
169170
state, err := getState(ctx, cfg)
170171
if err != nil {
171-
return nil, err
172+
return nil, failure(err)
172173
}
173174
stacksMap := map[string]stack{}
174175
for _, resource := range state.Values.RootModule.Resources {
@@ -198,7 +199,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
198199
managedStacks = append(managedStacks, stack)
199200
}
200201

201-
return managedStacks, nil
202+
return managedStacks, returnResult
202203
}
203204

204205
func createManagementStackServiceAccount(ctx context.Context, cloudClient *gcom.APIClient, stack gcom.FormattedApiInstance, saName string) error {

‎pkg/generate/generate.go‎

Lines changed: 94 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,65 @@ var (
2222
allowedTerraformChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
2323
)
2424

25-
func Generate(ctx context.Context, cfg *Config) error {
25+
// ResourceError is an error that occurred while generating a resource.
26+
// It can be filtered out by the caller if it is not critical that a single resource failed.
27+
type ResourceError struct {
28+
Resource *common.Resource
29+
Err error
30+
}
31+
32+
func (e ResourceError) Error() string {
33+
return fmt.Sprintf("resource %s: %v", e.Resource.Name, e.Err)
34+
}
35+
36+
type GenerationSuccess struct {
37+
Resource *common.Resource
38+
Blocks int
39+
}
40+
41+
type GenerationResult struct {
42+
Success []GenerationSuccess
43+
Errors []error
44+
}
45+
46+
func (r GenerationResult) Blocks() int {
47+
blocks := 0
48+
for _, s := range r.Success {
49+
blocks += s.Blocks
50+
}
51+
return blocks
52+
}
53+
54+
func failure(err error) GenerationResult {
55+
return GenerationResult{
56+
Errors: []error{err},
57+
}
58+
}
59+
60+
func failuref(format string, args ...any) GenerationResult {
61+
return failure(fmt.Errorf(format, args...))
62+
}
63+
64+
func Generate(ctx context.Context, cfg *Config) GenerationResult {
2665
var err error
2766
if !filepath.IsAbs(cfg.OutputDir) {
2867
if cfg.OutputDir, err = filepath.Abs(cfg.OutputDir); err != nil {
29-
return fmt.Errorf("failed to get absolute path for %s: %w", cfg.OutputDir, err)
68+
return failuref("failed to get absolute path for %s: %w", cfg.OutputDir, err)
3069
}
3170
}
3271

3372
if _, err := os.Stat(cfg.OutputDir); err == nil && cfg.Clobber {
3473
log.Printf("Deleting all files in %s", cfg.OutputDir)
3574
if err := os.RemoveAll(cfg.OutputDir); err != nil {
36-
return fmt.Errorf("failed to delete %s: %s", cfg.OutputDir, err)
75+
return failuref("failed to delete %s: %s", cfg.OutputDir, err)
3776
}
3877
} else if err == nil && !cfg.Clobber {
39-
return fmt.Errorf("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir)
78+
return failuref("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir)
4079
}
4180

4281
log.Printf("Generating resources to %s", cfg.OutputDir)
4382
if err := os.MkdirAll(cfg.OutputDir, 0755); err != nil {
44-
return fmt.Errorf("failed to create output directory %s: %s", cfg.OutputDir, err)
83+
return failuref("failed to create output directory %s: %s", cfg.OutputDir, err)
4584
}
4685

4786
// Generate provider installation block
@@ -59,22 +98,21 @@ func Generate(ctx context.Context, cfg *Config) error {
5998
tf, err := setupTerraform(cfg)
6099
// Terraform init to download the provider
61100
if err != nil {
62-
return fmt.Errorf("failed to run terraform init: %w", err)
101+
return failuref("failed to run terraform init: %w", err)
63102
}
64103
cfg.Terraform = tf
65104

105+
var returnResult GenerationResult
66106
if cfg.Cloud != nil {
67107
log.Printf("Generating cloud resources")
68-
stacks, err := generateCloudResources(ctx, cfg)
69-
if err != nil {
70-
return err
71-
}
108+
var stacks []stack
109+
stacks, returnResult = generateCloudResources(ctx, cfg)
72110

73111
for _, stack := range stacks {
74112
stack.name = "stack-" + stack.slug
75-
if err := generateGrafanaResources(ctx, cfg, stack, false); err != nil {
76-
return err
77-
}
113+
stackResult := generateGrafanaResources(ctx, cfg, stack, false)
114+
returnResult.Success = append(returnResult.Success, stackResult.Success...)
115+
returnResult.Errors = append(returnResult.Errors, stackResult.Errors...)
78116
}
79117
}
80118

@@ -89,29 +127,42 @@ func Generate(ctx context.Context, cfg *Config) error {
89127
onCallURL: cfg.Grafana.OnCallURL,
90128
}
91129
log.Printf("Generating Grafana resources")
92-
if err := generateGrafanaResources(ctx, cfg, stack, true); err != nil {
93-
return err
130+
returnResult = generateGrafanaResources(ctx, cfg, stack, true)
131+
}
132+
133+
if !cfg.OutputCredentials && cfg.Format != OutputFormatCrossplane {
134+
if err := postprocessing.RedactCredentials(cfg.OutputDir); err != nil {
135+
return failuref("failed to redact credentials: %w", err)
94136
}
95137
}
96138

97-
if cfg.Format == OutputFormatCrossplane {
98-
return convertToCrossplane(cfg)
139+
if returnResult.Blocks() == 0 {
140+
if err := os.WriteFile(filepath.Join(cfg.OutputDir, "resources.tf"), []byte("# No resources were found\n"), 0600); err != nil {
141+
return failure(err)
142+
}
143+
if err := os.WriteFile(filepath.Join(cfg.OutputDir, "imports.tf"), []byte("# No resources were found\n"), 0600); err != nil {
144+
return failure(err)
145+
}
146+
return returnResult
99147
}
100148

101-
if !cfg.OutputCredentials {
102-
if err := postprocessing.RedactCredentials(cfg.OutputDir); err != nil {
103-
return fmt.Errorf("failed to redact credentials: %w", err)
149+
if cfg.Format == OutputFormatCrossplane {
150+
if err := convertToCrossplane(cfg); err != nil {
151+
return failure(err)
104152
}
153+
return returnResult
105154
}
106155

107156
if cfg.Format == OutputFormatJSON {
108-
return convertToTFJSON(cfg.OutputDir)
157+
if err := convertToTFJSON(cfg.OutputDir); err != nil {
158+
return failure(err)
159+
}
109160
}
110161

111-
return nil
162+
return returnResult
112163
}
113164

114-
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, cfg *Config, provider string) error {
165+
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, cfg *Config, provider string) GenerationResult {
115166
generatedFilename := func(suffix string) string {
116167
if provider == "" {
117168
return filepath.Join(cfg.OutputDir, suffix)
@@ -122,7 +173,7 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
122173

123174
resources, err := filterResources(resources, cfg.IncludeResources)
124175
if err != nil {
125-
return err
176+
return failure(err)
126177
}
127178

128179
// Generate HCL blocks in parallel with a wait group
@@ -207,12 +258,21 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
207258
wg.Wait()
208259
close(results)
209260

261+
returnResult := GenerationResult{}
210262
resultsSlice := []result{}
211263
for r := range results {
212264
if r.err != nil {
213-
return fmt.Errorf("failed to generate %s resources: %w", r.resource.Name, r.err)
265+
returnResult.Errors = append(returnResult.Errors, ResourceError{
266+
Resource: r.resource,
267+
Err: r.err,
268+
})
269+
} else {
270+
resultsSlice = append(resultsSlice, r)
271+
returnResult.Success = append(returnResult.Success, GenerationSuccess{
272+
Resource: r.resource,
273+
Blocks: len(r.blocks),
274+
})
214275
}
215-
resultsSlice = append(resultsSlice, r)
216276
}
217277
sort.Slice(resultsSlice, func(i, j int) bool {
218278
return resultsSlice[i].resource.Name < resultsSlice[j].resource.Name
@@ -225,23 +285,21 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
225285
}
226286

227287
if len(allBlocks) == 0 {
228-
if err := os.WriteFile(generatedFilename("resources.tf"), []byte("# No resources were found\n"), 0600); err != nil {
229-
return err
230-
}
231-
if err := os.WriteFile(generatedFilename("imports.tf"), []byte("# No resources were found\n"), 0600); err != nil {
232-
return err
233-
}
234-
return nil
288+
return returnResult
235289
}
236290

237291
if err := writeBlocks(generatedFilename("imports.tf"), allBlocks...); err != nil {
238-
return err
292+
return failure(err)
239293
}
240294
_, err = cfg.Terraform.Plan(ctx, tfexec.GenerateConfigOut(generatedFilename("resources.tf")))
241295
if err != nil {
242-
return fmt.Errorf("failed to generate resources: %w", err)
296+
return failuref("failed to generate resources: %w", err)
243297
}
244-
return sortResourcesFile(generatedFilename("resources.tf"))
298+
if err := sortResourcesFile(generatedFilename("resources.tf")); err != nil {
299+
return failure(err)
300+
}
301+
302+
return returnResult
245303
}
246304

247305
func filterResources(resources []*common.Resource, includedResources []string) ([]*common.Resource, error) {

0 commit comments

Comments
 (0)