Skip to content

Commit 7732fd5

Browse files
philrhurstPhilip Hurst
philrhurst
and
Philip Hurst
authored
Improve Logging and Error Handling (#115)
* refactor logic to continue if encountering errors with support export. * When there are errors with patronictl and pgbackrest commands, write stderr to the support export log for troubleshooting. * clarifying that info for Kubernetes nodes is gathered at this step, not Postgres nodes. * updated WriteInfo to include data from `err` and `stderr` * `stderr` contains the error message from the subsystem (pgbackrest or patroni) which is critical for the user to have for troubleshooting. Giving both provides consistency with error reporting in other parts of the support export. * remove redundant call to `strings.TrimSpace` --------- Co-authored-by: Philip Hurst <[email protected]>
1 parent b4a2ca9 commit 7732fd5

File tree

1 file changed

+108
-78
lines changed

1 file changed

+108
-78
lines changed

‎internal/cmd/export.go‎

Lines changed: 108 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -387,88 +387,111 @@ Collecting PGO CLI logs...
387387

388388
// PGO CLI version
389389
err = gatherPGOCLIVersion(ctx, clusterName, tw, cmd)
390+
if err != nil {
391+
writeInfo(cmd, fmt.Sprintf("Error gathering PGO CLI Version: %s", err))
392+
}
390393

391-
if err == nil {
392-
err = gatherPostgresClusterNames(clusterName, ctx, cmd, tw, postgresClient)
394+
// Postgres Cluster Names
395+
err = gatherPostgresClusterNames(clusterName, ctx, cmd, tw, postgresClient)
396+
if err != nil {
397+
writeInfo(cmd, fmt.Sprintf("Error gathering Postgres Cluster Names: %s", err))
393398
}
394399

395400
// Current Kubernetes context
396-
if err == nil {
397-
err = gatherKubeContext(ctx, config, clusterName, tw, cmd)
401+
err = gatherKubeContext(ctx, config, clusterName, tw, cmd)
402+
if err != nil {
403+
writeInfo(cmd, fmt.Sprintf("Error gathering current Kubernetes context: %s", err))
398404
}
399405

400-
// Gather cluster wide resources
401-
if err == nil {
402-
err = gatherKubeServerVersion(ctx, discoveryClient, clusterName, tw, cmd)
406+
// Gather Kubernetes Server Version
407+
err = gatherKubeServerVersion(ctx, discoveryClient, clusterName, tw, cmd)
408+
if err != nil {
409+
writeInfo(cmd, fmt.Sprintf("Error gathering Kubernetes server version: %s", err))
403410
}
404411

405-
if err == nil {
406-
err = gatherNodes(ctx, clientset, clusterName, tw, cmd)
412+
// Gather list of Kubernetes nodes
413+
err = gatherNodes(ctx, clientset, clusterName, tw, cmd)
414+
if err != nil {
415+
writeInfo(cmd, fmt.Sprintf("Error gathering list of Kubernetes nodes: %s", err))
407416
}
408417

409-
if err == nil {
410-
err = gatherCurrentNamespace(ctx, clientset, namespace, clusterName, tw, cmd)
418+
// Gather namespace information
419+
err = gatherCurrentNamespace(ctx, clientset, namespace, clusterName, tw, cmd)
420+
if err != nil {
421+
writeInfo(cmd, fmt.Sprintf("Error gathering namespace information: %s", err))
411422
}
412423

413-
// Namespaced resources
414-
if err == nil {
415-
err = gatherClusterSpec(get, clusterName, tw, cmd)
424+
// Gather PostgresCluster manifest
425+
err = gatherClusterSpec(get, clusterName, tw, cmd)
426+
if err != nil {
427+
writeInfo(cmd, fmt.Sprintf("Error gathering PostgresCluster manifest: %s", err))
416428
}
417429

418430
// TODO (jmckulk): pod describe output
419-
if err == nil {
420-
// get Namespaced resources that have cluster label
421-
nsListOpts := metav1.ListOptions{
422-
LabelSelector: "postgres-operator.crunchydata.com/cluster=" + clusterName,
423-
}
424-
err = gatherNamespacedAPIResources(ctx, dynamicClient, namespace,
425-
clusterName, clusterNamespacedResources, nsListOpts, tw, cmd)
431+
// Gather Namespaced API Resources
432+
// get Namespaced resources that have cluster label
433+
nsListOpts := metav1.ListOptions{
434+
LabelSelector: "postgres-operator.crunchydata.com/cluster=" + clusterName,
435+
}
436+
err = gatherNamespacedAPIResources(ctx, dynamicClient, namespace,
437+
clusterName, clusterNamespacedResources, nsListOpts, tw, cmd)
438+
if err != nil {
439+
writeInfo(cmd, fmt.Sprintf("Error gathering Namespaced API Resources: %s", err))
426440
}
427441

428-
if err == nil {
429-
// get other Namespaced resources that do not have the cluster label
430-
// but may otherwise impact the PostgresCluster's operation
431-
otherListOpts := metav1.ListOptions{}
432-
err = gatherNamespacedAPIResources(ctx, dynamicClient, namespace,
433-
clusterName, otherNamespacedResources, otherListOpts, tw, cmd)
442+
// Gather Namespaced API Resources
443+
// get other Namespaced resources that do not have the cluster label
444+
// but may otherwise impact the PostgresCluster's operation
445+
otherListOpts := metav1.ListOptions{}
446+
err = gatherNamespacedAPIResources(ctx, dynamicClient, namespace,
447+
clusterName, otherNamespacedResources, otherListOpts, tw, cmd)
448+
if err != nil {
449+
writeInfo(cmd, fmt.Sprintf("Error gathering Namespaced API Resources: %s", err))
434450
}
435451

436-
if err == nil {
437-
err = gatherEvents(ctx, clientset, namespace, clusterName, tw, cmd)
452+
// Gather Events
453+
err = gatherEvents(ctx, clientset, namespace, clusterName, tw, cmd)
454+
if err != nil {
455+
writeInfo(cmd, fmt.Sprintf("Error gathering Events: %s", err))
438456
}
439457

440458
// Logs
441459
// All Postgres Logs on the Postgres Instances (primary and replicas)
442460
if numLogs > 0 {
443-
if err == nil {
444-
err = gatherPostgresLogsAndConfigs(ctx, clientset, restConfig,
445-
namespace, clusterName, outputDir, outputFile, numLogs, tw, cmd)
461+
err = gatherPostgresLogsAndConfigs(ctx, clientset, restConfig,
462+
namespace, clusterName, outputDir, outputFile, numLogs, tw, cmd)
463+
if err != nil {
464+
writeInfo(cmd, fmt.Sprintf("Error gathering Postgres Logs and Config: %s", err))
446465
}
447466
}
448467

449468
// All pgBackRest Logs on the Postgres Instances
450-
if err == nil {
451-
err = gatherDbBackrestLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
469+
err = gatherDbBackrestLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
470+
if err != nil {
471+
writeInfo(cmd, fmt.Sprintf("Error gathering pgBackRest DB Hosts Logs: %s", err))
452472
}
453473

454474
// All pgBackRest Logs on the Repo Host
455-
if err == nil {
456-
err = gatherRepoHostLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
475+
err = gatherRepoHostLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
476+
if err != nil {
477+
writeInfo(cmd, fmt.Sprintf("Error gathering pgBackRest Repo Host Logs: %s", err))
457478
}
458479

459480
// get PostgresCluster Pod logs
460-
if err == nil {
461-
writeInfo(cmd, "Collecting PostgresCluster pod logs...")
462-
err = gatherPodLogs(ctx, clientset, namespace, fmt.Sprintf("%s=%s", util.LabelCluster, clusterName), clusterName, tw, cmd)
481+
writeInfo(cmd, "Collecting PostgresCluster pod logs...")
482+
err = gatherPodLogs(ctx, clientset, namespace, fmt.Sprintf("%s=%s", util.LabelCluster, clusterName), clusterName, tw, cmd)
483+
if err != nil {
484+
writeInfo(cmd, fmt.Sprintf("Error gathering PostgresCluster pod logs: %s", err))
463485
}
464486

465487
// get monitoring Pod logs
466488
if monitoringNamespace == "" {
467489
monitoringNamespace = namespace
468490
}
469-
if err == nil {
470-
writeInfo(cmd, "Collecting monitoring pod logs...")
471-
err = gatherPodLogs(ctx, clientset, monitoringNamespace, util.LabelMonitoring, "monitoring", tw, cmd)
491+
writeInfo(cmd, "Collecting monitoring pod logs...")
492+
err = gatherPodLogs(ctx, clientset, monitoringNamespace, util.LabelMonitoring, "monitoring", tw, cmd)
493+
if err != nil {
494+
writeInfo(cmd, fmt.Sprintf("Error gathering monitoring pod logs: %s", err))
472495
}
473496

474497
// get operator Pod logs and descriptions
@@ -478,44 +501,55 @@ Collecting PGO CLI logs...
478501
// Operator and Operator upgrade pods should have
479502
// "postgres-operator.crunchydata.com/control-plane" label
480503
// but with different values
481-
if err == nil {
482-
req, _ := labels.NewRequirement(util.LabelOperator,
483-
selection.Exists, []string{},
484-
)
485-
nsListOpts := metav1.ListOptions{
486-
LabelSelector: req.String(),
487-
}
488-
err = gatherNamespacedAPIResources(ctx, dynamicClient,
489-
operatorNamespace, "operator", operatorNamespacedResources,
490-
nsListOpts, tw, cmd)
504+
req, _ := labels.NewRequirement(util.LabelOperator,
505+
selection.Exists, []string{},
506+
)
507+
nsListOpts = metav1.ListOptions{
508+
LabelSelector: req.String(),
509+
}
510+
err = gatherNamespacedAPIResources(ctx, dynamicClient,
511+
operatorNamespace, "operator", operatorNamespacedResources,
512+
nsListOpts, tw, cmd)
513+
if err != nil {
514+
writeInfo(cmd, fmt.Sprintf("Error gathering Operator Namespace API Resources: %s", err))
491515
}
492-
if err == nil {
493-
writeInfo(cmd, "Collecting operator pod logs...")
494-
err = gatherPodLogs(ctx, clientset, operatorNamespace, util.LabelOperator, "operator", tw, cmd)
516+
517+
// Gather Operator Pod Logs
518+
writeInfo(cmd, "Collecting operator pod logs...")
519+
err = gatherPodLogs(ctx, clientset, operatorNamespace, util.LabelOperator, "operator", tw, cmd)
520+
if err != nil {
521+
writeInfo(cmd, fmt.Sprintf("Error gathering Operator Pod logs: %s", err))
495522
}
496523

497-
// Exec resources
498-
if err == nil {
499-
err = gatherPatroniInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
524+
// Exec to get Patroni Information
525+
err = gatherPatroniInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
526+
if err != nil {
527+
writeInfo(cmd, fmt.Sprintf("Error gathering Patroni Info: %s", err))
500528
}
501529

502-
if err == nil {
503-
err = gatherPgBackRestInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
530+
// Exec to get pgBackRest Information
531+
err = gatherPgBackRestInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
532+
if err != nil {
533+
writeInfo(cmd, fmt.Sprintf("Error gathering pgBackRest Info: %s", err))
504534
}
505535

506536
// Exec to get Container processes
507-
if err == nil {
508-
err = gatherProcessInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
537+
err = gatherProcessInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
538+
if err != nil {
539+
writeInfo(cmd, fmt.Sprintf("Error gathering container processes: %s", err))
509540
}
510541

511542
// Exec to get Container system time
512-
if err == nil {
513-
err = gatherSystemTime(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
543+
err = gatherSystemTime(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
544+
if err != nil {
545+
writeInfo(cmd, fmt.Sprintf("Error gathering container system time: %s", err))
514546
}
515547

516-
if err == nil {
517-
writeInfo(cmd, "Collecting list of kubectl plugins...")
518-
err = gatherPluginList(clusterName, tw, cmd)
548+
// Get kubectl plugins
549+
writeInfo(cmd, "Collecting list of kubectl plugins...")
550+
err = gatherPluginList(clusterName, tw, cmd)
551+
if err != nil {
552+
writeInfo(cmd, fmt.Sprintf("Error gathering kubectl plugins: %s", err))
519553
}
520554

521555
// Print cli output
@@ -526,13 +560,8 @@ Collecting PGO CLI logs...
526560
}
527561

528562
// Print final message
529-
if err == nil {
530-
info, err := os.Stat(outputDir + "/" + outputFile)
531-
532-
if err == nil {
533-
fmt.Print(exportSizeReport(float64(info.Size())))
534-
}
535-
}
563+
info, err := os.Stat(outputDir + "/" + outputFile)
564+
fmt.Print(exportSizeReport(float64(info.Size())))
536565

537566
return err
538567
}
@@ -579,6 +608,7 @@ func gatherPGOCLIVersion(_ context.Context,
579608
cmd *cobra.Command,
580609
) error {
581610
writeInfo(cmd, "Collecting PGO CLI version...")
611+
writeInfo(cmd, fmt.Sprintf("PGO CLI version is %s", clientVersion))
582612
path := clusterName + "/pgo-cli-version"
583613
if err := writeTar(tw, []byte(clientVersion), path, cmd); err != nil {
584614
return err
@@ -1493,7 +1523,7 @@ func gatherPatroniInfo(ctx context.Context,
14931523
writeInfo(cmd, err.Error())
14941524
return nil
14951525
}
1496-
return err
1526+
writeInfo(cmd, fmt.Sprintf("Error with patronictl list: %s: %s", err, strings.TrimSpace(stderr)))
14971527
}
14981528

14991529
buf.Write([]byte(stdout))
@@ -1508,7 +1538,7 @@ func gatherPatroniInfo(ctx context.Context,
15081538
writeInfo(cmd, err.Error())
15091539
return nil
15101540
}
1511-
return err
1541+
writeInfo(cmd, fmt.Sprintf("Error with patronictl history: %s: %s", err, strings.TrimSpace(stderr)))
15121542
}
15131543

15141544
buf.Write([]byte(stdout))
@@ -1571,7 +1601,7 @@ func gatherPgBackRestInfo(ctx context.Context,
15711601
writeInfo(cmd, err.Error())
15721602
return nil
15731603
}
1574-
return err
1604+
writeInfo(cmd, fmt.Sprintf("Error with pgbackrest info: %s: %s", err, strings.TrimSpace(stderr)))
15751605
}
15761606

15771607
buf.Write([]byte(stdout))
@@ -1586,7 +1616,7 @@ func gatherPgBackRestInfo(ctx context.Context,
15861616
writeInfo(cmd, err.Error())
15871617
return nil
15881618
}
1589-
return err
1619+
writeInfo(cmd, fmt.Sprintf("Error with pgbackrest check: %s: %s", err, strings.TrimSpace(stderr)))
15901620
}
15911621

15921622
buf.Write([]byte(stdout))

0 commit comments

Comments
 (0)