Weave report analyzers

The IPAM pool analyzer checks that utilization of the pod IP subnet is
less than 85%. For example, if using 10.32.0.0/12, this analyzer will
warn if 3,482 IPs are currently allocated to pods.

The pending allocation analyzer checks that the IPAM status in the
report has no items for the PendingAllocates field. This indicates the
IPAM service is not ready according to the code in the weave status
template
https://github.com/weaveworks/weave/blob/e3712152d2a0fe3bc998964c948e45bdf8ff6144/prog/weaver/http.go#L186.

The weave connections analyzer checks that all connections to remote
peers are in the established state. The state will be "pending" if UDP
is blocked between nodes and will be "failed" if the weave pod on the
remote node is in a crash loop. To force a pending state for testing,
run the commands `iptables -A INPUT -p udp --dport 6784 -j REJECT` and
`iptables -A INPUT -p udp --dport 6783 -j REJECT` on a peer.

The weave connections analyzer also checks that all connections are
using the fastdp protocol. A commopn issue seen in the field on
CentOS/RHEL 7 is that some sides of a connection are using fastdp and
other sides have fallen back to sleeve. Set the WEAVE_NO_FASTDP env var
on the weave daemonset to "true" to test this analyzer.

Author: Andrew Reed

examples/support-bundle/weave-analyzer.yaml

@@ -0,0 +1,24 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: SupportBundle
+metadata:
+  name: collector-sample
+spec:
+  collectors:
+    - exec:
+        collectorName: weave-report
+        command:
+          - /home/weave/weave
+        args:
+          - --local
+          - report
+        containerName: weave
+        exclude: ""
+        name: kots/kurl/weave
+        namespace: kube-system
+        selector:
+          - name=weave-net
+        timeout: 10s
+
+  analyzers:
+    - weaveReport:
+        reportFileGlob: kots/kurl/weave/kube-system/*/weave-report-stdout.txt

pkg/analyze/analyzer.go

@@ -339,5 +339,20 @@ func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileCont
 		return []*AnalyzeResult{result}, nil
 	}
 
+	if analyzer.WeaveReport != nil {
+		isExcluded, err := isExcluded(analyzer.WeaveReport.Exclude)
+		if err != nil {
+			return nil, err
+		}
+		if isExcluded {
+			return nil, nil
+		}
+		results, err := analyzeWeaveReport(analyzer.WeaveReport, findFiles)
+		if err != nil {
+			return nil, err
+		}
+		return results, nil
+	}
+
 	return nil, errors.New("invalid analyzer")
 }

pkg/analyze/weave.go

@@ -0,0 +1,173 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+)
+
+// relevant fields from https://github.com/weaveworks/weave/blob/e3712152d2a0fe3bc998964c948e45bdf8ff6144/prog/weaver/http.go#L295
+type WeaveReport struct {
+	Router WeaveRouter
+	IPAM   WeaveIPAM
+}
+
+type WeaveRouter struct {
+	NickName    string // this is the hostname
+	Connections []WeaveConnection
+}
+
+type WeaveIPAM struct {
+	RangeNumIPs      int
+	ActiveIPs        int
+	PendingAllocates []string
+}
+
+type WeaveConnection struct {
+	State string
+	Info  string
+	Attrs WeaveAttributes
+}
+
+type WeaveAttributes struct {
+	Encrypted bool   `json:"encrypted"`
+	MTU       int    `json:"mtu"`
+	Name      string `json:"name"`
+}
+
+func analyzeWeaveReport(analyzer *troubleshootv1beta2.WeaveReportAnalyze, findFiles func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
+	files, err := findFiles(analyzer.ReportFileGlob)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find weave report files in %q", analyzer.ReportFileGlob)
+	}
+
+	if len(files) == 0 {
+		return nil, nil
+	}
+
+	reports := map[string]WeaveReport{}
+
+	for name, file := range files {
+		report := WeaveReport{}
+
+		if err := json.Unmarshal(file, &report); err != nil {
+			return nil, errors.Wrapf(err, "failed to unmarshal weave report json from %s", name)
+		}
+
+		reports[report.Router.NickName] = report
+	}
+
+	results := []*AnalyzeResult{}
+
+	if result := analyzeWeaveIPAMPools(reports); result != nil {
+		results = append(results, result)
+	}
+
+	if result := analyzeWeavePendingAllocation(reports); result != nil {
+		results = append(results, result)
+	}
+
+	if result := analyzeWeaveConnections(reports); result != nil {
+		results = append(results, result)
+	}
+
+	if len(results) == 0 {
+		results = append(results, &AnalyzeResult{
+			Title:   "Weave Report",
+			IsPass:  true,
+			Message: "No issues detected in weave report",
+		})
+	}
+
+	return results, nil
+}
+
+func analyzeWeaveIPAMPools(reports map[string]WeaveReport) *AnalyzeResult {
+	for _, report := range reports {
+		if result := analyzeWeaveIPAMPool(&report); result != nil {
+			return result
+		}
+	}
+
+	return nil
+}
+
+func analyzeWeaveIPAMPool(report *WeaveReport) *AnalyzeResult {
+	if report.IPAM.RangeNumIPs == 0 {
+		return nil
+	}
+
+	ipsUsed := float64(report.IPAM.ActiveIPs) / float64(report.IPAM.RangeNumIPs)
+	if ipsUsed < 0.85 {
+		return nil
+	}
+
+	return &AnalyzeResult{
+		Title:   "Available Pod IPs",
+		IsWarn:  true,
+		Message: fmt.Sprintf("%d of %d total available IPs have been assigned", report.IPAM.ActiveIPs, report.IPAM.RangeNumIPs),
+	}
+}
+
+func analyzeWeavePendingAllocation(reports map[string]WeaveReport) *AnalyzeResult {
+	for _, report := range reports {
+		if len(report.IPAM.PendingAllocates) > 0 {
+			return &AnalyzeResult{
+				Title:   "Pending IP Allocation",
+				IsWarn:  true,
+				Message: "Waiting for IPs to become available",
+			}
+		}
+	}
+
+	return nil
+}
+
+// Get the peer hostname for logging purposes from a string like: "encrypted   fastdp 1a:5b:a9:53:2b:11(areed-aka-kkz0)"
+var weaveConnectionInfoPeerRegex = regexp.MustCompile(`\(([^)]+)\)$`)
+
+func parseWeaveConnectionInfoHostname(info string) string {
+	matches := weaveConnectionInfoPeerRegex.FindStringSubmatch(info)
+	if len(matches) == 2 {
+		return matches[1]
+	}
+	return ""
+}
+
+func analyzeWeaveConnections(reports map[string]WeaveReport) *AnalyzeResult {
+	for host, report := range reports {
+		for _, connection := range report.Router.Connections {
+			// older versions of weave show connection to self as failed
+			if strings.HasPrefix(connection.Info, "cannot connect to ourself") {
+				continue
+			}
+
+			peer := parseWeaveConnectionInfoHostname(connection.Info)
+			if peer == "" {
+				peer = "peer"
+			}
+
+			if connection.State != "established" {
+				return &AnalyzeResult{
+					Title:   "Weave Inter-Node Connections",
+					IsWarn:  true,
+					Message: fmt.Sprintf("Connection from %s to %s is %s", host, peer, connection.State),
+				}
+			}
+
+			if connection.Attrs.Name != "" && connection.Attrs.Name != "fastdp" {
+				return &AnalyzeResult{
+					Title:   "Weave Inter-Node Connections",
+					IsWarn:  true,
+					Message: fmt.Sprintf("Connection from %s to %s protocol is %q, not fastdp", host, peer, connection.Attrs.Name),
+				}
+			}
+		}
+	}
+
+	return nil
+}