Add agent self-observability

.github/workflows/plugin-tests.yaml

@@ -104,6 +104,7 @@ jobs:
           - segmentio-kafka
           - go-elasticsearchv8
           - goframe
+          - so11y
     steps:
       - uses: actions/checkout@v2
         with:

agent/core/compile.go

@@ -39,6 +39,7 @@ import (
 	_ "unsafe"
 
 	//go:nolint
+	_ "github.com/apache/skywalking-go/agent/core/metrics"
 	_ "github.com/apache/skywalking-go/agent/core/operator"
 	_ "github.com/apache/skywalking-go/agent/core/tracing"
 	_ "github.com/apache/skywalking-go/agent/reporter"

agent/core/metrics/compile.go

@@ -0,0 +1,27 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	//go:nolint
+	_ "fmt"
+	_ "sync"
+
+	//go:nolint
+	_ "github.com/apache/skywalking-go/agent/core/operator"
+)

go.work

@@ -67,6 +67,7 @@ use (
 	./test/plugins/scenarios/logging-activation
 	./test/plugins/scenarios/metric-activation
 	./test/plugins/scenarios/goframe
+	./test/plugins/scenarios/so11y
 
 	./tools/go-agent
 

plugins/core/metrics/bridge.go

@@ -18,11 +18,11 @@
 package metrics
 
 func newMeterOpts() *Opts {
-	return &Opts{labels: make(map[string]string)}
+	return &Opts{Labels: make(map[string]string)}
 }
 
 func (b *Opts) GetLabels() map[string]string {
-	return b.labels
+	return b.Labels
 }
 
 type counterImpl struct {

plugins/core/metrics/metrics.go

@@ -22,13 +22,13 @@ import "github.com/apache/skywalking-go/plugins/core/operator"
 type Opt func(opts *Opts)
 
 type Opts struct {
-	labels map[string]string
+	Labels map[string]string
 }
 
 // WithLabel adds a label to the metrics.
 func WithLabel(key, value string) Opt {
 	return func(meter *Opts) {
-		meter.labels[key] = value
+		meter.Labels[key] = value
 	}
 }
 

plugins/core/operator/common.go

@@ -30,6 +30,7 @@ type Operator interface {
 	Entity() interface{}      // Get the entity of the service
 	Metrics() interface{}     // to MetricsOperator
 	LogReporter() interface{} // to LogReporter
+	So11y() interface{}       // to So11yOperator
 }
 
 type Entity interface {

plugins/core/operator/so11y.go

@@ -0,0 +1,48 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package operator
+
+type So11yOperator interface {
+	CollectErrorOfPlugin(pluginName string)
+	GenNanoTime() int64
+	CollectDurationOfInterceptor(costTime int64)
+}
+
+func ErrorOfPlugin(pluginName string) {
+	op := GetOperator()
+	if op == nil {
+		return
+	}
+	op.(So11yOperator).CollectErrorOfPlugin(pluginName)
+}
+
+func NanoTime() int64 {
+	op := GetOperator()
+	if op == nil {
+		return 0
+	}
+	return op.(So11yOperator).GenNanoTime()
+}
+
+func DurationOfInterceptor(costTime int64) {
+	op := GetOperator()
+	if op == nil {
+		return
+	}
+	op.(So11yOperator).CollectDurationOfInterceptor(costTime)
+}

plugins/core/so11y.go

@@ -0,0 +1,165 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package core
+
+import (
+	"sync"
+	"time"
+
+	"github.com/apache/skywalking-go/plugins/core/metrics"
+)
+
+var (
+	instance *So11y
+	once     sync.Once
+)
+
+type So11y struct {
+	propagatedContextCounter       metrics.Counter
+	propagatedIgnoreContextCounter metrics.Counter
+
+	samplerContextCounter       metrics.Counter
+	samplerIgnoreContextCounter metrics.Counter
+
+	finishContextCounter       metrics.Counter
+	finishIgnoreContextCounter metrics.Counter
+
+	leakedContextCounter       metrics.Counter
+	leakedIgnoreContextCounter metrics.Counter
+
+	errorCounterMap     sync.Map
+	interceptorTimeCost metrics.Histogram
+}
+
+func GetSo11y() *So11y {
+	once.Do(func() {
+		instance = &So11y{}
+	})
+	return instance
+}
+
+func (s *So11y) MeasureTracingContextCreation(t *Tracer, isForceSample, isIgnored bool) {
+	if isForceSample {
+		if isIgnored {
+			if s.propagatedIgnoreContextCounter == nil {
+				s.propagatedIgnoreContextCounter = t.NewCounter("sw_go_created_ignored_context_counter",
+					&metrics.Opts{
+						Labels: map[string]string{"created_by": "propagated"},
+					}).(metrics.Counter)
+			}
+			s.propagatedIgnoreContextCounter.Inc(1)
+		} else {
+			if s.propagatedContextCounter == nil {
+				s.propagatedContextCounter = t.NewCounter("sw_go_created_tracing_context_counter",
+					&metrics.Opts{
+						Labels: map[string]string{"created_by": "propagated"},
+					}).(metrics.Counter)
+			}
+			s.propagatedContextCounter.Inc(1)
+		}
+	} else {
+		if isIgnored {
+			if s.samplerIgnoreContextCounter == nil {
+				s.samplerIgnoreContextCounter = t.NewCounter("sw_go_created_ignored_context_counter",
+					&metrics.Opts{
+						Labels: map[string]string{"created_by": "sampler"},
+					}).(metrics.Counter)
+			}
+			s.samplerIgnoreContextCounter.Inc(1)
+		} else {
+			if s.samplerContextCounter == nil {
+				s.samplerContextCounter = t.NewCounter("sw_go_created_tracing_context_counter", &metrics.Opts{
+					Labels: map[string]string{"created_by": "sampler"},
+				}).(metrics.Counter)
+			}
+			s.samplerContextCounter.Inc(1)
+		}
+	}
+}
+
+func (s *So11y) MeasureTracingContextCompletion(t *Tracer, isIgnored bool) {
+	if isIgnored {
+		if s.finishIgnoreContextCounter == nil {
+			s.finishIgnoreContextCounter = t.NewCounter(
+				"sw_go_finished_ignored_context_counter", nil).(metrics.Counter)
+		}
+		s.finishIgnoreContextCounter.Inc(1)
+	} else {
+		if s.finishContextCounter == nil {
+			s.finishContextCounter = t.NewCounter(
+				"sw_go_finished_tracing_context_counter", nil).(metrics.Counter)
+		}
+		s.finishContextCounter.Inc(1)
+	}
+}
+
+func (s *So11y) MeasureLeakedTracingContext(t *Tracer, isIgnored bool) {
+	if isIgnored {
+		if s.leakedIgnoreContextCounter == nil {
+			s.leakedIgnoreContextCounter = t.NewCounter("sw_go_possible_leaked_context_counter",
+				&metrics.Opts{
+					Labels: map[string]string{"source": "ignore"},
+				}).(metrics.Counter)
+		}
+		s.leakedIgnoreContextCounter.Inc(1)
+	} else {
+		if s.leakedContextCounter == nil {
+			s.leakedContextCounter = t.NewCounter("sw_go_possible_leaked_context_counter",
+				&metrics.Opts{
+					Labels: map[string]string{"created_by": "tracing"},
+				}).(metrics.Counter)
+		}
+		s.leakedContextCounter.Inc(1)
+	}
+}
+
+func (t *Tracer) So11y() interface{} {
+	return t
+}
+
+func (t *Tracer) CollectErrorOfPlugin(pluginName string) {
+	if counter, ok := GetSo11y().errorCounterMap.Load(pluginName); ok {
+		if c, ok := counter.(metrics.Counter); ok {
+			c.Inc(1)
+		}
+	} else {
+		counter, _ := GetSo11y().errorCounterMap.LoadOrStore(pluginName, t.NewCounter(
+			"sw_go_interceptor_error_counter",
+			&metrics.Opts{
+				Labels: map[string]string{"plugin_name": pluginName},
+			}).(metrics.Counter))
+		if c, ok := counter.(metrics.Counter); ok {
+			c.Inc(1)
+		}
+	}
+}
+
+func (t *Tracer) GenNanoTime() int64 {
+	return time.Now().UnixNano()
+}
+
+func (t *Tracer) CollectDurationOfInterceptor(costTime int64) {
+	if GetSo11y().interceptorTimeCost == nil {
+		GetSo11y().interceptorTimeCost = t.NewHistogram("sw_go_tracing_context_performance", 0,
+			[]float64{
+				1000, 10000, 50000, 100000, 300000, 500000,
+				1000000, 5000000, 10000000, 20000000, 50000000, 100000000,
+			}, nil).(metrics.Histogram)
+	}
+	GetSo11y().interceptorTimeCost.Observe(float64(costTime))
+}

plugins/core/span_default.go

@@ -158,6 +158,7 @@ func (ds *DefaultSpan) ErrorOccured() {
 
 func (ds *DefaultSpan) End(changeParent bool) {
 	ds.EndTime = time.Now()
+	GetSo11y().MeasureTracingContextCompletion(ds.tracer, false)
 	if changeParent {
 		if ctx := getTracingContext(); ctx != nil {
 			ctx.SaveActiveSpan(ds.Parent)

plugins/core/span_noop.go

@@ -25,6 +25,7 @@ const noopContextValue = "N/A"
 
 type NoopSpan struct {
 	stackCount int
+	tracer     *Tracer
 }
 
 func newSnapshotNoopSpan() *NoopSpan {
@@ -34,9 +35,10 @@ func newSnapshotNoopSpan() *NoopSpan {
 	}
 }
 
-func newNoopSpan() *NoopSpan {
+func newNoopSpan(tracer *Tracer) *NoopSpan {
 	return &NoopSpan{
 		stackCount: 1,
+		tracer:     tracer,
 	}
 }
 
@@ -99,6 +101,7 @@ func (n *NoopSpan) enterNoSpan() {
 func (n *NoopSpan) End() {
 	n.stackCount--
 	if n.stackCount == 0 {
+		GetSo11y().MeasureTracingContextCompletion(n.tracer, true)
 		if ctx := getTracingContext(); ctx != nil {
 			ctx.SaveActiveSpan(nil)
 		}

plugins/core/tracing.go

@@ -233,10 +233,12 @@ func (s *ContextSnapshot) IsValid() bool {
 
 func (t *Tracer) createNoop(operationName string) (*TracingContext, TracingSpan, bool) {
 	if !t.InitSuccess() || t.Reporter.ConnectionStatus() == reporter.ConnectionStatusDisconnect {
-		return nil, newNoopSpan(), true
+		GetSo11y().MeasureTracingContextCreation(t, false, true)
+		return nil, newNoopSpan(t), true
 	}
 	if tracerIgnore(operationName, t.ignoreSuffix, t.traceIgnorePath) {
-		return nil, newNoopSpan(), true
+		GetSo11y().MeasureTracingContextCreation(t, false, true)
+		return nil, newNoopSpan(t), true
 	}
 	ctx := getTracingContext()
 	if ctx != nil {
@@ -265,11 +267,12 @@ func (t *Tracer) createSpan0(ctx *TracingContext, parent TracingSpan, pluginOpts
 	isForceSample := len(ds.Refs) > 0
 	// Try to sample when it is not force sample
 	if parentSpan == nil && !isForceSample {
-		// Force sample
-		sampled := t.Sampler.IsSampled(ds.OperationName)
-		if !sampled {
+		isSampled := t.Sampler.IsSampled(ds.OperationName)
+		if !isSampled {
+			GetSo11y().MeasureTracingContextCreation(t, false, true)
+			GetSo11y().MeasureLeakedTracingContext(t, true)
 			// Filter by sample just return noop span
-			return newNoopSpan(), true, nil
+			return newNoopSpan(t), true, nil
 		}
 	}
 	// process the opts from agent core for prepare building segment span
@@ -284,6 +287,7 @@ func (t *Tracer) createSpan0(ctx *TracingContext, parent TracingSpan, pluginOpts
 	for _, opt := range pluginOpts {
 		opt.(tracing.SpanOption).Apply(s)
 	}
+	GetSo11y().MeasureTracingContextCreation(t, isForceSample, false)
 	return s, false, nil
 }