feat: refactor cni telemetry (#3149)

* move telemetry to dedicated package and create package variable

* send telemetry for ipam config, epinfo, and error on cni add completion

* add new telemetry logs for debugging

* replace nwcfg subcontext with containerid for correlation

* add mutex

* address linter issues

* add unit tests

* set operation id before sending any events for sampling

* remove newly added telemetry events

we will split this part of the pr into its own pr
a telemetry event was added back which was previously removed
undo this pr to add those telemetry statements back

* refactor plugin main

remove reflect
remove duplicated telemetry and telemetry buffer
remove unused fields in report manager
force access to telemetry client fields through methods
move telemetry start/connect code closer to start of plugin execution

* add unit tests and simplify

we use SendError where we would have previously called reportPluginError (no log emitted)
we don't set error message in cni report because the error message and event message fields both end up in the Message field in the cni telemetry service

* remove duplicate error telemetry

* fix comment

* remove writing telemetry events to the log file

* address feedback

* remove stutter

* remove unused map (noop)

* address feedback

* fix condition

* align flow with previous telemetry flow

tested and none panic:
telemetry service running, lock acquired
telemetry service not running, lock acquired
telemetry service running, lock not acquired
telemetry service not running, lock not acquired
stateless if telemetry service not running
stateless if telemetry service is running

* address linter

* re-implement add and delete timing metrics

Author: QxBytes

cni/network/common.go

@@ -4,26 +4,14 @@ import (
 	"encoding/json"
 	"io"
 	"os"
-	"reflect"
 
 	"github.com/Azure/azure-container-networking/cni"
-	"github.com/Azure/azure-container-networking/telemetry"
 	"github.com/containernetworking/cni/pkg/skel"
 	cniTypes "github.com/containernetworking/cni/pkg/types"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
-// send error report to hostnetagent if CNI encounters any error.
-func ReportPluginError(reportManager *telemetry.ReportManager, tb *telemetry.TelemetryBuffer, err error) {
-	logger.Error("Report plugin error")
-	reflect.ValueOf(reportManager.Report).Elem().FieldByName("ErrorMessage").SetString(err.Error())
-
-	if err := reportManager.SendReport(tb); err != nil {
-		logger.Error("SendReport failed", zap.Error(err))
-	}
-}
-
 func validateConfig(jsonBytes []byte) error {
 	var conf struct {
 		Name string `json:"name"`