Skip to content

Supporting JMX annotations #240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 14, 2024
Merged

Supporting JMX annotations #240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b8dfc39
Support JMX annotations.
jefchien Aug 28, 2024
c8e6e2f
Supporting JMX annotations
mitali-salvi Oct 3, 2024
f8bb908
add support for jmx container insights port and updating env vars for…
mitali-salvi Oct 11, 2024
02ffe60
updating env vars
mitali-salvi Oct 11, 2024
bff2e35
addressing comments
mitali-salvi Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions internal/manifests/collector/adapters/config_from.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ type Traces struct {
type MetricsCollected struct {
StatsD *statsD `json:"statsd,omitempty"`
CollectD *collectD `json:"collectd,omitempty"`
JMX *jmx `json:"jmx,omitempty"`
}

type LogMetricsCollected struct {
Expand Down Expand Up @@ -86,9 +87,13 @@ type AppSignals struct {

type emf struct {
}

type jmx struct{}

type kubernetes struct {
EnhancedContainerInsights bool `json:"enhanced_container_insights,omitempty"`
AcceleratedComputeMetrics bool `json:"accelerated_compute_metrics,omitempty"`
JMXContainerInsights bool `json:"jmx_container_insights,omitempty"`
}

type xray struct {
Expand Down
19 changes: 19 additions & 0 deletions internal/manifests/collector/ports.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@ const (
EMFTcp = "emf-tcp"
EMFUdp = "emf-udp"
CWA = "cwa-"
JmxHttp = "jmx-http"
)

var receiverDefaultPortsMap = map[string]int32{
StatsD: 8125,
CollectD: 25826,
XrayTraces: 2000,
JmxHttp: 4314,
OtlpGrpc: 4317,
OtlpHttp: 4318,
EMF: 25888,
Expand Down Expand Up @@ -143,6 +145,9 @@ func getMetricsReceiversServicePorts(logger logr.Logger, config *adapters.CwaCon
if config.Metrics.MetricsCollected.CollectD != nil {
getReceiverServicePort(logger, config.Metrics.MetricsCollected.CollectD.ServiceAddress, CollectD, corev1.ProtocolUDP, servicePortsMap)
}
if config.Metrics.MetricsCollected.JMX != nil {
getReceiverServicePort(logger, "", JmxHttp, corev1.ProtocolTCP, servicePortsMap)
}
}

func getReceiverServicePort(logger logr.Logger, serviceAddress string, receiverName string, protocol corev1.Protocol, servicePortsMap map[int32][]corev1.ServicePort) {
Expand Down Expand Up @@ -195,6 +200,20 @@ func getLogsReceiversServicePorts(logger logr.Logger, config *adapters.CwaConfig
servicePortsMap[receiverDefaultPortsMap[EMF]] = []corev1.ServicePort{tcp, udp}
}
}

//JMX Container Insights
if config.Logs != nil && config.Logs.LogMetricsCollected != nil && config.Logs.LogMetricsCollected.Kubernetes != nil && config.Logs.LogMetricsCollected.Kubernetes.JMXContainerInsights {
if _, ok := servicePortsMap[receiverDefaultPortsMap[JmxHttp]]; ok {
logger.Info("Duplicate port has been configured in Agent Config for port", zap.Int32("port", receiverDefaultPortsMap[JmxHttp]))
} else {
tcp := corev1.ServicePort{
Name: JmxHttp,
Port: receiverDefaultPortsMap[JmxHttp],
Protocol: corev1.ProtocolTCP,
}
servicePortsMap[receiverDefaultPortsMap[JmxHttp]] = []corev1.ServicePort{tcp}
}
}
}

func getTracesReceiversServicePorts(logger logr.Logger, config *adapters.CwaConfig, servicePortsMap map[int32][]corev1.ServicePort) []corev1.ServicePort {
Expand Down
18 changes: 18 additions & 0 deletions internal/manifests/collector/ports_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,24 @@ func TestInvalidConfigGetContainerPorts(t *testing.T) {

}

func TestJMXGetContainerPorts(t *testing.T) {
cfg := getJSONStringFromFile("./test-resources/jmxAgentConfig.json")
containerPorts := getContainerPorts(logger, cfg, []corev1.ServicePort{})
assert.Equal(t, 1, len(containerPorts))
assert.Equal(t, int32(4314), containerPorts[JmxHttp].ContainerPort)
assert.Equal(t, JmxHttp, containerPorts[JmxHttp].Name)
assert.Equal(t, corev1.ProtocolTCP, containerPorts[JmxHttp].Protocol)
}

func TestJMXContainerInsightsGetContainerPorts(t *testing.T) {
cfg := getJSONStringFromFile("./test-resources/jmxContainerInsightsConfig.json")
containerPorts := getContainerPorts(logger, cfg, []corev1.ServicePort{})
assert.Equal(t, 1, len(containerPorts))
assert.Equal(t, int32(4314), containerPorts[JmxHttp].ContainerPort)
assert.Equal(t, JmxHttp, containerPorts[JmxHttp].Name)
assert.Equal(t, corev1.ProtocolTCP, containerPorts[JmxHttp].Protocol)
}

func getJSONStringFromFile(path string) string {
buf, err := os.ReadFile(path)
if err != nil {
Expand Down
8 changes: 8 additions & 0 deletions internal/manifests/collector/test-resources/jmxAgentConfig.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"metrics": {
"metrics_collected": {
"jmx": {
}
}
}
}
10 changes: 10 additions & 0 deletions internal/manifests/collector/test-resources/jmxContainerInsightsConfig.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"logs": {
"metrics_collected": {
"kubernetes": {
"cluster_name": "TestCluster",
"jmx_container_insights": true
}
}
}
}
163 changes: 107 additions & 56 deletions pkg/instrumentation/defaultinstrumentation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ import (
"fmt"
"os"

"k8s.io/apimachinery/pkg/api/resource"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/aws/amazon-cloudwatch-agent-operator/apis/v1alpha1"
"github.com/aws/amazon-cloudwatch-agent-operator/internal/manifests/collector/adapters"
"github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation/jmx"
)

const (
Expand Down Expand Up @@ -50,7 +50,7 @@ func getInstrumentationConfigForResource(langStr string, resourceStr string) cor
return instrumentationConfigForResource
}

func getDefaultInstrumentation(agentConfig *adapters.CwaConfig, isWindowsPod bool) (*v1alpha1.Instrumentation, error) {
func getDefaultInstrumentation(agentConfig *adapters.CwaConfig, additionalEnvs map[Type]map[string]string, isWindowsPod bool) (*v1alpha1.Instrumentation, error) {
javaInstrumentationImage, ok := os.LookupEnv("AUTO_INSTRUMENTATION_JAVA")
if !ok {
return nil, errors.New("unable to determine java instrumentation image")
Expand All @@ -77,9 +77,10 @@ func getDefaultInstrumentation(agentConfig *adapters.CwaConfig, isWindowsPod boo

// set protocol by checking cloudwatch agent config for tls setting
exporterPrefix := http
if agentConfig != nil {
appSignalsConfig := agentConfig.GetApplicationSignalsConfig()
if appSignalsConfig != nil && appSignalsConfig.TLS != nil {
isApplicationSignalsEnabled := agentConfig != nil && agentConfig.GetApplicationSignalsConfig() != nil

if isApplicationSignalsEnabled {
if agentConfig.GetApplicationSignalsConfig().TLS != nil {
exporterPrefix = https
}
}
Expand All @@ -103,77 +104,31 @@ func getDefaultInstrumentation(agentConfig *adapters.CwaConfig, isWindowsPod boo
},
Java: v1alpha1.Java{
Image: javaInstrumentationImage,
Env: []corev1.EnvVar{
{Name: "OTEL_AWS_APP_SIGNALS_ENABLED", Value: "true"}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
},
Env: getJavaEnvs(isApplicationSignalsEnabled, cloudwatchAgentServiceEndpoint, exporterPrefix, additionalEnvs[TypeJava]),
Resources: corev1.ResourceRequirements{
Limits: getInstrumentationConfigForResource(java, limit),
Requests: getInstrumentationConfigForResource(java, request),
},
},
Python: v1alpha1.Python{
Image: pythonInstrumentationImage,
Env: []corev1.EnvVar{
{Name: "OTEL_AWS_APP_SIGNALS_ENABLED", Value: "true"}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_PYTHON_DISTRO", Value: "aws_distro"},
{Name: "OTEL_PYTHON_CONFIGURATOR", Value: "aws_configurator"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
},
Env: getPythonEnvs(isApplicationSignalsEnabled, cloudwatchAgentServiceEndpoint, exporterPrefix, additionalEnvs[TypePython]),
Resources: corev1.ResourceRequirements{
Limits: getInstrumentationConfigForResource(python, limit),
Requests: getInstrumentationConfigForResource(python, request),
},
},
DotNet: v1alpha1.DotNet{
Image: dotNetInstrumentationImage,
Env: []corev1.EnvVar{
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_DOTNET_DISTRO", Value: "aws_distro"},
{Name: "OTEL_DOTNET_CONFIGURATOR", Value: "aws_configurator"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
{Name: "OTEL_DOTNET_AUTO_PLUGINS", Value: "AWS.Distro.OpenTelemetry.AutoInstrumentation.Plugin, AWS.Distro.OpenTelemetry.AutoInstrumentation"},
},
Env: getDotNetEnvs(isApplicationSignalsEnabled, cloudwatchAgentServiceEndpoint, exporterPrefix, additionalEnvs[TypeDotNet]),
Resources: corev1.ResourceRequirements{
Limits: getInstrumentationConfigForResource(dotNet, limit),
Requests: getInstrumentationConfigForResource(dotNet, request),
},
},
NodeJS: v1alpha1.NodeJS{
Image: nodeJSInstrumentationImage,
Env: []corev1.EnvVar{
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
},
Env: getNodeJSEnvs(isApplicationSignalsEnabled, cloudwatchAgentServiceEndpoint, exporterPrefix, additionalEnvs[TypeDotNet]),
Resources: corev1.ResourceRequirements{
Limits: getInstrumentationConfigForResource(nodeJS, limit),
Requests: getInstrumentationConfigForResource(nodeJS, request),
Expand All @@ -182,3 +137,99 @@ func getDefaultInstrumentation(agentConfig *adapters.CwaConfig, isWindowsPod boo
},
}, nil
}

func getJavaEnvs(isAppSignalsEnabled bool, cloudwatchAgentServiceEndpoint, exporterPrefix string, additionalEnvs map[string]string) []corev1.EnvVar {
envs := []corev1.EnvVar{
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
}

if isAppSignalsEnabled {
appSignalsEnvs := []corev1.EnvVar{
{Name: "OTEL_AWS_APP_SIGNALS_ENABLED", Value: "true"}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
}
envs = append(envs, appSignalsEnvs...)
} else {
envs = append(envs, corev1.EnvVar{
Name: "OTEL_TRACES_EXPORTER", Value: "none",
})
}

var jmxEnvs []corev1.EnvVar
if targetSystems, ok := additionalEnvs[jmx.EnvTargetSystem]; ok {
jmxEnvs = []corev1.EnvVar{
{Name: "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT", Value: fmt.Sprintf("%s://%s:4314/v1/metrics", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_JMX_TARGET_SYSTEM", Value: targetSystems},
}
}
if len(jmxEnvs) != 0 {
envs = append(envs, jmxEnvs...)
}
return envs
}

func getPythonEnvs(isAppSignalsEnabled bool, cloudwatchAgentServiceEndpoint, exporterPrefix string, additionalEnvs map[string]string) []corev1.EnvVar {
var envs []corev1.EnvVar
if isAppSignalsEnabled {
envs = []corev1.EnvVar{
{Name: "OTEL_AWS_APP_SIGNALS_ENABLED", Value: "true"}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)}, //TODO: remove in favor of new name once safe
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_PYTHON_DISTRO", Value: "aws_distro"},
{Name: "OTEL_PYTHON_CONFIGURATOR", Value: "aws_configurator"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
}
}
return envs
}

func getDotNetEnvs(isAppSignalsEnabled bool, cloudwatchAgentServiceEndpoint, exporterPrefix string, additionalEnvs map[string]string) []corev1.EnvVar {
var envs []corev1.EnvVar
if isAppSignalsEnabled {
envs = []corev1.EnvVar{
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", http, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_DOTNET_DISTRO", Value: "aws_distro"},
{Name: "OTEL_DOTNET_CONFIGURATOR", Value: "aws_configurator"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
{Name: "OTEL_DOTNET_AUTO_PLUGINS", Value: "AWS.Distro.OpenTelemetry.AutoInstrumentation.Plugin, AWS.Distro.OpenTelemetry.AutoInstrumentation"},
}
}
return envs
}

func getNodeJSEnvs(isAppSignalsEnabled bool, cloudwatchAgentServiceEndpoint, exporterPrefix string, additionalEnvs map[string]string) []corev1.EnvVar {
var envs []corev1.EnvVar
if isAppSignalsEnabled {
envs = []corev1.EnvVar{
{Name: "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", Value: "true"},
{Name: "OTEL_TRACES_SAMPLER_ARG", Value: fmt.Sprintf("endpoint=%s://%s:2000", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_TRACES_SAMPLER", Value: "xray"},
{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "http/protobuf"},
{Name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/traces", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", Value: fmt.Sprintf("%s://%s:4316/v1/metrics", exporterPrefix, cloudwatchAgentServiceEndpoint)},
{Name: "OTEL_METRICS_EXPORTER", Value: "none"},
{Name: "OTEL_LOGS_EXPORTER", Value: "none"},
}
}
return envs
}
Loading
Loading