Skip to content

Commit d5b3cd2

Browse files
ginglis13ginglis13
authored
Merge pull request #671 from ginglis13/metrics-integ
add metrics integration tests
2 parents 5712655 + 9ca9ce7 commit d5b3cd2

File tree

2 files changed

+181
-1
lines changed

2 files changed

+181
-1
lines changed

‎snapshotter/metrics_integ_test.go

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
package main
14+
15+
import (
16+
"bytes"
17+
"context"
18+
"encoding/json"
19+
"fmt"
20+
"io"
21+
"net/http"
22+
"strconv"
23+
"testing"
24+
25+
"github.com/containerd/containerd"
26+
"github.com/containerd/containerd/namespaces"
27+
"github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client"
28+
"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
29+
"github.com/firecracker-microvm/firecracker-containerd/proto"
30+
"github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source"
31+
"github.com/stretchr/testify/require"
32+
"golang.org/x/sync/errgroup"
33+
)
34+
35+
const (
36+
serviceDiscoveryEndpoint = "http://localhost:8080"
37+
38+
stargzFsOperation = "stargz_fs_operation_duration_milliseconds_bucket"
39+
stargzFsOperationHelp = "# HELP stargz_fs_operation_duration_milliseconds Latency"
40+
stargzFsOperationType = "# TYPE stargz_fs_operation_duration_milliseconds histogram"
41+
)
42+
43+
func TestSnapshotterMetrics_Isolated(t *testing.T) {
44+
integtest.Prepare(t)
45+
46+
vmID := 0
47+
48+
ctx := namespaces.WithNamespace(context.Background(), strconv.Itoa(vmID))
49+
50+
fcClient, err := integtest.NewFCControlClient(integtest.ContainerdSockPath)
51+
defer fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: strconv.Itoa(vmID)})
52+
require.NoError(t, err, "Failed to create fccontrol client")
53+
54+
require.NoError(t, pullImageWithRemoteSnapshotterInVM(ctx, strconv.Itoa(vmID), fcClient))
55+
verifyMetricsResponse(t, 1)
56+
}
57+
58+
func TestSnapshotterMetricsMultipleVMs_Isolated(t *testing.T) {
59+
integtest.Prepare(t)
60+
61+
numberOfVms := integtest.NumberOfVms
62+
fcClient, err := integtest.NewFCControlClient(integtest.ContainerdSockPath)
63+
require.NoError(t, err, "Failed to create fccontrol client")
64+
65+
group, ctx := errgroup.WithContext(context.Background())
66+
67+
for vmID := 0; vmID < numberOfVms; vmID++ {
68+
id := vmID
69+
ctxNamespace := namespaces.WithNamespace(ctx, strconv.Itoa(id))
70+
defer fcClient.StopVM(ctxNamespace, &proto.StopVMRequest{VMID: strconv.Itoa(id)})
71+
72+
group.Go(
73+
func() error {
74+
return pullImageWithRemoteSnapshotterInVM(ctxNamespace, strconv.Itoa(id), fcClient)
75+
},
76+
)
77+
78+
}
79+
80+
err = group.Wait()
81+
require.NoError(t, err)
82+
verifyMetricsResponse(t, numberOfVms)
83+
}
84+
85+
// TODO (ginglis13): ensure service discovery response changes as VMs are started and stopped.
86+
// pending https://github.com/firecracker-microvm/firecracker-containerd/issues/651
87+
// func TestSnapshotterMetricsMultipleVMsStopStart_Isolated(t *testing.T) {}
88+
89+
func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClient *client.Client) error {
90+
client, err := containerd.New(integtest.ContainerdSockPath, containerd.WithDefaultRuntime(integtest.FirecrackerRuntime))
91+
if err != nil {
92+
return fmt.Errorf("Unable to create client to containerd service at %s, is containerd running?: %v", integtest.ContainerdSockPath, err)
93+
}
94+
if _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
95+
VMID: vmID,
96+
RootDrive: &proto.FirecrackerRootDrive{
97+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img",
98+
},
99+
NetworkInterfaces: []*proto.FirecrackerNetworkInterface{
100+
{
101+
AllowMMDS: true,
102+
CNIConfig: &proto.CNIConfiguration{
103+
NetworkName: "fcnet",
104+
InterfaceName: "veth0",
105+
},
106+
},
107+
},
108+
MachineCfg: &proto.FirecrackerMachineConfiguration{
109+
VcpuCount: 1,
110+
MemSizeMib: 512,
111+
},
112+
ContainerCount: 1,
113+
}); err != nil {
114+
return fmt.Errorf("Failed to create microVM[%s]: %v", vmID, err)
115+
}
116+
117+
if _, err = fcClient.SetVMMetadata(ctx, &proto.SetVMMetadataRequest{
118+
VMID: vmID,
119+
Metadata: fmt.Sprintf(dockerMetadataTemplate, "ghcr.io", noAuth, noAuth),
120+
}); err != nil {
121+
return fmt.Errorf("Failed to set metadata on microVM[%s]: %v", vmID, err)
122+
}
123+
124+
image, err := client.Pull(ctx, al2stargz,
125+
containerd.WithPullUnpack,
126+
containerd.WithPullSnapshotter(snapshotterName),
127+
containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*1024*1024)),
128+
)
129+
if err != nil {
130+
return fmt.Errorf("Failed to pull image on microVM[%s]: %v", vmID, err)
131+
}
132+
if err = client.ImageService().Delete(ctx, image.Name()); err != nil {
133+
return fmt.Errorf("Failed to delete image on microVM[%s]: %v", vmID, err)
134+
}
135+
136+
return nil
137+
}
138+
139+
type metricsTarget struct {
140+
Targets []string `json:"targets"`
141+
Labels map[string]string `json:"labels"`
142+
}
143+
144+
func verifyMetricsResponse(t *testing.T, numberOfVms int) {
145+
sdResponse, err := http.Get(serviceDiscoveryEndpoint)
146+
require.NoError(t, err, "Get service discovery failed")
147+
defer sdResponse.Body.Close()
148+
149+
sdBytes, err := io.ReadAll(sdResponse.Body)
150+
require.NoError(t, err, "Failed to read service discovery response body")
151+
152+
var metricsTargets []metricsTarget
153+
err = json.Unmarshal(sdBytes, &metricsTargets)
154+
require.NoError(t, err, "Failed to unmarshall service discovery response")
155+
156+
require.Len(t, metricsTargets, numberOfVms)
157+
158+
// Test pulling individual metrics proxies
159+
// uniqueTargets acts as a set to ensure each metrics target is a unique host:port
160+
uniqueTargets := make(map[string]struct{})
161+
for _, mt := range metricsTargets {
162+
uniqueTargets[mt.Targets[0]] = struct{}{}
163+
metricsResponse, err := http.Get("http://" + mt.Targets[0] + "/metrics")
164+
require.NoError(t, err, "Failed to get metrics proxy")
165+
require.Equal(t, metricsResponse.StatusCode, http.StatusOK)
166+
defer metricsResponse.Body.Close()
167+
168+
mBytes, err := io.ReadAll(metricsResponse.Body)
169+
require.NoError(t, err, "Failed to read metrics response body")
170+
require.NotEmpty(t, mBytes, "Empty metrics response body")
171+
require.True(t, bytes.Contains(mBytes, []byte(stargzFsOperation)), "metrics response missing fs operations bucket")
172+
require.True(t, bytes.Contains(mBytes, []byte(stargzFsOperationHelp)), "metrics response missing fs operations HELP")
173+
require.True(t, bytes.Contains(mBytes, []byte(stargzFsOperationType)), "metrics response missing fs operations TYPE")
174+
}
175+
require.Len(t, uniqueTargets, numberOfVms)
176+
}

‎tools/docker/entrypoint.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,11 @@ cat > /etc/demux-snapshotter/config.toml <<EOF
4545
[snapshotter.proxy.address.resolver]
4646
type = "http"
4747
address = "http://127.0.0.1:10001"
48-
48+
[snapshotter.metrics]
49+
enable = true
50+
port_range = "9000-9999"
51+
host = "0.0.0.0"
52+
service_discovery_port = 8080
4953
[debug]
5054
logLevel = "debug"
5155
EOF

0 commit comments

Comments
 (0)