From cdb75d2406a5d9d0a8348fec58c88e2adc041d10 Mon Sep 17 00:00:00 2001 From: Kern Walster Date: Mon, 18 Jul 2022 22:38:17 +0000 Subject: [PATCH] Replace address resolve with snapshot labels Previously, the demux-snapshotter had to make an http request to the http-address-resolver to find the vsock address and metrics configuration for a remote snapshotter. With this change, the http-address-resolver is removed and this information is sent to the demux-snapshotter via snapshot labels directly from the client. The labels are sent to the demux-snapshotter in the first pepare call at which point the demux-snapshotter will establish connection with the remote snapshotter. The connection is then cached via the namespace key as it was in the previous implementation for subsequent calls. Since each microVM houses its own remote snapshotter in a clean slate on boot, any snapshot API call before a prepare will be operating on an empty snapshotter and therefore will be a noop/not found. Therefore, if we don't have a connection for an API, we can treat it as the noop/not found case. Signed-off-by: Kern Walster --- examples/cmd/remote-snapshotter/go.sum | 17 ++ .../remote-snapshotter/remote_snapshotter.go | 11 +- proto/firecracker.pb.go | 142 ++++++++------- proto/firecracker.proto | 1 + runtime/service.go | 1 + snapshotter/Makefile | 9 +- snapshotter/app/service.go | 65 ++----- snapshotter/demux/cache/cache.go | 21 ++- snapshotter/demux/cache/cache_test.go | 27 +-- snapshotter/demux/cache/evict_test.go | 12 +- snapshotter/demux/opt.go | 49 ++++++ .../demux/proxy/address/http_resolver.go | 74 -------- .../demux/proxy/address/http_resolver_test.go | 164 ------------------ snapshotter/demux/proxy/address/resolver.go | 41 ----- snapshotter/demux/proxy/snapshotter.go | 8 + snapshotter/demux/snapshotter.go | 97 ++++++++++- snapshotter/demux/snapshotter_test.go | 56 +----- snapshotter/internal/http_address_resolver.go | 150 ---------------- snapshotter/metrics_integ_test.go | 14 +- snapshotter/service_integ_test.go | 8 +- snapshotter/volume_integ_test.go | 31 ++-- tools/docker/entrypoint.sh | 5 - volume/.gitignore | 2 + volume/guest_image.go | 2 +- volume/image.go | 12 +- 25 files changed, 358 insertions(+), 661 deletions(-) create mode 100644 snapshotter/demux/opt.go delete mode 100644 snapshotter/demux/proxy/address/http_resolver.go delete mode 100644 snapshotter/demux/proxy/address/http_resolver_test.go delete mode 100644 snapshotter/demux/proxy/address/resolver.go delete mode 100644 snapshotter/internal/http_address_resolver.go create mode 100644 volume/.gitignore diff --git a/examples/cmd/remote-snapshotter/go.sum b/examples/cmd/remote-snapshotter/go.sum index b11b7edf0..65e3f0802 100644 --- a/examples/cmd/remote-snapshotter/go.sum +++ b/examples/cmd/remote-snapshotter/go.sum @@ -93,6 +93,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -185,6 +186,7 @@ github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -231,6 +233,7 @@ github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= @@ -352,6 +355,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/firecracker-microvm/firecracker-go-sdk v0.22.1-0.20220427214706-47505a9cf951 h1:j1zRfar/9U22TeptQIwXB07hyuctgb6++HIxP0nXBhI= github.com/firecracker-microvm/firecracker-go-sdk v0.22.1-0.20220427214706-47505a9cf951/go.mod h1:60W3x6ftClUbRKpqXl7XvrhM/Uv3tochNRq+RlZsd1M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -382,6 +386,7 @@ github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= @@ -562,6 +567,7 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -570,6 +576,7 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -633,10 +640,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -661,9 +670,12 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mdlayher/socket v0.2.0 h1:EY4YQd6hTAg2tcXF84p5DTHazShE50u5HeBzBaNgjkA= github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/mdlayher/vsock v1.1.1 h1:8lFuiXQnmICBrCIIA9PMgVSke6Fg6V4+r0v7r55k88I= github.com/mdlayher/vsock v1.1.1/go.mod h1:Y43jzcy7KM3QB+/FK15pfqGxDMCMzUXWegEfIbSM18U= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.25 h1:dFwPR6SfLtrSwgDcIq2bcU/gVutB4sNApq2HBdqcakg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -848,7 +860,9 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -990,6 +1004,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1014,6 +1029,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1480,6 +1496,7 @@ gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/examples/cmd/remote-snapshotter/remote_snapshotter.go b/examples/cmd/remote-snapshotter/remote_snapshotter.go index 0984f1834..123a4e156 100644 --- a/examples/cmd/remote-snapshotter/remote_snapshotter.go +++ b/examples/cmd/remote-snapshotter/remote_snapshotter.go @@ -26,6 +26,7 @@ import ( fcclient "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" ) const ( @@ -92,7 +93,7 @@ func main() { defer fcClient.Close() fmt.Println("Creating VM") - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, NetworkInterfaces: []*proto.FirecrackerNetworkInterface{{ AllowMMDS: true, @@ -121,10 +122,14 @@ func main() { fmt.Println("Pulling the image") image, err := client.Pull(ctx, imageRef, - containerd.WithPullSnapshotter(snapshotter), + containerd.WithPullSnapshotter(snapshotter, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), containerd.WithPullUnpack, // stargz labels to tell the snapshotter to lazily load the image - containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(imageRef, 10*1024*1024))) + containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(imageRef, 10*1024*1024)), + ) if err != nil { fmt.Println(err) return diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index e1ee2970f..6bbe15ad2 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -207,6 +207,7 @@ type CreateVMResponse struct { LogFifoPath string `protobuf:"bytes,3,opt,name=LogFifoPath,json=logFifoPath,proto3" json:"LogFifoPath,omitempty"` MetricsFifoPath string `protobuf:"bytes,4,opt,name=MetricsFifoPath,json=metricsFifoPath,proto3" json:"MetricsFifoPath,omitempty"` CgroupPath string `protobuf:"bytes,5,opt,name=CgroupPath,json=cgroupPath,proto3" json:"CgroupPath,omitempty"` + VSockPath string `protobuf:"bytes,6,opt,name=VSockPath,json=vSockPath,proto3" json:"VSockPath,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -271,6 +272,13 @@ func (m *CreateVMResponse) GetCgroupPath() string { return "" } +func (m *CreateVMResponse) GetVSockPath() string { + if m != nil { + return m.VSockPath + } + return "" +} + type PauseVMRequest struct { VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1174,71 +1182,71 @@ func init() { func init() { proto.RegisterFile("firecracker.proto", fileDescriptor_a73317e9fb8da571) } var fileDescriptor_a73317e9fb8da571 = []byte{ - // 1044 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0x4f, 0x6f, 0xe2, 0x46, - 0x14, 0xaf, 0x63, 0x20, 0xe1, 0x11, 0x48, 0x18, 0x25, 0x59, 0xef, 0x2a, 0x8a, 0x90, 0xd5, 0x6e, - 0xd1, 0xaa, 0x8d, 0xd4, 0xa4, 0x87, 0xaa, 0x97, 0x2e, 0x81, 0x25, 0xcb, 0x6e, 0x9d, 0x45, 0x43, - 0x12, 0xa9, 0xed, 0x69, 0x62, 0x1e, 0xc4, 0x8b, 0xed, 0xa1, 0x9e, 0x31, 0xbb, 0xf9, 0x5a, 0x3d, - 0xf5, 0x6b, 0xb4, 0xdf, 0xa5, 0xf7, 0x6a, 0xc6, 0x06, 0x1b, 0x92, 0xb2, 0x91, 0x7a, 0xda, 0x13, - 0xcc, 0xef, 0xfd, 0x66, 0xde, 0x9f, 0xdf, 0x9b, 0x37, 0x86, 0xfa, 0xc8, 0x8b, 0xd0, 0x8d, 0x98, - 0x3b, 0xc1, 0xe8, 0x78, 0x1a, 0x71, 0xc9, 0x9f, 0x55, 0xe4, 0xdd, 0x14, 0x45, 0xb2, 0xb0, 0xff, - 0x2c, 0xc2, 0x4e, 0x3b, 0x42, 0x26, 0xf1, 0xda, 0xa1, 0xf8, 0x7b, 0x8c, 0x42, 0x12, 0x02, 0x85, - 0x6b, 0xa7, 0xd7, 0xb1, 0x8c, 0x86, 0xd1, 0x2c, 0xd3, 0xc2, 0xcc, 0xe9, 0x75, 0xc8, 0x4b, 0x00, - 0x87, 0xb9, 0xb7, 0x5e, 0x88, 0xed, 0xd1, 0xd8, 0xda, 0x68, 0x18, 0xcd, 0xca, 0x49, 0xe3, 0xb8, - 0x9b, 0x1d, 0x3e, 0xb7, 0xf2, 0x70, 0xe4, 0x8d, 0xe3, 0x88, 0x49, 0x8f, 0x87, 0x14, 0x82, 0xc5, - 0x1e, 0xd2, 0x84, 0x9d, 0xb7, 0x18, 0x85, 0xe8, 0xf7, 0x02, 0x36, 0xc6, 0x3e, 0x93, 0xb7, 0x96, - 0xa9, 0x1d, 0xec, 0x4c, 0x96, 0x61, 0x72, 0x04, 0x90, 0x30, 0x5b, 0xd1, 0x58, 0x58, 0x05, 0x4d, - 0x82, 0xc9, 0x02, 0x21, 0xa7, 0x50, 0xa6, 0x9c, 0xcb, 0x4e, 0xe4, 0xcd, 0xd0, 0x2a, 0xea, 0x50, - 0xf6, 0xf3, 0xa1, 0x2c, 0x8c, 0xb4, 0x1c, 0xcd, 0xff, 0x92, 0x1f, 0xa0, 0xa2, 0xff, 0x38, 0x3c, - 0x0e, 0xa5, 0xb0, 0x4a, 0x0d, 0xb3, 0x59, 0x39, 0x39, 0xc8, 0x6f, 0xcb, 0xcc, 0xb4, 0x32, 0xcc, - 0xa8, 0xe4, 0x0d, 0xd4, 0x2f, 0x50, 0x7e, 0xe0, 0xd1, 0xa4, 0x17, 0x4a, 0x8c, 0x46, 0xcc, 0x45, - 0x61, 0x6d, 0xea, 0xfd, 0x87, 0xf9, 0xfd, 0xab, 0x24, 0x5a, 0x0f, 0x57, 0xb7, 0x91, 0xe7, 0x50, - 0x6b, 0xf3, 0x50, 0x32, 0x2f, 0xc4, 0xa8, 0xad, 0x8e, 0xb7, 0xb6, 0x1a, 0x46, 0xb3, 0x48, 0x6b, - 0xee, 0x12, 0x4a, 0x7e, 0x04, 0xeb, 0xd5, 0x47, 0x4f, 0xb6, 0x46, 0x12, 0xa3, 0x96, 0xef, 0x5f, - 0x32, 0x31, 0x11, 0x1d, 0xf4, 0x51, 0xe2, 0xd0, 0x2a, 0x37, 0x8c, 0xe6, 0x16, 0xb5, 0xf0, 0x3f, - 0xec, 0xe4, 0x3b, 0xd8, 0x7e, 0xc3, 0x3c, 0x5f, 0x1d, 0xa5, 0xb4, 0xb0, 0x40, 0x57, 0xa8, 0x7a, - 0x9c, 0x07, 0xe9, 0xf6, 0xfb, 0xdc, 0x4a, 0x85, 0x75, 0xe9, 0x05, 0xc8, 0x63, 0x39, 0x40, 0x97, - 0x87, 0x43, 0x61, 0x55, 0x1a, 0x46, 0xb3, 0x4a, 0x6b, 0x72, 0x09, 0x25, 0x0d, 0xa8, 0xfc, 0xcc, - 0xc7, 0x5d, 0x6f, 0xc4, 0xb5, 0x7e, 0xdb, 0x5a, 0x9a, 0x8a, 0x9f, 0x41, 0x4a, 0x65, 0x07, 0x65, - 0xe4, 0xb9, 0x62, 0xc1, 0xaa, 0x26, 0x2a, 0x07, 0xcb, 0x30, 0xf9, 0x09, 0xaa, 0x67, 0xcc, 0xf7, - 0x39, 0x0f, 0x3b, 0x38, 0xf3, 0x5c, 0xb4, 0x6a, 0x3a, 0xce, 0xa7, 0xf9, 0x92, 0x2e, 0x11, 0x68, - 0xf5, 0x26, 0xbf, 0xb4, 0xff, 0x30, 0x60, 0x37, 0x6b, 0x5d, 0x31, 0xe5, 0xa1, 0xc0, 0x07, 0x7b, - 0xf7, 0x08, 0x60, 0xc0, 0xdd, 0x09, 0x4a, 0x1d, 0xce, 0x46, 0xd2, 0x4f, 0x62, 0x81, 0xac, 0x66, - 0x65, 0x3e, 0x2a, 0xab, 0xc2, 0xc3, 0x59, 0x1d, 0x01, 0xb4, 0xc7, 0x11, 0x8f, 0xa7, 0x9a, 0x54, - 0x4c, 0x7c, 0xb9, 0x0b, 0xc4, 0xfe, 0x12, 0x6a, 0x7d, 0x16, 0x8b, 0xf5, 0xb7, 0xcd, 0xfe, 0x0a, - 0x76, 0x28, 0x8a, 0x38, 0xf8, 0x04, 0xed, 0x2d, 0x54, 0x07, 0x92, 0x4f, 0xd7, 0xdf, 0xdc, 0xfb, - 0xda, 0x6e, 0x3c, 0xa4, 0xad, 0xfd, 0x1c, 0x76, 0xcf, 0x51, 0x5e, 0x3b, 0xbd, 0x70, 0xc4, 0xd7, - 0x39, 0xfd, 0xcb, 0x80, 0x7a, 0x8e, 0xf8, 0x79, 0xd4, 0x9d, 0x1c, 0x42, 0xf9, 0x5a, 0x05, 0xa3, - 0xcd, 0x25, 0x6d, 0x2e, 0xcf, 0xe6, 0x80, 0xdd, 0x85, 0xbd, 0x81, 0x4a, 0xc9, 0x41, 0xc9, 0x86, - 0x4c, 0xb2, 0x75, 0xf5, 0x7c, 0x06, 0x5b, 0x73, 0x5a, 0x9a, 0xd3, 0x56, 0x90, 0xae, 0xed, 0x1e, - 0x3c, 0xb9, 0x9a, 0x0e, 0x75, 0x47, 0xfe, 0xdf, 0xa3, 0x5e, 0xc0, 0xde, 0xf9, 0x23, 0x43, 0xb2, - 0x4f, 0x61, 0x7f, 0x85, 0x9b, 0xaa, 0x92, 0x77, 0x60, 0xac, 0x38, 0xf8, 0xdb, 0x58, 0x9e, 0x13, - 0x64, 0x0f, 0x8a, 0x17, 0x28, 0x2f, 0x06, 0x29, 0xb3, 0x18, 0xaa, 0x85, 0xf2, 0xd7, 0xee, 0x5f, - 0x89, 0x34, 0xbe, 0x82, 0xdb, 0xbf, 0x12, 0x0a, 0x73, 0x30, 0x10, 0xa9, 0x62, 0x85, 0x00, 0x03, - 0x41, 0x76, 0xc1, 0xbc, 0xea, 0x75, 0xb4, 0x3c, 0x55, 0x6a, 0xc6, 0xbd, 0x8e, 0x42, 0xce, 0x7b, - 0x1d, 0xad, 0x45, 0x95, 0x9a, 0xe3, 0xa4, 0x21, 0x72, 0x22, 0x95, 0xee, 0x89, 0xf4, 0x12, 0xea, - 0x7a, 0x08, 0xbf, 0xfa, 0x38, 0xe5, 0x02, 0xfb, 0xdc, 0xf7, 0xdc, 0x3b, 0x6b, 0xb3, 0x61, 0x34, - 0x6b, 0x27, 0xe4, 0xf8, 0x9e, 0x85, 0xd6, 0x87, 0xab, 0x90, 0xfd, 0x1a, 0xf6, 0x12, 0x01, 0xd2, - 0xc9, 0xb1, 0xae, 0xfa, 0x87, 0x50, 0x6e, 0x05, 0x6a, 0xda, 0x3a, 0xde, 0x8d, 0x4e, 0xcf, 0xa4, - 0x65, 0x36, 0x07, 0xec, 0x6f, 0xe1, 0xc9, 0x39, 0xca, 0xf4, 0x98, 0x74, 0x68, 0xae, 0x91, 0xe0, - 0x37, 0xb0, 0xee, 0xd3, 0x53, 0x15, 0xb2, 0x49, 0x97, 0x4e, 0x64, 0xe3, 0xb1, 0x93, 0x2e, 0xe1, - 0xdb, 0xdf, 0xc0, 0x41, 0x76, 0xf8, 0x40, 0x32, 0x29, 0xd6, 0x85, 0xf2, 0x8f, 0x99, 0x0f, 0x3d, - 0xa5, 0xa7, 0xa1, 0xa8, 0x9c, 0x5d, 0x19, 0x33, 0x5f, 0xe5, 0x6c, 0xa4, 0x39, 0xcf, 0x01, 0x75, - 0x21, 0x13, 0x6b, 0x9f, 0x8d, 0x51, 0xa4, 0x35, 0xa9, 0xb0, 0x0c, 0x52, 0x17, 0xb2, 0x35, 0x63, - 0x9e, 0xcf, 0x6e, 0x7c, 0x74, 0x30, 0xe0, 0xd1, 0x9d, 0x6e, 0x02, 0x93, 0xee, 0xb0, 0x65, 0x58, - 0x69, 0xdd, 0xf1, 0xc4, 0xa4, 0xcd, 0xdc, 0x5b, 0x4c, 0x1e, 0x71, 0x93, 0xc2, 0x70, 0x81, 0x28, - 0x7b, 0x37, 0xc2, 0xf9, 0x21, 0xc5, 0xc4, 0x3e, 0x5a, 0x20, 0xe4, 0x18, 0xc8, 0xeb, 0x78, 0x8c, - 0xd2, 0xbf, 0x69, 0xf9, 0x3e, 0x77, 0xf5, 0xf7, 0x84, 0xd0, 0x3d, 0x63, 0x52, 0x72, 0x7b, 0xcf, - 0xa2, 0x22, 0x4b, 0xf9, 0x5d, 0xe6, 0xf9, 0x71, 0xa4, 0xdf, 0x68, 0x1d, 0xd9, 0xed, 0x32, 0xac, - 0xb2, 0x74, 0xd8, 0x7b, 0x1e, 0x75, 0x59, 0xec, 0x4b, 0xa1, 0x1f, 0x60, 0x93, 0x56, 0x82, 0x0c, - 0xd2, 0x0c, 0x2f, 0x5c, 0x30, 0xca, 0x29, 0x23, 0x83, 0xc8, 0x01, 0x94, 0x06, 0x1f, 0xd8, 0xb4, - 0x17, 0xea, 0xd7, 0xd5, 0xa4, 0x25, 0xa1, 0x57, 0xc4, 0x82, 0x4d, 0x85, 0xbf, 0x8b, 0xa5, 0x7e, - 0x41, 0x4d, 0xba, 0x29, 0x92, 0xa5, 0xaa, 0xfc, 0x25, 0x8b, 0xc6, 0xa8, 0xbb, 0x6d, 0x3b, 0xa9, - 0xbc, 0x9c, 0x03, 0xca, 0x63, 0x62, 0x4d, 0x2a, 0x5f, 0x4d, 0x3c, 0xca, 0x0c, 0xd2, 0x0c, 0x2e, - 0x99, 0x9f, 0x16, 0xac, 0x96, 0x32, 0x32, 0xc8, 0x46, 0x78, 0xba, 0xd4, 0xfb, 0x9f, 0x6a, 0x14, - 0xf2, 0x3d, 0xec, 0x6b, 0x4e, 0x9f, 0xfb, 0xbe, 0x17, 0x8e, 0xf5, 0x67, 0xca, 0x8c, 0xf9, 0x73, - 0xe1, 0xf7, 0xc5, 0x43, 0xc6, 0x17, 0x5f, 0x3f, 0x70, 0x49, 0xc9, 0x16, 0x14, 0xda, 0xef, 0xfa, - 0xbf, 0xec, 0x7e, 0xa1, 0xfe, 0x9d, 0xf5, 0x2e, 0x3a, 0xbb, 0xc6, 0xd9, 0xe6, 0xaf, 0x45, 0xfd, - 0x8d, 0x79, 0x53, 0xd2, 0x3f, 0xa7, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x58, 0x5c, 0x22, 0x3c, - 0x8c, 0x0a, 0x00, 0x00, + // 1042 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x56, 0xdd, 0x6e, 0xe2, 0x46, + 0x14, 0xae, 0x63, 0x20, 0xe1, 0x10, 0x48, 0x18, 0x25, 0x59, 0xef, 0x2a, 0x8a, 0x10, 0x6a, 0xb7, + 0x68, 0xd5, 0x46, 0x6a, 0xd2, 0x8b, 0xaa, 0x37, 0x5d, 0x02, 0x4b, 0x96, 0xdd, 0x3a, 0x8b, 0x86, + 0x24, 0x52, 0xdb, 0xab, 0x89, 0x39, 0x38, 0x5e, 0x6c, 0x0f, 0xf5, 0x8c, 0xd9, 0xcd, 0x9b, 0xf5, + 0x31, 0xda, 0xbe, 0x4b, 0xef, 0xab, 0x19, 0x1b, 0x6c, 0x48, 0xca, 0x46, 0xea, 0xdd, 0x5e, 0xc1, + 0x7c, 0xe7, 0x9b, 0x39, 0x3f, 0xdf, 0x99, 0x33, 0x86, 0xfa, 0xd8, 0x8b, 0xd0, 0x89, 0x98, 0x33, + 0xc1, 0xe8, 0x78, 0x1a, 0x71, 0xc9, 0x9f, 0x55, 0xe4, 0xdd, 0x14, 0x45, 0xb2, 0x68, 0xfe, 0x51, + 0x84, 0x9d, 0x4e, 0x84, 0x4c, 0xe2, 0xb5, 0x4d, 0xf1, 0xf7, 0x18, 0x85, 0x24, 0x04, 0x0a, 0xd7, + 0x76, 0xbf, 0x6b, 0x19, 0x0d, 0xa3, 0x55, 0xa6, 0x85, 0x99, 0xdd, 0xef, 0x92, 0x97, 0x00, 0x36, + 0x73, 0x6e, 0xbd, 0x10, 0x3b, 0x63, 0xd7, 0xda, 0x68, 0x18, 0xad, 0xca, 0x49, 0xe3, 0xb8, 0x97, + 0x1d, 0x3e, 0xb7, 0xf2, 0x70, 0xec, 0xb9, 0x71, 0xc4, 0xa4, 0xc7, 0x43, 0x0a, 0xc1, 0x62, 0x0f, + 0x69, 0xc1, 0xce, 0x5b, 0x8c, 0x42, 0xf4, 0xfb, 0x01, 0x73, 0x71, 0xc0, 0xe4, 0xad, 0x65, 0x6a, + 0x07, 0x3b, 0x93, 0x65, 0x98, 0x1c, 0x01, 0x24, 0xcc, 0x76, 0xe4, 0x0a, 0xab, 0xa0, 0x49, 0x30, + 0x59, 0x20, 0xe4, 0x14, 0xca, 0x94, 0x73, 0xd9, 0x8d, 0xbc, 0x19, 0x5a, 0x45, 0x1d, 0xca, 0x7e, + 0x3e, 0x94, 0x85, 0x91, 0x96, 0xa3, 0xf9, 0x5f, 0xf2, 0x03, 0x54, 0xf4, 0x1f, 0x9b, 0xc7, 0xa1, + 0x14, 0x56, 0xa9, 0x61, 0xb6, 0x2a, 0x27, 0x07, 0xf9, 0x6d, 0x99, 0x99, 0x56, 0x46, 0x19, 0x95, + 0xbc, 0x81, 0xfa, 0x05, 0xca, 0x0f, 0x3c, 0x9a, 0xf4, 0x43, 0x89, 0xd1, 0x98, 0x39, 0x28, 0xac, + 0x4d, 0xbd, 0xff, 0x30, 0xbf, 0x7f, 0x95, 0x44, 0xeb, 0xe1, 0xea, 0x36, 0xf2, 0x1c, 0x6a, 0x1d, + 0x1e, 0x4a, 0xe6, 0x85, 0x18, 0x75, 0xd4, 0xf1, 0xd6, 0x56, 0xc3, 0x68, 0x15, 0x69, 0xcd, 0x59, + 0x42, 0xc9, 0x8f, 0x60, 0xbd, 0xfa, 0xe8, 0xc9, 0xf6, 0x58, 0x62, 0xd4, 0xf6, 0xfd, 0x4b, 0x26, + 0x26, 0xa2, 0x8b, 0x3e, 0x4a, 0x1c, 0x59, 0xe5, 0x86, 0xd1, 0xda, 0xa2, 0x16, 0xfe, 0x87, 0x9d, + 0x7c, 0x07, 0xdb, 0x6f, 0x98, 0xe7, 0xab, 0xa3, 0x94, 0x16, 0x16, 0xe8, 0x0a, 0x55, 0x8f, 0xf3, + 0x20, 0xdd, 0x7e, 0x9f, 0x5b, 0xa9, 0xb0, 0x2e, 0xbd, 0x00, 0x79, 0x2c, 0x87, 0xe8, 0xf0, 0x70, + 0x24, 0xac, 0x4a, 0xc3, 0x68, 0x55, 0x69, 0x4d, 0x2e, 0xa1, 0xa4, 0x01, 0x95, 0x9f, 0xb9, 0xdb, + 0xf3, 0xc6, 0x5c, 0xeb, 0xb7, 0xad, 0xa5, 0xa9, 0xf8, 0x19, 0xa4, 0x54, 0xb6, 0x51, 0x46, 0x9e, + 0x23, 0x16, 0xac, 0x6a, 0xa2, 0x72, 0xb0, 0x0c, 0x93, 0x9f, 0xa0, 0x7a, 0xc6, 0x7c, 0x9f, 0xf3, + 0xb0, 0x8b, 0x33, 0xcf, 0x41, 0xab, 0xa6, 0xe3, 0x7c, 0x9a, 0x2f, 0xe9, 0x12, 0x81, 0x56, 0x6f, + 0xf2, 0xcb, 0xe6, 0x9f, 0x06, 0xec, 0x66, 0xad, 0x2b, 0xa6, 0x3c, 0x14, 0xf8, 0x60, 0xef, 0x1e, + 0x01, 0x0c, 0xb9, 0x33, 0x41, 0xa9, 0xc3, 0xd9, 0x48, 0xfa, 0x49, 0x2c, 0x90, 0xd5, 0xac, 0xcc, + 0x47, 0x65, 0x55, 0x78, 0x38, 0xab, 0x23, 0x80, 0x8e, 0x1b, 0xf1, 0x78, 0xaa, 0x49, 0xc5, 0xc4, + 0x97, 0xb3, 0x40, 0xc8, 0x21, 0x94, 0xaf, 0x55, 0x30, 0xda, 0x5c, 0xd2, 0xe6, 0xf2, 0x6c, 0x0e, + 0x34, 0xbf, 0x84, 0xda, 0x80, 0xc5, 0x62, 0xfd, 0x5d, 0x6c, 0x7e, 0x05, 0x3b, 0x14, 0x45, 0x1c, + 0x7c, 0x82, 0xf6, 0x16, 0xaa, 0x43, 0xc9, 0xa7, 0xeb, 0xef, 0xf5, 0x7d, 0xe5, 0x37, 0x1e, 0x52, + 0xbe, 0xf9, 0x1c, 0x76, 0xcf, 0x51, 0x5e, 0xdb, 0xfd, 0x70, 0xcc, 0xd7, 0x39, 0xfd, 0xcb, 0x80, + 0x7a, 0x8e, 0xf8, 0x59, 0xa8, 0xd2, 0x83, 0xbd, 0xa1, 0x4a, 0xc9, 0x46, 0xc9, 0x46, 0x4c, 0xb2, + 0x75, 0xf5, 0x7c, 0x06, 0x5b, 0x73, 0x5a, 0x9a, 0xd3, 0x56, 0x90, 0xae, 0x9b, 0x7d, 0x78, 0x72, + 0x35, 0x1d, 0xe9, 0x7e, 0xfd, 0xbf, 0x47, 0xbd, 0x80, 0xbd, 0xf3, 0x47, 0x86, 0xd4, 0x3c, 0x85, + 0xfd, 0x15, 0x6e, 0xaa, 0x4a, 0xde, 0x81, 0xb1, 0xe2, 0xe0, 0x6f, 0x63, 0x79, 0x8a, 0x90, 0x3d, + 0x28, 0x5e, 0xa0, 0xbc, 0x18, 0xa6, 0xcc, 0x62, 0xa8, 0x16, 0xca, 0x5f, 0x67, 0x70, 0x25, 0xd2, + 0xf8, 0x0a, 0xce, 0xe0, 0x4a, 0x28, 0xcc, 0xc6, 0x40, 0xa4, 0x8a, 0x15, 0x02, 0x0c, 0x04, 0xd9, + 0x05, 0xf3, 0xaa, 0xdf, 0xd5, 0xf2, 0x54, 0xa9, 0x19, 0xf7, 0xbb, 0x0a, 0x39, 0xef, 0x77, 0xb5, + 0x16, 0x55, 0x6a, 0xba, 0x49, 0x43, 0xe4, 0x44, 0x2a, 0xdd, 0x13, 0xe9, 0x25, 0xd4, 0xf5, 0x88, + 0x7e, 0xf5, 0x71, 0xca, 0x05, 0x0e, 0xb8, 0xef, 0x39, 0x77, 0xd6, 0x66, 0xc3, 0x68, 0xd5, 0x4e, + 0xc8, 0xf1, 0x3d, 0x0b, 0xad, 0x8f, 0x56, 0xa1, 0xe6, 0x6b, 0xd8, 0x4b, 0x04, 0x48, 0xe7, 0xca, + 0xba, 0xea, 0x1f, 0x42, 0xb9, 0x1d, 0xa8, 0x59, 0x6c, 0x7b, 0x37, 0x3a, 0x3d, 0x93, 0x96, 0xd9, + 0x1c, 0x68, 0x7e, 0x0b, 0x4f, 0xce, 0x51, 0xa6, 0xc7, 0xa4, 0x23, 0x75, 0x8d, 0x04, 0xbf, 0x81, + 0x75, 0x9f, 0x9e, 0xaa, 0x90, 0xcd, 0xc1, 0x74, 0x5e, 0x1b, 0x8f, 0x9d, 0x83, 0x09, 0xbf, 0xf9, + 0x0d, 0x1c, 0x64, 0x87, 0x0f, 0x25, 0x93, 0x62, 0x5d, 0x28, 0xff, 0x98, 0xf9, 0xd0, 0x53, 0x7a, + 0x1a, 0x8a, 0xca, 0xd9, 0x91, 0x31, 0xf3, 0x55, 0xce, 0x46, 0x9a, 0xf3, 0x1c, 0x50, 0x17, 0x32, + 0xb1, 0x0e, 0x98, 0x8b, 0x22, 0xad, 0x49, 0x85, 0x65, 0x90, 0xba, 0x90, 0xed, 0x19, 0xf3, 0x7c, + 0x76, 0xe3, 0xa3, 0x8d, 0x01, 0x8f, 0xee, 0x74, 0x13, 0x98, 0x74, 0x87, 0x2d, 0xc3, 0x4a, 0xeb, + 0xae, 0x27, 0x26, 0x1d, 0xe6, 0xdc, 0x62, 0xf2, 0xc4, 0x9b, 0x14, 0x46, 0x0b, 0x44, 0xd9, 0x7b, + 0x11, 0xce, 0x0f, 0x29, 0x26, 0xf6, 0xf1, 0x02, 0x21, 0xc7, 0x40, 0x5e, 0xc7, 0x2e, 0x4a, 0xff, + 0xa6, 0xed, 0xfb, 0xdc, 0xd1, 0x5f, 0x1b, 0x42, 0xf7, 0x8c, 0x49, 0xc9, 0xed, 0x3d, 0x8b, 0x8a, + 0x2c, 0xe5, 0xf7, 0x98, 0xe7, 0xc7, 0x91, 0x7e, 0xc1, 0x75, 0x64, 0xb7, 0xcb, 0xb0, 0xca, 0xd2, + 0x66, 0xef, 0x79, 0xd4, 0x63, 0xb1, 0x2f, 0x85, 0x7e, 0x9e, 0x4d, 0x5a, 0x09, 0x32, 0x48, 0x33, + 0xbc, 0x70, 0xc1, 0x28, 0xa7, 0x8c, 0x0c, 0x22, 0x07, 0x50, 0x1a, 0x7e, 0x60, 0xd3, 0x7e, 0xa8, + 0xdf, 0x5e, 0x93, 0x96, 0x84, 0x5e, 0x11, 0x0b, 0x36, 0x15, 0xfe, 0x2e, 0x96, 0xfa, 0x7d, 0x35, + 0xe9, 0xa6, 0x48, 0x96, 0xaa, 0xf2, 0x97, 0x2c, 0x72, 0x51, 0x77, 0xdb, 0x76, 0x52, 0x79, 0x39, + 0x07, 0x94, 0xc7, 0xc4, 0x9a, 0x54, 0xbe, 0x9a, 0x78, 0x94, 0x19, 0xa4, 0x19, 0x5c, 0x32, 0x3f, + 0x2d, 0x58, 0x2d, 0x65, 0x64, 0x50, 0x13, 0xe1, 0xe9, 0x52, 0xef, 0x7f, 0xaa, 0x51, 0xc8, 0xf7, + 0xb0, 0xaf, 0x39, 0x03, 0xee, 0xfb, 0x5e, 0xe8, 0xea, 0x8f, 0x98, 0x19, 0xf3, 0xe7, 0xc2, 0xef, + 0x8b, 0x87, 0x8c, 0x2f, 0xbe, 0x7e, 0xe0, 0x92, 0x92, 0x2d, 0x28, 0x74, 0xde, 0x0d, 0x7e, 0xd9, + 0xfd, 0x42, 0xfd, 0x3b, 0xeb, 0x5f, 0x74, 0x77, 0x8d, 0xb3, 0xcd, 0x5f, 0x8b, 0xfa, 0x0b, 0xf4, + 0xa6, 0xa4, 0x7f, 0x4e, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xd1, 0x2a, 0x11, 0x46, 0xaa, 0x0a, + 0x00, 0x00, } diff --git a/proto/firecracker.proto b/proto/firecracker.proto index 3dfe44db9..86f308bc7 100644 --- a/proto/firecracker.proto +++ b/proto/firecracker.proto @@ -49,6 +49,7 @@ message CreateVMResponse { string LogFifoPath = 3; string MetricsFifoPath = 4; string CgroupPath = 5; + string VSockPath = 6; } message PauseVMRequest { diff --git a/runtime/service.go b/runtime/service.go index 6eb5ccb3c..fdeebaad8 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -502,6 +502,7 @@ func (s *service) CreateVM(requestCtx context.Context, request *proto.CreateVMRe resp.MetricsFifoPath = s.machineConfig.MetricsFifo resp.LogFifoPath = s.machineConfig.LogFifo resp.SocketPath = s.shimDir.FirecrackerSockPath() + resp.VSockPath = s.shimDir.FirecrackerVSockPath() if c, ok := s.jailer.(cgroupPather); ok { resp.CgroupPath = c.CgroupPath() } diff --git a/snapshotter/Makefile b/snapshotter/Makefile index bfc663aae..118e4abed 100644 --- a/snapshotter/Makefile +++ b/snapshotter/Makefile @@ -34,15 +34,12 @@ INTEG_TESTNAMES=$(shell docker run --rm \ $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ -c "go test -list . | sed '$$d' | grep $(INTEG_TEST_SUFFIX)") -all: demux-snapshotter http-address-resolver +all: demux-snapshotter demux-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM) go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ -http-address-resolver: $(SOURCES) $(GOMOD) $(GOSUM) - go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ internal/http_address_resolver.go - -install: demux-snapshotter http-address-resolver +install: demux-snapshotter install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $^ test: @@ -79,6 +76,8 @@ logs: clean: - rm -f demux-snapshotter +# Leaving legacy binary cleanup so that existing copies can get to a clean +# state after pulling from main - rm -f http-address-resolver distclean: clean diff --git a/snapshotter/app/service.go b/snapshotter/app/service.go index 101b2d2fe..b654daf80 100644 --- a/snapshotter/app/service.go +++ b/snapshotter/app/service.go @@ -37,7 +37,6 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/metrics" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/metrics/discovery" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" - proxyaddress "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy/address" ) // Run the demultiplexing snapshotter service. @@ -137,25 +136,7 @@ func Run(config config.Config) error { return nil } -func initResolver(config config.Config) (proxyaddress.Resolver, error) { - resolverConfig := config.Snapshotter.Proxy.Address.Resolver - switch resolverConfig.Type { - case "http": - return proxyaddress.NewHTTPResolver(resolverConfig.Address), nil - default: - return nil, fmt.Errorf("invalid resolver type: %s", resolverConfig.Type) - } -} - -const base10 = 10 -const bits32 = 32 - func initCache(config config.Config, monitor *metrics.Monitor) (*cache.RemoteSnapshotterCache, error) { - resolver, err := initResolver(config) - if err != nil { - return nil, err - } - dialTimeout, err := time.ParseDuration(config.Snapshotter.Dialer.Timeout) if err != nil { return nil, fmt.Errorf("Error parsing dialer retry interval from config: %w", err) @@ -172,48 +153,31 @@ func initCache(config config.Config, monitor *metrics.Monitor) (*cache.RemoteSna ) } - dial := func(ctx context.Context, namespace string) (net.Conn, error) { - r := resolver - response, err := r.Get(namespace) - - if err != nil { - return nil, err - } - host := response.Address - port, err := strconv.ParseUint(response.SnapshotterPort, base10, bits32) - if err != nil { - return nil, err - } - + dial := func(ctx context.Context, host string) (net.Conn, error) { + // Todo: wire RemoteSnapshotterConfig here somehow + port := uint64(10000) return vsockDial(ctx, host, port) } dialer := proxy.Dialer{Dial: dial, Timeout: dialTimeout} - fetch := func(ctx context.Context, namespace string) (*proxy.RemoteSnapshotter, error) { - r := resolver - response, err := r.Get(namespace) - if err != nil { - return nil, err - } - host := response.Address - port, err := strconv.ParseUint(response.SnapshotterPort, base10, bits32) - if err != nil { - return nil, err - } + fetch := func(ctx context.Context, snapshotterConfig proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { dial := func(ctx context.Context, namespace string) (net.Conn, error) { - return vsockDial(ctx, host, port) + return vsockDial(ctx, snapshotterConfig.VSockPath, uint64(snapshotterConfig.RemoteSnapshotterPort)) } var metricsProxy *metrics.Proxy if config.Snapshotter.Metrics.Enable { - metricsProxy, err = initMetricsProxy(config, monitor, host, response.MetricsPort, response.Labels) + metricsProxy, err = initMetricsProxy(config, monitor, + snapshotterConfig.VSockPath, + snapshotterConfig.MetricsPort, + snapshotterConfig.MetricsLabels) if err != nil { return nil, err } } - return proxy.NewRemoteSnapshotter(ctx, host, dial, metricsProxy) + return proxy.NewRemoteSnapshotter(ctx, snapshotterConfig.VSockPath, dial, metricsProxy) } opts := make([]cache.SnapshotterCacheOption, 0) @@ -247,14 +211,9 @@ func initMetricsProxyMonitor(portRange string) (*metrics.Monitor, error) { return metrics.NewMonitor(lower, upper) } -func initMetricsProxy(config config.Config, monitor *metrics.Monitor, host, port string, labels map[string]string) (*metrics.Proxy, error) { - metricsPort, err := strconv.ParseUint(port, base10, bits32) - if err != nil { - return nil, err - } - +func initMetricsProxy(config config.Config, monitor *metrics.Monitor, host string, port uint32, labels map[string]string) (*metrics.Proxy, error) { metricsDialer := func(ctx context.Context, _, _ string) (net.Conn, error) { - return vsock.DialContext(ctx, host, uint32(metricsPort), vsock.WithLogger(log.G(ctx))) + return vsock.DialContext(ctx, host, port, vsock.WithLogger(log.G(ctx))) } metricsHost := config.Snapshotter.Metrics.Host diff --git a/snapshotter/demux/cache/cache.go b/snapshotter/demux/cache/cache.go index 04d397711..fee21bc0e 100644 --- a/snapshotter/demux/cache/cache.go +++ b/snapshotter/demux/cache/cache.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/snapshots" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" "github.com/hashicorp/go-multierror" @@ -26,7 +27,7 @@ import ( ) // SnapshotterProvider defines a snapshotter fetch function. -type SnapshotterProvider = func(context.Context, string) (*proxy.RemoteSnapshotter, error) +type SnapshotterProvider = func(context.Context, proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) // RemoteSnapshotterCache implements a cache for remote snapshotters. type RemoteSnapshotterCache struct { @@ -101,8 +102,8 @@ func (c *RemoteSnapshotterCache) List() []string { return keys } -// Get fetches and caches the snapshotter for a given key. -func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +// Put creates and adds a snapshotter to the cache. +func (c *RemoteSnapshotterCache) Put(ctx context.Context, key string, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { c.mutex.RLock() snapshotter, ok := c.snapshotters[key] c.mutex.RUnlock() @@ -113,7 +114,7 @@ func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.Re c.mutex.Unlock() if !ok { - newSnapshotter, err := c.fetch(ctx, key) + newSnapshotter, err := c.fetch(ctx, config) if err != nil { return nil, err } @@ -131,6 +132,18 @@ func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.Re return snapshotter, nil } +// Get retrieves the snapshotter for a given key. +func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { + c.mutex.RLock() + snapshotter, ok := c.snapshotters[key] + c.mutex.RUnlock() + + if !ok { + return nil, fmt.Errorf("error getting snapshotter for %s: %w", key, errdefs.ErrNotFound) + } + return snapshotter, nil +} + // WalkAll applies the provided function to all cached snapshotters. func (c *RemoteSnapshotterCache) WalkAll(ctx context.Context, fn snapshots.WalkFunc, filters ...string) error { c.mutex.RLock() diff --git a/snapshotter/demux/cache/cache_test.go b/snapshotter/demux/cache/cache_test.go index abf10b010..b9f72cebe 100644 --- a/snapshotter/demux/cache/cache_test.go +++ b/snapshotter/demux/cache/cache_test.go @@ -19,6 +19,7 @@ import ( "fmt" "testing" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/snapshots" "github.com/hashicorp/go-multierror" @@ -26,22 +27,22 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" ) -func getSnapshotterOkFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotterOkFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.SuccessfulSnapshotter{}}, nil } -func getSnapshotterErrorFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotterErrorFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return nil, errors.New("Mock retrieve snapshotter error") } -func getFailingSnapshotterOkFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getFailingSnapshotterOkFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.FailingSnapshotter{}}, nil } func getSnapshotterFromEmptyCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) _, err := uut.Get(context.Background(), "SnapshotterKey") - if err != nil { + if err != nil && !errors.Is(err, errdefs.ErrNotFound) { return fmt.Errorf("Fetch from empty cache incorrectly resulted in error: %w", err) } return nil @@ -49,7 +50,7 @@ func getSnapshotterFromEmptyCache() error { func getCachedSnapshotter() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -81,10 +82,10 @@ func applyWalkFunctionOnEmptyCache() error { func applyWalkFunctionToAllCachedSnapshotters() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "Snapshotter-A"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-A", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter A to empty cache incorrectly resulted in error: %w", err) } - if _, err := uut.Get(context.Background(), "Snapshotter-B"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-B", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter B to cache incorrectly resulted in error: %w", err) } if err := uut.WalkAll(context.Background(), successfulWalk); err != nil { @@ -95,7 +96,7 @@ func applyWalkFunctionToAllCachedSnapshotters() error { func applyWalkFunctionPropagatesErrors() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "Snapshotter-A"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-A", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter A to empty cache incorrectly resulted in error: %w", err) } // The failing snapshotter mock will fail all Walk calls before applying @@ -120,7 +121,7 @@ func evictSnapshotterFromEmptyCache() error { func evictSnapshotterFromCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -132,7 +133,7 @@ func evictSnapshotterFromCache() error { func evictSnapshotterFromCachePropagatesCloseError() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -162,8 +163,8 @@ func closeCacheWithNonEmptyCache() error { func closeCacheReturnsAllOccurringErrors() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - uut.Get(context.Background(), "FailingSnapshotterKey") - uut.Get(context.Background(), "AnotherFailingSnapshotterKey") + uut.Put(context.Background(), "FailingSnapshotterKey", proxy.RemoteSnapshotterConfig{}) + uut.Put(context.Background(), "AnotherFailingSnapshotterKey", proxy.RemoteSnapshotterConfig{}) err := uut.Close() if err == nil { @@ -189,7 +190,7 @@ func listEmptyCache() error { func listNonEmptyCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - uut.Get(context.Background(), "OkSnapshotterKey") + uut.Put(context.Background(), "OkSnapshotterKey", proxy.RemoteSnapshotterConfig{}) keys := uut.List() if len(keys) != 1 { diff --git a/snapshotter/demux/cache/evict_test.go b/snapshotter/demux/cache/evict_test.go index 8145991ab..2b7061ab7 100644 --- a/snapshotter/demux/cache/evict_test.go +++ b/snapshotter/demux/cache/evict_test.go @@ -26,11 +26,11 @@ import ( "go.uber.org/goleak" ) -func getSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.SuccessfulSnapshotter{}}, nil } -func getErrorSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getErrorSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.FailingSnapshotter{}}, nil } @@ -52,7 +52,7 @@ func TestCacheEvictionOnConnectionFailure(t *testing.T) { cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) @@ -88,7 +88,7 @@ func TestCacheNotEvictedIfConnectionIsHealthy(t *testing.T) { cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) @@ -121,7 +121,7 @@ func TestBackgroundEnforcersCanBeStopped(t *testing.T) { dialer := proxy.Dialer{Dial: dial, Timeout: 1 * time.Second} cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") cache.Close() @@ -140,7 +140,7 @@ func TestLogErrorOnEvictionFailure(t *testing.T) { cache := NewRemoteSnapshotterCache(getErrorSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) diff --git a/snapshotter/demux/opt.go b/snapshotter/demux/opt.go new file mode 100644 index 000000000..3b9ade525 --- /dev/null +++ b/snapshotter/demux/opt.go @@ -0,0 +1,49 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package demux + +import ( + "encoding/json" + "strconv" + + "github.com/containerd/containerd/snapshots" +) + +// WithVSockPath sets the Firecracker vsock path +func WithVSockPath(vsockPath string) snapshots.Opt { + return snapshots.WithLabels(map[string]string{VSockPathLabel: vsockPath}) +} + +// WithRemoteSnapshotterPort sets the vsock port that the remote-snapshotter +// is listening on for its snapshot server +func WithRemoteSnapshotterPort(port uint32) snapshots.Opt { + return snapshots.WithLabels(map[string]string{RemoteSnapshotterPortLabel: strconv.FormatUint(uint64(port), 10)}) +} + +// WithProxyMetricsPort sets the vsock port that the remote-snapshotter +// is listening on for its metrics server +func WithProxyMetricsPort(port uint32) snapshots.Opt { + return snapshots.WithLabels(map[string]string{MetricsPortLabel: strconv.FormatUint(uint64(port), 10)}) +} + +// WithProxyMetricLabels sets labels that will be attached to proxy metrics +func WithProxyMetricLabels(labels map[string]string) (snapshots.Opt, error) { + serialized, err := json.Marshal(labels) + if err != nil { + return nil, err + } + return snapshots.WithLabels(map[string]string{ + MetricsLabelsLabel: string(serialized), + }), nil +} diff --git a/snapshotter/demux/proxy/address/http_resolver.go b/snapshotter/demux/proxy/address/http_resolver.go deleted file mode 100644 index 19f009338..000000000 --- a/snapshotter/demux/proxy/address/http_resolver.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" -) - -// HTTPClient defines the interface for the client used -// for HTTP communications in the resolver. -type HTTPClient interface { - Get(string) (*http.Response, error) -} - -// ResponseReader defines the reading function interface. -type ResponseReader func(io.Reader) ([]byte, error) - -// HTTPResolver implements a proxy address resolver via HTTP. -type HTTPResolver struct { - url string - client HTTPClient -} - -// NewHTTPResolver creates a new instance of HttpResolver with specified the URL. -func NewHTTPResolver(url string) HTTPResolver { - return HTTPResolver{url: url, client: http.DefaultClient} -} - -func requestURL(url string, namespace string) string { - return fmt.Sprintf("%s/address?namespace=%s", url, namespace) -} - -// Get queries the proxy network type and address for the specified namespace. -func (h HTTPResolver) Get(namespace string) (Response, error) { - url := requestURL(h.url, namespace) - - httpResponse, err := h.client.Get(url) - if err != nil { - return Response{}, err - } - defer httpResponse.Body.Close() - - body, err := ioutil.ReadAll(httpResponse.Body) - if err != nil { - return Response{}, err - } - - code := httpResponse.StatusCode - if code != 200 { - return Response{}, fmt.Errorf("failed to GET %s: status=%d, body=%s", url, code, body) - } - - var response Response - err = json.Unmarshal(body, &response) - if err != nil { - return Response{}, err - } - return response, nil -} diff --git a/snapshotter/demux/proxy/address/http_resolver_test.go b/snapshotter/demux/proxy/address/http_resolver_test.go deleted file mode 100644 index a8b1676c8..000000000 --- a/snapshotter/demux/proxy/address/http_resolver_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -import ( - "errors" - "fmt" - "io" - "net/http" - "testing" -) - -func returnErrorOnHTTPClientError() error { - client := mockClient{} - client.getError = errors.New("error on HTTP GET") - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on HTTP client GET") - } - - return nil -} - -func returnErrorOnResponseReadError() error { - reader := mockReader{response: nil, err: errors.New("mock error on read")} - client := mockClient{getError: nil, getResponse: http.Response{Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on body read") - } - - return nil -} - -func returnErrorOnJSONParserError() error { - reader := mockReader{response: []byte(`{"network": "unix"`), err: io.EOF} - client := mockClient{getError: nil, getResponse: http.Response{Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on JSON parsing") - } - - return nil -} - -func happyPath() error { - reader := mockReader{response: []byte( - `{ - "network": "unix", - "address": "/path/to/snapshotter.vsock", - "snapshotter_port": "10000", - "metrics_port": "10001", - "labels": { - "test1": "label1", - "test2": "label2" - } - }`), err: io.EOF} - client := mockClient{getError: nil, getResponse: http.Response{StatusCode: 200, Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - actual, err := uut.Get("namespace") - if err != nil { - return fmt.Errorf("expected no error from HTTP resolver, but got %+v", err) - } - - if actual.Network != "unix" { - return fmt.Errorf("Expected network 'unix' but actual %s", actual.Address) - } - if actual.Address != "/path/to/snapshotter.vsock" { - return fmt.Errorf("Expected address '/path/to/snapshotter.vsock' but actual %s", actual.Address) - } - if actual.SnapshotterPort != "10000" { - return fmt.Errorf("Expected metrics port '10000' but actual %s", actual.MetricsPort) - } - if actual.MetricsPort != "10001" { - return fmt.Errorf("Expected metrics port '10001' but actual %s", actual.MetricsPort) - } - if actual.Labels["test1"] != "label1" { - return fmt.Errorf("Expected metrics label key='test1' value='label1' but actual %s", actual.Labels["test1"]) - } - if actual.Labels["test2"] != "label2" { - return fmt.Errorf("Expected metrics label key='test2' value='label2' but actual %s", actual.Labels["test1"]) - } - return nil -} - -func TestHttpResolverGet(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - run func() error - }{ - {"HttpClientError", returnErrorOnHTTPClientError}, - {"ResponseReaderError", returnErrorOnResponseReadError}, - {"JsonParserError", returnErrorOnJSONParserError}, - {"HappyPath", happyPath}, - } - - for _, test := range tests { - if err := test.run(); err != nil { - t.Fatalf(test.name+" test failed: %s", err) - } - } -} - -func TestRequestUrlFormat(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - url string - namespace string - expected string - }{ - {"NS-1", "http://127.0.0.1:10001", "ns-1", "http://127.0.0.1:10001/address?namespace=ns-1"}, - {"NS-2", "http://localhost:10001", "ns-2", "http://localhost:10001/address?namespace=ns-2"}, - } - - for _, test := range tests { - if actual := requestURL(test.url, test.namespace); actual != test.expected { - t.Fatalf("%s failed: expected %s actual %s", test.name, test.expected, actual) - } - } -} - -type mockClient struct { - getResponse http.Response - getError error -} - -func (c *mockClient) Get(url string) (*http.Response, error) { - if c.getError != nil { - return nil, c.getError - } - return &c.getResponse, nil -} - -type mockReader struct { - response []byte - err error -} - -func (r *mockReader) Read(p []byte) (int, error) { - return copy(p, r.response), r.err -} - -func (r *mockReader) Close() error { - return nil -} diff --git a/snapshotter/demux/proxy/address/resolver.go b/snapshotter/demux/proxy/address/resolver.go deleted file mode 100644 index e3025944f..000000000 --- a/snapshotter/demux/proxy/address/resolver.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -// Response to the proxy network address resolver. -type Response struct { - // Network type used in net.Dial. - // - // Reference: https://pkg.go.dev/net#Dial - Network string `json:"network"` - - // Network address used in net.Dial. - Address string `json:"address"` - - // SnapshotterPort is the port used in vsock.DialContext for sending snapshotter API requests to the remote snapshotter. - SnapshotterPort string `json:"snapshotter_port"` - - // MetricsPort is the port used in vsock.DialContext for sending metrics requests to the remote snapshotter. - MetricsPort string `json:"metrics_port"` - - // Labels is a map used for applying labels to metrics. - Labels map[string]string `json:"labels"` -} - -// Resolver for the proxy network address. -type Resolver interface { - // Fetch the network address used to forward snapshotter - // requests by namespace lookup from a remote service call. - Get(namespace string) (Response, error) -} diff --git a/snapshotter/demux/proxy/snapshotter.go b/snapshotter/demux/proxy/snapshotter.go index 39315128f..08906fc23 100644 --- a/snapshotter/demux/proxy/snapshotter.go +++ b/snapshotter/demux/proxy/snapshotter.go @@ -36,6 +36,14 @@ type Dialer struct { Timeout time.Duration } +// RemoteSnapshotterConfig is the configuration needed to create a new remote snapshotter. +type RemoteSnapshotterConfig struct { + VSockPath string + RemoteSnapshotterPort uint32 + MetricsPort uint32 + MetricsLabels map[string]string +} + // RemoteSnapshotter embeds a snapshots.Snapshotter and its metrics proxy. type RemoteSnapshotter struct { snapshots.Snapshotter diff --git a/snapshotter/demux/snapshotter.go b/snapshotter/demux/snapshotter.go index 96c33902a..bb126a150 100644 --- a/snapshotter/demux/snapshotter.go +++ b/snapshotter/demux/snapshotter.go @@ -15,7 +15,12 @@ package demux import ( "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" @@ -25,9 +30,23 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/vm" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/cache" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" + mountutil "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/mount" ) +const ( + // VSockPathLabel is a snapshot label that contains the VM's vsock path. + VSockPathLabel = "containerd.io/snapshot/remote/demux.vsock.address" + // RemoteSnapshotterPortLabel is a snapshot label that contains the vsock port the remote snapshotter is listening on inside the VM. + RemoteSnapshotterPortLabel = "containerd.io/snapshot/remote/demux.vsock.port" + // MetricsPortLabel is a snapshot label that contains the vsock port the remote snapshotter's metrics server is listening on inside the VM. + MetricsPortLabel = "containerd.io/snapshot/remote/demux.metrics.port" + // MetricsLabelsLabel is a snapshot label that contains the set of labels to attach to metrics when scraping. + MetricsLabelsLabel = "containerd.io/snapshot/remote/demux.metrics.labels" +) + +var errNoVsockPath = fmt.Errorf("the snapshot is missing the %s label", VSockPathLabel) + // Snapshotter routes snapshotter requests to their destined // remote snapshotter via their snapshotter namespace. // @@ -109,7 +128,17 @@ func (s *Snapshotter) Prepare(ctx context.Context, key string, parent string, op snapshotter, err := s.getSnapshotterFromCache(ctx, contextLogger) if err != nil { - return []mount.Mount{}, err + if !errors.Is(err, errdefs.ErrNotFound) { + return nil, err + } + snapshotterConfig, err := extractRemoteSnapshotterConfig(opts...) + if err != nil { + return nil, err + } + snapshotter, err = s.putSnapshotterIntoCache(ctx, snapshotterConfig, contextLogger) + if err != nil { + return nil, err + } } mounts, err := snapshotter.Prepare(ctx, key, parent, opts...) @@ -221,6 +250,19 @@ func (s *Snapshotter) getSnapshotterFromCache(ctx context.Context, log *logrus.E return snapshotter, nil } +func (s *Snapshotter) putSnapshotterIntoCache(ctx context.Context, snapshotterConfig proxy.RemoteSnapshotterConfig, log *logrus.Entry) (*proxy.RemoteSnapshotter, error) { + namespace, err := getNamespaceFromContext(ctx, log) + if err != nil { + return nil, err + } + snapshotter, err := s.cache.Put(ctx, namespace, snapshotterConfig) + if err != nil { + log.WithField("namespace", namespace).WithError(err).Error(snapshotterNotFoundErrorString) + return nil, err + } + return snapshotter, nil +} + const missingNamespaceErrorString = "Function called without namespaced context" func getNamespaceFromContext(ctx context.Context, log *logrus.Entry) (string, error) { @@ -231,3 +273,56 @@ func getNamespaceFromContext(ctx context.Context, log *logrus.Entry) (string, er } return namespace, nil } + +func extractRemoteSnapshotterConfig(opts ...snapshots.Opt) (proxy.RemoteSnapshotterConfig, error) { + var base snapshots.Info + for _, opt := range opts { + if err := opt(&base); err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + // extract vsock path + path, ok := base.Labels[VSockPathLabel] + if !ok { + return proxy.RemoteSnapshotterConfig{}, errNoVsockPath + } + + // extract remote snapshotter port + snapshotterPortStr, ok := base.Labels[RemoteSnapshotterPortLabel] + if !ok { + return proxy.RemoteSnapshotterConfig{}, fmt.Errorf("the snapshot is missing the %s label", RemoteSnapshotterPortLabel) + } + snapshotterPort, err := strconv.Atoi(snapshotterPortStr) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + + // extract metrics port + metricsPortStr, ok := base.Labels[MetricsPortLabel] + var metricsPort int + if ok { + metricsPort, err = strconv.Atoi(metricsPortStr) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + // extract metrics labels + var metricsLabels map[string]string + metricsLabelsSerialied := base.Labels[MetricsLabelsLabel] + if metricsLabelsSerialied != "" { + err := json.Unmarshal([]byte(metricsLabelsSerialied), &metricsLabels) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + return proxy.RemoteSnapshotterConfig{ + VSockPath: path, + RemoteSnapshotterPort: uint32(snapshotterPort), + MetricsPort: uint32(metricsPort), + MetricsLabels: metricsLabels, + }, nil + +} diff --git a/snapshotter/demux/snapshotter_test.go b/snapshotter/demux/snapshotter_test.go index 06a1660f7..3fa76276a 100644 --- a/snapshotter/demux/snapshotter_test.go +++ b/snapshotter/demux/snapshotter_test.go @@ -27,29 +27,18 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" ) -func fetchOkSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func fetchOkSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { var snapshotter internal.SuccessfulSnapshotter = internal.SuccessfulSnapshotter{} return &proxy.RemoteSnapshotter{Snapshotter: &snapshotter}, nil } -func fetchSnapshotterNotFound(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func fetchSnapshotterNotFound(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return nil, errors.New("mock snapshotter not found") } func createSnapshotterCacheWithSuccessfulSnapshotter(namespace string) *cache.RemoteSnapshotterCache { cache := cache.NewRemoteSnapshotterCache(fetchOkSnapshotter) - cache.Get(context.Background(), namespace) - return cache -} - -func fetchFailingSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { - var snapshotter internal.FailingSnapshotter = internal.FailingSnapshotter{} - return &proxy.RemoteSnapshotter{Snapshotter: &snapshotter}, nil -} - -func createSnapshotterCacheWithFailingSnapshotter(namespace string) *cache.RemoteSnapshotterCache { - cache := cache.NewRemoteSnapshotterCache(fetchFailingSnapshotter) - cache.Get(context.Background(), namespace) + cache.Put(context.Background(), namespace, proxy.RemoteSnapshotterConfig{}) return cache } @@ -144,45 +133,6 @@ func TestReturnErrorWhenSnapshotterNotFound(t *testing.T) { } } -func TestReturnErrorAfterProxyFunctionFailure(t *testing.T) { - t.Parallel() - - const namespace = "testing" - cache := createSnapshotterCacheWithFailingSnapshotter(namespace) - ctx := namespaces.WithNamespace(context.Background(), namespace) - ctx = logtest.WithT(ctx, t) - - uut := NewSnapshotter(cache) - - tests := []struct { - name string - run func() error - }{ - {"Stat", func() error { _, err := uut.Stat(ctx, "layerKey"); return err }}, - {"Update", func() error { _, err := uut.Update(ctx, snapshots.Info{}); return err }}, - {"Usage", func() error { _, err := uut.Usage(ctx, "layerKey"); return err }}, - {"Mounts", func() error { _, err := uut.Mounts(ctx, "layerKey"); return err }}, - {"Prepare", func() error { _, err := uut.Prepare(ctx, "layerKey", ""); return err }}, - {"View", func() error { _, err := uut.View(ctx, "layerKey", ""); return err }}, - {"Commit", func() error { return uut.Commit(ctx, "layer1", "layerKey") }}, - {"Remove", func() error { return uut.Remove(ctx, "layerKey") }}, - {"Walk", func() error { - var callback = func(c context.Context, i snapshots.Info) error { return nil } - return uut.Walk(ctx, callback) - }}, - {"Cleanup", func() error { return uut.(snapshots.Cleaner).Cleanup(ctx) }}, - {"Close", func() error { return uut.Close() }}, - } - - for _, test := range tests { - t.Run(test.name+"ProxyFailure", func(t *testing.T) { - if err := test.run(); err == nil { - t.Fatalf("%s call did not return error", test.name) - } - }) - } -} - func TestNoErrorIsReturnedOnSuccessfulProxyExecution(t *testing.T) { t.Parallel() diff --git a/snapshotter/internal/http_address_resolver.go b/snapshotter/internal/http_address_resolver.go deleted file mode 100644 index 5c5e64a9a..000000000 --- a/snapshotter/internal/http_address_resolver.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - - "github.com/containerd/containerd/namespaces" - "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" - - "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" - "github.com/firecracker-microvm/firecracker-containerd/proto" - proxyaddress "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy/address" -) - -var ( - port int - remotePort int - metricsRemotePort int - containerdSockPath string - logger *logrus.Logger -) - -func init() { - flag.IntVar(&port, "port", 10001, "service port for address resolver") - flag.StringVar(&containerdSockPath, "containerdSocket", "/run/firecracker-containerd/containerd.sock", "filepath to the containerd socket") - flag.IntVar(&remotePort, "remotePort", 10000, "the remote port on which the remote snapshotter is listening") - flag.IntVar(&metricsRemotePort, "metricsRemotePort", 10002, "the remote port on which the remote snapshotter metrics server is listening") - logger = logrus.New() -} - -// Simple example of an HTTP service to resolve snapshotter namespace -// to a forwarding address for the demultiplexing snapshotter. -// -// Example: -// curl -X GET "http://localhost:10001/address?namespace=ns-1" -// -// Response: -// { -// "network": "unix", -// "address": "/var/lib/firecracker-containerd/shim-base/default#cbfad871-0862-4dd6-ae7a-52e9b1c16ede/firecracker.vsock" -// } -func main() { - ctx, cancel := context.WithCancel(context.Background()) - - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - - <-c - cancel() - }() - - if !flag.Parsed() { - flag.Parse() - } - - http.HandleFunc("/address", queryAddress) - httpServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - } - - logger.Info(fmt.Sprintf("http resolver serving at port %d", port)) - g, gCtx := errgroup.WithContext(ctx) - g.Go(func() error { - return httpServer.ListenAndServe() - }) - g.Go(func() error { - <-gCtx.Done() - return httpServer.Shutdown(context.Background()) - }) - - if err := g.Wait(); err != http.ErrServerClosed { - logger.WithError(err).Error() - os.Exit(1) - } - - logger.Info("http: server closed") -} - -func queryAddress(writ http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - http.Error(writ, fmt.Sprintf("%s method not allowed", req.Method), http.StatusForbidden) - return - } - - writ.Header().Set("Content-Type", "application/json") - - keys, ok := req.URL.Query()["namespace"] - if !ok { - http.Error(writ, "Missing 'namespace' query", http.StatusBadRequest) - return - } - - sock := containerdSockPath + ".ttrpc" - fcClient, err := client.New(sock) - if err != nil { - logger.WithError(err).Error("could not create firecracker client") - http.Error(writ, fmt.Sprintf("failed to connect %q", sock), http.StatusInternalServerError) - return - } - defer fcClient.Close() - - namespace := keys[0] - ctx := namespaces.WithNamespace(req.Context(), namespace) - vmInfo, err := fcClient.GetVMInfo(ctx, &proto.GetVMInfoRequest{VMID: namespace}) - if err != nil { - logger.WithField("VMID", namespace).WithError(err).Error("unable to retrieve VM Info") - http.Error(writ, fmt.Sprintf("failed to get VM %q", namespace), http.StatusNotFound) - return - } - - writ.WriteHeader(http.StatusOK) - - result := proxyaddress.Response{ - Network: "unix", - Address: vmInfo.VSockPath, - SnapshotterPort: strconv.Itoa(remotePort), - MetricsPort: strconv.Itoa(metricsRemotePort), - Labels: map[string]string{ - "VMID": namespace, - }, - } - serialized, err := json.Marshal(&result) - if err != nil { - http.Error(writ, fmt.Sprintf("failed to marshal %+v", result), http.StatusInternalServerError) - return - } - writ.Write(serialized) -} diff --git a/snapshotter/metrics_integ_test.go b/snapshotter/metrics_integ_test.go index 63bfe0ad9..b42e05ed9 100644 --- a/snapshotter/metrics_integ_test.go +++ b/snapshotter/metrics_integ_test.go @@ -27,6 +27,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -91,7 +92,7 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie if err != nil { return fmt.Errorf("Unable to create client to containerd service at %s, is containerd running?: %v", integtest.ContainerdSockPath, err) } - if _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, RootDrive: &proto.FirecrackerRootDrive{ HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img", @@ -110,7 +111,8 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie MemSizeMib: 512, }, ContainerCount: 1, - }); err != nil { + }) + if err != nil { return fmt.Errorf("Failed to create microVM[%s]: %v", vmID, err) } @@ -123,7 +125,11 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie image, err := client.Pull(ctx, al2stargz, containerd.WithPullUnpack, - containerd.WithPullSnapshotter(snapshotterName), + containerd.WithPullSnapshotter(snapshotterName, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + demux.WithProxyMetricsPort(10002), + ), containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*1024*1024)), ) if err != nil { @@ -162,7 +168,7 @@ func verifyMetricsResponse(t *testing.T, numberOfVms int) { uniqueTargets[mt.Targets[0]] = struct{}{} metricsResponse, err := http.Get("http://" + mt.Targets[0] + "/metrics") require.NoError(t, err, "Failed to get metrics proxy") - require.Equal(t, metricsResponse.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, metricsResponse.StatusCode) defer metricsResponse.Body.Close() mBytes, err := io.ReadAll(metricsResponse.Body) diff --git a/snapshotter/service_integ_test.go b/snapshotter/service_integ_test.go index 7882352bb..0841407d4 100644 --- a/snapshotter/service_integ_test.go +++ b/snapshotter/service_integ_test.go @@ -25,6 +25,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -93,7 +94,7 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string) // https://github.com/firecracker-microvm/firecracker/blob/v1.1.0/docs/prod-host-setup.md kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs + " 8250.nr_uarts=0 quiet loglevel=1" - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, KernelArgs: kernelArgs, RootDrive: &proto.FirecrackerRootDrive{ @@ -130,7 +131,10 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string) image, err := client.Pull(ctx, al2stargz, containerd.WithPullUnpack, - containerd.WithPullSnapshotter(snapshotterName), + containerd.WithPullSnapshotter(snapshotterName, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib)), ) if err != nil { diff --git a/snapshotter/volume_integ_test.go b/snapshotter/volume_integ_test.go index 006a139aa..1e895b2fd 100644 --- a/snapshotter/volume_integ_test.go +++ b/snapshotter/volume_integ_test.go @@ -24,6 +24,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/firecracker-microvm/firecracker-containerd/volume" "github.com/stretchr/testify/assert" @@ -56,24 +57,12 @@ func TestGuestVolumeFrom_Isolated(t *testing.T) { err = vs.AddFrom(ctx, localImage) require.NoError(t, err) - // Add a stargz image. - // The volume directories must be specified since the host's containerd doesn't know about the image. - remoteImage := volume.FromGuestImage( - client, vmID, al2stargz, "al2-snapshot", []string{"/etc/yum"}, - volume.WithSnapshotter("demux"), - volume.WithPullOptions(containerd.WithImageHandlerWrapper( - source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib), - )), - ) - err = vs.AddFrom(ctx, remoteImage) - require.NoError(t, err) - // PrepareDriveMount only copies images that are available before starting the VM. // In this case, only postgres. mount, err := vs.PrepareDriveMount(ctx, 10*mib) require.NoError(t, err) - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, RootDrive: &proto.FirecrackerRootDrive{ HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img", @@ -97,6 +86,22 @@ func TestGuestVolumeFrom_Isolated(t *testing.T) { require.NoErrorf(t, err, "Failed to create microVM[%s]", vmID) defer fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: vmID}) + // Add a stargz image. + // The volume directories must be specified since the host's containerd doesn't know about the image. + remoteImage := volume.FromGuestImage( + client, vmID, al2stargz, "al2-snapshot", []string{"/etc/yum"}, + volume.WithSnapshotter("demux"), + volume.WithPullOptions(containerd.WithImageHandlerWrapper( + source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib), + )), + volume.WithSnapshotOptions( + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), + ) + err = vs.AddFrom(ctx, remoteImage) + require.NoError(t, err) + _, err = fcClient.SetVMMetadata(ctx, &proto.SetVMMetadataRequest{ VMID: vmID, Metadata: fmt.Sprintf(dockerMetadataTemplate, "ghcr.io", noAuth, noAuth), diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh index 77db62128..71fcabfbe 100755 --- a/tools/docker/entrypoint.sh +++ b/tools/docker/entrypoint.sh @@ -46,10 +46,6 @@ cat > /etc/demux-snapshotter/config.toml <> ${FICD_CONTAINERD_OUTFILE} & -/usr/local/bin/http-address-resolver &>> ${FICD_LOG_DIR}/http-address-resolver.out & /usr/local/bin/demux-snapshotter &>> ${FICD_LOG_DIR}/demux-snapshotter.out & exec /bin/bash -c "$@" diff --git a/volume/.gitignore b/volume/.gitignore new file mode 100644 index 000000000..3084a1dc7 --- /dev/null +++ b/volume/.gitignore @@ -0,0 +1,2 @@ +volume-init + diff --git a/volume/guest_image.go b/volume/guest_image.go index 43a6b42d1..e224c3cc5 100644 --- a/volume/guest_image.go +++ b/volume/guest_image.go @@ -90,7 +90,7 @@ func (p *GuestVolumeImageProvider) Name() string { func (p *GuestVolumeImageProvider) pull(ctx context.Context) error { remoteOpts := []containerd.RemoteOpt{ containerd.WithPullUnpack, - containerd.WithPullSnapshotter(p.config.snapshotter), + containerd.WithPullSnapshotter(p.config.snapshotter, p.config.snapshotOpts...), } remoteOpts = append(remoteOpts, p.config.pullOpts...) image, err := p.client.Pull(ctx, p.image, remoteOpts...) diff --git a/volume/image.go b/volume/image.go index 41cb80b77..7cf2eae68 100644 --- a/volume/image.go +++ b/volume/image.go @@ -32,8 +32,9 @@ import ( ) type imageProviderConfig struct { - snapshotter string - pullOpts []containerd.RemoteOpt + snapshotter string + pullOpts []containerd.RemoteOpt + snapshotOpts []snapshots.Opt } type imageVolumeProvider struct { @@ -61,6 +62,13 @@ func WithPullOptions(opts ...containerd.RemoteOpt) ImageOpt { } } +// WithSnapshotOptions sets the snapshotter's snapshot options. +func WithSnapshotOptions(opts ...snapshots.Opt) ImageOpt { + return func(c *imageProviderConfig) { + c.snapshotOpts = opts + } +} + func getV1ImageConfig(ctx context.Context, image containerd.Image) (*v1.Image, error) { var result v1.Image