From 085fbc5b84a9b806884d30f53ab97e4b9ffb5720 Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Fri, 13 Dec 2024 13:48:02 -0500 Subject: [PATCH 1/7] feat: CNS checks apiserver in healthz --- cns/healthserver/healthz.go | 54 +++++++++++++++++++++++++++++++++++++ cns/service/main.go | 3 ++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 cns/healthserver/healthz.go diff --git a/cns/healthserver/healthz.go b/cns/healthserver/healthz.go new file mode 100644 index 0000000000..f15d2b2a27 --- /dev/null +++ b/cns/healthserver/healthz.go @@ -0,0 +1,54 @@ +package healthserver + +import ( + "net/http" + + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" +) + +var schema = runtime.NewScheme() + +func init() { + utilruntime.Must(v1alpha.AddToScheme(schema)) +} + +func NewHealthzHandlerWithChecks() http.Handler { + cfg, err := ctrl.GetConfig() + if err != nil { + panic(err) + } + cli, err := client.New(cfg, client.Options{ + Scheme: schema, + }) + if err != nil { + panic(err) + } + + checks := map[string]healthz.Checker{ + "nnc": func(req *http.Request) error { + ctx := req.Context() + // we just care that we're allowed to List NNCs so set limit to 1 to minimize + // additional load on apiserver + if err := cli.List(ctx, &v1alpha.NodeNetworkConfigList{}, &client.ListOptions{ + Namespace: metav1.NamespaceSystem, + Limit: int64(1), + }); err != nil { + return errors.Wrap(err, "failed to list NodeNetworkConfig") + } + return nil + }, + } + + // strip prefix so that it runs through all checks registered on the handler. + // otherwise it will look for a check named "healthz" and return a 404 if not there. + return http.StripPrefix("/healthz", &healthz.Handler{ + Checks: checks, + }) +} diff --git a/cns/service/main.go b/cns/service/main.go index 36f24dffa7..5a7a1743e7 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -642,7 +642,8 @@ func main() { return nil }), } - go healthserver.Start(z, cnsconfig.MetricsBindAddress, &healthz.Handler{}, readyChecker) + healthzHandler := healthserver.NewHealthzHandlerWithChecks() + go healthserver.Start(z, cnsconfig.MetricsBindAddress, healthzHandler, readyChecker) nmaConfig, err := nmagent.NewConfig(cnsconfig.WireserverIP) if err != nil { From 9c1afe3767978d782680064300858fb3bf3155ad Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Mon, 16 Dec 2024 12:00:05 -0500 Subject: [PATCH 2/7] chore: only check NNCs if `ChannelMode` is `CRD` not every instance of CNS will need (or can) check NNCs. The `CRD` channel mode is used by AKS to indicate that CNS will be reading/watching NNCs. `AzureHost` is a newer mode that's used in nodesubnet where NNCs aren't used and therefore CNS has no reason to have its health depend on NNC access. --- cns/healthserver/healthz.go | 22 +++++++++++++--------- cns/service/main.go | 7 ++++--- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/cns/healthserver/healthz.go b/cns/healthserver/healthz.go index f15d2b2a27..1094b689df 100644 --- a/cns/healthserver/healthz.go +++ b/cns/healthserver/healthz.go @@ -1,8 +1,11 @@ package healthserver import ( + "fmt" "net/http" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,26 +16,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" ) -var schema = runtime.NewScheme() +var scheme = runtime.NewScheme() func init() { - utilruntime.Must(v1alpha.AddToScheme(schema)) + utilruntime.Must(v1alpha.AddToScheme(scheme)) } -func NewHealthzHandlerWithChecks() http.Handler { +func NewHealthzHandlerWithChecks(cnsConfig *configuration.CNSConfig) http.Handler { cfg, err := ctrl.GetConfig() if err != nil { - panic(err) + panic(fmt.Errorf("unable to get config: %w", err)) } cli, err := client.New(cfg, client.Options{ - Scheme: schema, + Scheme: scheme, }) if err != nil { - panic(err) + panic(fmt.Errorf("unable to create client: %w", err)) } - checks := map[string]healthz.Checker{ - "nnc": func(req *http.Request) error { + checks := make(map[string]healthz.Checker) + if cnsConfig.ChannelMode == cns.CRD { + checks["nnc"] = func(req *http.Request) error { ctx := req.Context() // we just care that we're allowed to List NNCs so set limit to 1 to minimize // additional load on apiserver @@ -43,7 +47,7 @@ func NewHealthzHandlerWithChecks() http.Handler { return errors.Wrap(err, "failed to list NodeNetworkConfig") } return nil - }, + } } // strip prefix so that it runs through all checks registered on the handler. diff --git a/cns/service/main.go b/cns/service/main.go index 5a7a1743e7..73d33e39d6 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -642,7 +642,8 @@ func main() { return nil }), } - healthzHandler := healthserver.NewHealthzHandlerWithChecks() + + healthzHandler := healthserver.NewHealthzHandlerWithChecks(cnsconfig) go healthserver.Start(z, cnsconfig.MetricsBindAddress, healthzHandler, readyChecker) nmaConfig, err := nmagent.NewConfig(cnsconfig.WireserverIP) @@ -982,7 +983,7 @@ func main() { // Start fs watcher here z.Info("AsyncPodDelete is enabled") logger.Printf("AsyncPodDelete is enabled") - cnsclient, err := cnsclient.New("", cnsReqTimeout) //nolint + cnsclient, err := cnsclient.New("", cnsReqTimeout) // nolint if err != nil { z.Error("failed to create cnsclient", zap.Error(err)) } @@ -1483,7 +1484,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // wait for the Reconciler to run once on a NNC that was made for this Node. // the nncReadyCtx has a timeout of 15 minutes, after which we will consider // this false and the NNC Reconciler stuck/failed, log and retry. - nncReadyCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) //nolint // it will time out and not leak + nncReadyCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) // nolint // it will time out and not leak if started, err := nncReconciler.Started(nncReadyCtx); !started { logger.Errorf("NNC reconciler has not started, does the NNC exist? err: %v", err) nncReconcilerStartFailures.Inc() From 375fb6a20f4a1cb9ec080a84bb217e8644397007 Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Mon, 16 Dec 2024 12:00:44 -0500 Subject: [PATCH 3/7] test: add unit tests --- cns/healthserver/healthz_test.go | 280 +++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 cns/healthserver/healthz_test.go diff --git a/cns/healthserver/healthz_test.go b/cns/healthserver/healthz_test.go new file mode 100644 index 0000000000..50f47c7af3 --- /dev/null +++ b/cns/healthserver/healthz_test.go @@ -0,0 +1,280 @@ +package healthserver + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/stretchr/testify/require" +) + +const nncCRD = `{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "acn.azure.com/v1alpha", + "resources": [ + { + "name": "nodenetworkconfigs", + "singularName": "nodenetworkconfig", + "namespaced": true, + "kind": "NodeNetworkConfig", + "verbs": [ + "delete", + "deletecollection", + "get", + "list", + "patch", + "create", + "update", + "watch" + ], + "shortNames": [ + "nnc" + ], + "storageVersionHash": "aGVsbG93cmxk" + }, + { + "name": "nodenetworkconfigs/status", + "singularName": "", + "namespaced": true, + "kind": "NodeNetworkConfig", + "verbs": [ + "get", + "patch", + "update" + ] + } + ] +}` + +const nncResult = `{ + "apiVersion": "acn.azure.com/v1alpha", + "items": [ + { + "apiVersion": "acn.azure.com/v1alpha", + "kind": "NodeNetworkConfig", + "metadata": { + "creationTimestamp": "2024-12-04T20:42:17Z", + "finalizers": [ + "finalizers.acn.azure.com/dnc-operations" + ], + "generation": 1, + "labels": { + "kubernetes.azure.com/podnetwork-delegationguid": "", + "kubernetes.azure.com/podnetwork-subnet": "", + "kubernetes.azure.com/podnetwork-type": "overlay", + "managed": "true", + "owner": "aks-nodepool1-1234567-vmss000000" + }, + "managedFields": [ + { + "apiVersion": "acn.azure.com/v1alpha", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:finalizers": { + ".": {}, + "v:\"finalizers.acn.azure.com/dnc-operations\"": {} + }, + "f:labels": { + ".": {}, + "f:kubernetes.azure.com/podnetwork-delegationguid": {}, + "f:kubernetes.azure.com/podnetwork-subnet": {}, + "f:kubernetes.azure.com/podnetwork-type": {}, + "f:managed": {}, + "f:owner": {} + }, + "f:ownerReferences": { + ".": {}, + "k:{\"uid\":\"f5117020-bbc5-11ef-8433-1b9e59caeb1d\"}": {} + } + }, + "f:spec": { + ".": {}, + "f:requestedIPCount": {} + } + }, + "manager": "dnc-rc", + "operation": "Update", + "time": "2024-12-04T20:42:17Z" + }, + { + "apiVersion": "acn.azure.com/v1alpha", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:status": { + ".": {}, + "f:assignedIPCount": {}, + "f:networkContainers": {} + } + }, + "manager": "dnc-rc", + "operation": "Update", + "subresource": "status", + "time": "2024-12-04T20:42:18Z" + } + ], + "name": "aks-nodepool1-1234567-vmss000000", + "namespace": "kube-system", + "ownerReferences": [ + { + "apiVersion": "v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Node", + "name": "aks-nodepool1-1234567-vmss000000", + "uid": "02df1fcc-bbc6-11ef-a76a-4b1af8d399a2" + } + ], + "resourceVersion": "123456789", + "uid": "0dc75e5e-bbc6-11ef-878f-ab45432262d6" + }, + "spec": { + "requestedIPCount": 0 + }, + "status": { + "assignedIPCount": 256, + "networkContainers": [ + { + "assignmentMode": "static", + "id": "13f630c0-bbc6-11ef-b3b7-bb8e46de5973", + "nodeIP": "10.224.0.4", + "primaryIP": "10.244.2.0/24", + "subnetAddressSpace": "10.244.0.0/16", + "subnetName": "routingdomain_1f7eb6ba-bbc6-11ef-8c54-7b2c1e3cbbe4_overlaysubnet", + "type": "overlay", + "version": 0 + } + ] + } + } + ], + "kind": "NodeNetworkConfigList", + "metadata": { + "continue": "", + "resourceVersion": "9876543210" + } +}` + +func TestNewHealthzHandlerWithChecks(t *testing.T) { + tests := []struct { + name string + cnsConfig *configuration.CNSConfig + apiStatusCode int + expectedHealthy bool + }{ + { + name: "list NNC gives 200 should indicate healthy", + cnsConfig: &configuration.CNSConfig{ + ChannelMode: "CRD", + }, + apiStatusCode: http.StatusOK, + expectedHealthy: true, + }, + { + name: "unauthorized (401) from apiserver should be unhealthy", + cnsConfig: &configuration.CNSConfig{ + ChannelMode: "CRD", + }, + apiStatusCode: http.StatusUnauthorized, + expectedHealthy: false, + }, + { + name: "channel nodesubnet should not call apiserver so it doesn't matter if the status code is a 401", + cnsConfig: &configuration.CNSConfig{ + ChannelMode: "AzureHost", + }, + apiStatusCode: http.StatusUnauthorized, + expectedHealthy: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configureLocalAPIServer(t, tt.apiStatusCode) + + responseRecorder := httptest.NewRecorder() + healthHandler := NewHealthzHandlerWithChecks(tt.cnsConfig) + healthHandler.ServeHTTP(responseRecorder, httptest.NewRequest("GET", "/healthz", nil)) + + require.Equal(t, tt.expectedHealthy, responseRecorder.Code == http.StatusOK) + }) + } +} + +func configureLocalAPIServer(t *testing.T, expectedNNCStatusCode int) { + // setup apiserver + server := setupMockAPIServer(expectedNNCStatusCode) + + // write kubeConfig for test server + kubeConfigFile, err := writeTmpKubeConfig(server.URL) + require.NoError(t, err) + + // set env var to kubeconfig + os.Setenv("KUBECONFIG", kubeConfigFile) + + t.Cleanup(func() { + server.Close() + os.Remove(kubeConfigFile) + os.Unsetenv("KUBECONFIG") + }) +} + +func writeTmpKubeConfig(host string) (string, error) { + tempKubeConfig := ` +apiVersion: v1 +clusters: +- cluster: + server: ` + host + ` + name: test-cluster +contexts: +- context: + cluster: test-cluster + user: test-user + name: test-context +current-context: test-context +kind: Config +preferences: {} +users: +- name: test-user + user: + token: test-token +` + kubeConfigFile, err := os.CreateTemp("", "kubeconfig") + if err != nil { + return "", fmt.Errorf("failed to create temp kubeconfig file: %w", err) + } + + _, err = kubeConfigFile.Write([]byte(tempKubeConfig)) + if err != nil { + return "", fmt.Errorf("failed to write kubeconfig to temp file: %w", err) + } + kubeConfigFile.Close() + return kubeConfigFile.Name(), nil +} + +func setupMockAPIServer(code int) *httptest.Server { + // Start a mock HTTP server + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Handle requests based on the path + switch r.URL.Path { + case "/apis/acn.azure.com/v1alpha": + w.Write([]byte(nncCRD)) + case "/apis/acn.azure.com/v1alpha/namespaces/kube-system/nodenetworkconfigs": + if code == http.StatusOK { + w.Header().Set("Cache-Control", "no-cache, private") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(nncResult)) + } else { + w.WriteHeader(code) + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + return mockServer +} From e099c53f4f1261e7e9f1f84186ed09100eb7fea2 Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Mon, 16 Dec 2024 12:07:09 -0500 Subject: [PATCH 4/7] refactor: return error from NewHealthzHandlerWithChecks instead of panicking --- cns/healthserver/healthz.go | 13 ++++++++----- cns/healthserver/healthz_test.go | 4 +++- cns/service/main.go | 6 +++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cns/healthserver/healthz.go b/cns/healthserver/healthz.go index 1094b689df..1ca93ac913 100644 --- a/cns/healthserver/healthz.go +++ b/cns/healthserver/healthz.go @@ -1,7 +1,6 @@ package healthserver import ( - "fmt" "net/http" "github.com/Azure/azure-container-networking/cns" @@ -22,16 +21,20 @@ func init() { utilruntime.Must(v1alpha.AddToScheme(scheme)) } -func NewHealthzHandlerWithChecks(cnsConfig *configuration.CNSConfig) http.Handler { +// NewHealthzHandlerWithChecks will return a [http.Handler] for CNS's /healthz endpoint. +// Depending on what we expect CNS to be able to read (based on the [configuration.CNSConfig]) +// then the checks registered to the handler will test for those expectations. For example, in +// ChannelMode: CRD, the health check will ensure that CNS is able to list NNCs successfully. +func NewHealthzHandlerWithChecks(cnsConfig *configuration.CNSConfig) (http.Handler, error) { cfg, err := ctrl.GetConfig() if err != nil { - panic(fmt.Errorf("unable to get config: %w", err)) + return nil, errors.Wrap(err, "unable to get a kubeconfig") } cli, err := client.New(cfg, client.Options{ Scheme: scheme, }) if err != nil { - panic(fmt.Errorf("unable to create client: %w", err)) + return nil, errors.Wrap(err, "unable to create a client") } checks := make(map[string]healthz.Checker) @@ -54,5 +57,5 @@ func NewHealthzHandlerWithChecks(cnsConfig *configuration.CNSConfig) http.Handle // otherwise it will look for a check named "healthz" and return a 404 if not there. return http.StripPrefix("/healthz", &healthz.Handler{ Checks: checks, - }) + }), nil } diff --git a/cns/healthserver/healthz_test.go b/cns/healthserver/healthz_test.go index 50f47c7af3..91c69d187a 100644 --- a/cns/healthserver/healthz_test.go +++ b/cns/healthserver/healthz_test.go @@ -197,7 +197,9 @@ func TestNewHealthzHandlerWithChecks(t *testing.T) { configureLocalAPIServer(t, tt.apiStatusCode) responseRecorder := httptest.NewRecorder() - healthHandler := NewHealthzHandlerWithChecks(tt.cnsConfig) + healthHandler, err := NewHealthzHandlerWithChecks(tt.cnsConfig) + require.NoError(t, err) + healthHandler.ServeHTTP(responseRecorder, httptest.NewRequest("GET", "/healthz", nil)) require.Equal(t, tt.expectedHealthy, responseRecorder.Code == http.StatusOK) diff --git a/cns/service/main.go b/cns/service/main.go index 73d33e39d6..b5ef339f95 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -643,7 +643,11 @@ func main() { }), } - healthzHandler := healthserver.NewHealthzHandlerWithChecks(cnsconfig) + healthzHandler, err := healthserver.NewHealthzHandlerWithChecks(cnsconfig) + if err != nil { + logger.Errorf("unable to initialize a healthz handler: %v", err) + return + } go healthserver.Start(z, cnsconfig.MetricsBindAddress, healthzHandler, readyChecker) nmaConfig, err := nmagent.NewConfig(cnsconfig.WireserverIP) From af975654e93e166859e1a50ec65362d92c4930bd Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Mon, 16 Dec 2024 12:24:14 -0500 Subject: [PATCH 5/7] chore: address lint errors --- cns/healthserver/healthz_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cns/healthserver/healthz_test.go b/cns/healthserver/healthz_test.go index 91c69d187a..58d5de1ae8 100644 --- a/cns/healthserver/healthz_test.go +++ b/cns/healthserver/healthz_test.go @@ -200,7 +200,7 @@ func TestNewHealthzHandlerWithChecks(t *testing.T) { healthHandler, err := NewHealthzHandlerWithChecks(tt.cnsConfig) require.NoError(t, err) - healthHandler.ServeHTTP(responseRecorder, httptest.NewRequest("GET", "/healthz", nil)) + healthHandler.ServeHTTP(responseRecorder, httptest.NewRequest("GET", "/healthz", http.NoBody)) require.Equal(t, tt.expectedHealthy, responseRecorder.Code == http.StatusOK) }) @@ -250,7 +250,7 @@ users: return "", fmt.Errorf("failed to create temp kubeconfig file: %w", err) } - _, err = kubeConfigFile.Write([]byte(tempKubeConfig)) + _, err = kubeConfigFile.WriteString(tempKubeConfig) if err != nil { return "", fmt.Errorf("failed to write kubeconfig to temp file: %w", err) } @@ -264,12 +264,18 @@ func setupMockAPIServer(code int) *httptest.Server { // Handle requests based on the path switch r.URL.Path { case "/apis/acn.azure.com/v1alpha": - w.Write([]byte(nncCRD)) + _, err := w.Write([]byte(nncCRD)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } case "/apis/acn.azure.com/v1alpha/namespaces/kube-system/nodenetworkconfigs": if code == http.StatusOK { w.Header().Set("Cache-Control", "no-cache, private") w.Header().Set("Content-Type", "application/json") - w.Write([]byte(nncResult)) + _, err := w.Write([]byte(nncResult)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } else { w.WriteHeader(code) } From 50e1057f0a58e08b4d67d68927236bd8229b814a Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Tue, 17 Dec 2024 08:26:55 -0500 Subject: [PATCH 6/7] refactor: only get kubeConfig when in CRD mode --- cns/healthserver/healthz.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cns/healthserver/healthz.go b/cns/healthserver/healthz.go index 1ca93ac913..45bc2f303c 100644 --- a/cns/healthserver/healthz.go +++ b/cns/healthserver/healthz.go @@ -26,19 +26,19 @@ func init() { // then the checks registered to the handler will test for those expectations. For example, in // ChannelMode: CRD, the health check will ensure that CNS is able to list NNCs successfully. func NewHealthzHandlerWithChecks(cnsConfig *configuration.CNSConfig) (http.Handler, error) { - cfg, err := ctrl.GetConfig() - if err != nil { - return nil, errors.Wrap(err, "unable to get a kubeconfig") - } - cli, err := client.New(cfg, client.Options{ - Scheme: scheme, - }) - if err != nil { - return nil, errors.Wrap(err, "unable to create a client") - } - checks := make(map[string]healthz.Checker) if cnsConfig.ChannelMode == cns.CRD { + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get kubeconfig") + } + cli, err := client.New(cfg, client.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to build client") + } + checks["nnc"] = func(req *http.Request) error { ctx := req.Context() // we just care that we're allowed to List NNCs so set limit to 1 to minimize From bcc574fc1611058f13502a218170e61c7e7efc45 Mon Sep 17 00:00:00 2001 From: Tyler Lloyd Date: Tue, 17 Dec 2024 09:24:45 -0500 Subject: [PATCH 7/7] chore: fix lint errors --- cns/healthserver/healthz_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cns/healthserver/healthz_test.go b/cns/healthserver/healthz_test.go index 58d5de1ae8..805509a1f4 100644 --- a/cns/healthserver/healthz_test.go +++ b/cns/healthserver/healthz_test.go @@ -267,6 +267,7 @@ func setupMockAPIServer(code int) *httptest.Server { _, err := w.Write([]byte(nncCRD)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + return } case "/apis/acn.azure.com/v1alpha/namespaces/kube-system/nodenetworkconfigs": if code == http.StatusOK { @@ -275,6 +276,7 @@ func setupMockAPIServer(code int) *httptest.Server { _, err := w.Write([]byte(nncResult)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + return } } else { w.WriteHeader(code)