Skip to content

Commit 4d9e5ec

Browse files
joebowbeerjoebowbeer
committed
Add tls-san, disable rootful tests
1 parent c4d7f5d commit 4d9e5ec

File tree

3 files changed

+134
-82
lines changed

3 files changed

+134
-82
lines changed

‎docs/modules/k3s.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,16 @@ npm install @testcontainers/k3s --save-dev
1111
## Examples
1212

1313
<!--codeinclude-->
14-
[Starting a K3S server](../../packages/modules/k3s/src/k3s-container.test.ts) inside_block:starting_k3s
14+
[Starting a K3s server:](../../packages/modules/k3s/src/k3s-container.test.ts) inside_block:starting_k3s
1515
<!--/codeinclude-->
1616

1717
<!--codeinclude-->
18-
[Connecting to the server](../../packages/modules/k3s/src/k3s-container.test.ts) inside_block:connecting_with_client
18+
[Connecting to the server using the Kubernetes JavaScript client:](../../packages/modules/k3s/src/k3s-container.test.ts) inside_block:connecting_with_client
1919
<!--/codeinclude-->
20+
21+
## Known limitations
22+
23+
!!! warning
24+
* K3sContainer runs as a privileged container and needs to be able to spawn its own containers. For these reasons,
25+
K3sContainer will not work in certain rootless Docker, Docker-in-Docker, or other environments where privileged
26+
containers are disallowed.

‎packages/modules/k3s/src/k3s-container.test.ts‎

Lines changed: 101 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,117 @@
11
import { K3sContainer } from "./k3s-container";
22
import * as k8s from "@kubernetes/client-node";
33
import { setTimeout } from "node:timers/promises";
4+
import { GenericContainer, Network, Wait } from "testcontainers";
45

56
describe("K3s", () => {
6-
jest.setTimeout(150_000);
7+
jest.setTimeout(120_000);
78

8-
it("should start and have listable node", async () => {
9-
// starting_k3s {
10-
const container = await new K3sContainer().start();
11-
// }
9+
it("should construct", () => {
10+
new K3sContainer("rancher/k3s:v1.31.2-k3s1");
11+
});
1212

13-
// connecting_with_client {
14-
// obtain a kubeconfig file which allows us to connect to k3s
15-
const kubeConfig = container.getKubeConfig();
13+
// K3sContainer runs as a privileged container
14+
if (!process.env["CI_ROOTLESS"]) {
15+
it("should start and have listable node", async () => {
16+
// starting_k3s {
17+
const container = await new K3sContainer("rancher/k3s:v1.31.2-k3s1").start();
18+
// }
1619

17-
const kc = new k8s.KubeConfig();
18-
kc.loadFromString(kubeConfig);
20+
// connecting_with_client {
21+
// obtain a kubeconfig file that allows us to connect to k3s
22+
const kubeConfig = container.getKubeConfig();
1923

20-
const client = kc.makeApiClient(k8s.CoreV1Api);
24+
const kc = new k8s.KubeConfig();
25+
kc.loadFromString(kubeConfig);
2126

22-
// interact with the running K3s server, e.g.:
23-
const nodeList = await client.listNode();
24-
// }
27+
const client = kc.makeApiClient(k8s.CoreV1Api);
2528

26-
expect(nodeList.items).toHaveLength(1);
29+
// interact with the running K3s server, e.g.:
30+
const nodeList = await client.listNode();
31+
// }
2732

28-
await container.stop();
29-
});
33+
expect(nodeList.items).toHaveLength(1);
3034

31-
it("should start a pod", async () => {
32-
const container = await new K3sContainer().start();
33-
const kc = new k8s.KubeConfig();
34-
kc.loadFromString(container.getKubeConfig());
35-
36-
const pod = {
37-
metadata: {
38-
name: "helloworld",
39-
},
40-
spec: {
41-
containers: [
42-
{
43-
name: "helloworld",
44-
image: "testcontainers/helloworld:1.1.0",
45-
ports: [
46-
{
47-
containerPort: 8080,
48-
},
49-
],
50-
readinessProbe: {
51-
tcpSocket: {
52-
port: 8080,
35+
await container.stop();
36+
});
37+
38+
it("should expose kubeconfig for a network alias", async () => {
39+
const network = await new Network().start();
40+
const container = await new K3sContainer("rancher/k3s:v1.31.2-k3s1")
41+
.withNetwork(network)
42+
.withNetworkAliases("k3s")
43+
.start();
44+
45+
// obtain a kubeconfig that allows us to connect on the custom network
46+
const kubeConfig = container.getAliasedKubeConfig("k3s");
47+
48+
const kubectlContainer = await new GenericContainer("rancher/kubectl:v1.31.2")
49+
.withNetwork(network)
50+
.withCopyContentToContainer([{ content: kubeConfig, target: "/home/kubectl/.kube/config" }])
51+
.withCommand(["get", "namespaces"])
52+
.withWaitStrategy(Wait.forOneShotStartup())
53+
.withStartupTimeout(30_000)
54+
.start();
55+
56+
const chunks = [];
57+
for await (const chunk of await kubectlContainer.logs()) {
58+
chunks.push(chunk);
59+
}
60+
expect(chunks).toEqual(expect.arrayContaining([expect.stringContaining("kube-system")]));
61+
62+
await kubectlContainer.stop();
63+
await container.stop();
64+
await network.stop();
65+
});
66+
67+
it("should start a pod", async () => {
68+
const container = await new K3sContainer("rancher/k3s:v1.31.2-k3s1").start();
69+
const kc = new k8s.KubeConfig();
70+
kc.loadFromString(container.getKubeConfig());
71+
72+
const pod = {
73+
metadata: {
74+
name: "helloworld",
75+
},
76+
spec: {
77+
containers: [
78+
{
79+
name: "helloworld",
80+
image: "testcontainers/helloworld:1.1.0",
81+
ports: [
82+
{
83+
containerPort: 8080,
84+
},
85+
],
86+
readinessProbe: {
87+
tcpSocket: {
88+
port: 8080,
89+
},
5390
},
5491
},
55-
},
56-
],
57-
},
58-
};
59-
60-
const client = kc.makeApiClient(k8s.CoreV1Api);
61-
await client.createNamespacedPod({ namespace: "default", body: pod });
62-
63-
// wait for pod to be ready
64-
let ready = false;
65-
for (const startTime = Date.now(); Date.now() - startTime < 60_000; ) {
66-
const podList = await client.listNamespacedPod({ namespace: "default" });
67-
const pod = podList.items.find((pod) => pod.metadata?.name === "helloworld");
68-
const status = pod?.status;
69-
ready =
70-
status?.phase === "Running" &&
71-
!!status?.conditions?.some((cond) => cond.type === "Ready" && cond.status === "True");
72-
if (ready) break;
73-
await setTimeout(3_000);
74-
}
75-
76-
expect(ready).toBe(true);
77-
78-
await container.stop();
79-
});
92+
],
93+
},
94+
};
95+
96+
const client = kc.makeApiClient(k8s.CoreV1Api);
97+
await client.createNamespacedPod({ namespace: "default", body: pod });
98+
99+
// wait for pod to be ready
100+
expect(await podIsReady(client, "default", "helloworld", 60_000)).toBe(true);
101+
102+
await container.stop();
103+
});
104+
}
80105
});
106+
107+
async function podIsReady(client: k8s.CoreV1Api, namespace: string, name: string, timeout: number): Promise<boolean> {
108+
for (const startTime = Date.now(); Date.now() - startTime < timeout; ) {
109+
const res = await client.readNamespacedPodStatus({ namespace, name });
110+
const ready =
111+
res.status?.phase === "Running" &&
112+
!!res.status?.conditions?.some((cond) => cond.type === "Ready" && cond.status === "True");
113+
if (ready) return true;
114+
await setTimeout(3_000);
115+
}
116+
return false;
117+
}

‎packages/modules/k3s/src/k3s-container.ts‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,59 @@
1-
import { AbstractStartedContainer, GenericContainer, Wait, type StartedTestContainer } from "testcontainers";
1+
import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers";
22
import tar from "tar-stream";
33
import { basename } from "node:path";
44

5-
// TODO: Update @types/dockerode
5+
// TODO: Implement GenericContainer.withCgroupnsMode
66
// https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71160
77
type CgroupnsModeConfig = { CgroupnsMode?: "private" | "host" };
88

99
const KUBE_CONFIG_PATH = "/etc/rancher/k3s/k3s.yaml";
1010
const KUBE_SECURE_PORT = 6443;
1111
const RANCHER_WEBHOOK_PORT = 8443;
1212

13-
/** Path to the k3s manifests directory. These are applied automatically on startup. */
14-
export const K3S_SERVER_MANIFESTS = "/var/lib/rancher/k3s/server/manifests/";
15-
1613
export class K3sContainer extends GenericContainer {
17-
constructor(image = "rancher/k3s:v1.31.2-k3s1") {
14+
constructor(image: string) {
1815
super(image);
1916
(this.hostConfig as CgroupnsModeConfig).CgroupnsMode = "host";
2017
this.withExposedPorts(KUBE_SECURE_PORT, RANCHER_WEBHOOK_PORT)
2118
.withPrivilegedMode()
22-
// TODO: Determine if/when bind mount is needed
23-
.withBindMounts([{ mode: "rw", source: "/sys/fs/cgroup", target: "/sys/fs/cgroup" }])
19+
// Why do Java and .NET implementations bind cgroup but Golang does not?
20+
.withBindMounts([{ source: "/sys/fs/cgroup", target: "/sys/fs/cgroup" }])
2421
.withTmpFs({ "/run": "rw" })
2522
.withTmpFs({ "/var/run": "rw" })
26-
// TODO: If tls-san is desirable, determine how to obtain the host address
27-
// .withCommand(["server", "--disable=traefik", `--tls-san=${this.getHost()}`])
28-
.withCommand(["server", "--disable=traefik"])
2923
.withWaitStrategy(Wait.forLogMessage("Node controller sync successful"))
3024
.withStartupTimeout(120_000);
3125
}
3226

3327
public override async start(): Promise<StartedK3sContainer> {
3428
const container = await super.start();
3529
const tarStream = await container.copyArchiveFromContainer(KUBE_CONFIG_PATH);
36-
const kubeConfig = await extractFromTarStream(tarStream, basename(KUBE_CONFIG_PATH));
37-
return new StartedK3sContainer(container, kubeConfig);
30+
const rawKubeConfig = await extractFromTarStream(tarStream, basename(KUBE_CONFIG_PATH));
31+
return new StartedK3sContainer(container, rawKubeConfig);
32+
}
33+
34+
protected override async beforeContainerCreated() {
35+
let command = this.createOpts.Cmd ?? ["server", "--disable=traefik"];
36+
if (this.networkMode && this.networkAliases.length > 0) {
37+
const aliases = this.networkAliases.join();
38+
command = [...command, `--tls-san=${aliases}`];
39+
}
40+
this.withCommand(command);
3841
}
3942
}
4043

4144
export class StartedK3sContainer extends AbstractStartedContainer {
42-
constructor(startedTestContainer: StartedTestContainer, private readonly kubeConfig: string) {
45+
constructor(startedTestContainer: StartedTestContainer, private readonly rawKubeConfig: string) {
4346
super(startedTestContainer);
4447
}
4548

4649
public getKubeConfig(): string {
4750
const serverUrl = `https://${this.getHost()}:${this.getMappedPort(KUBE_SECURE_PORT)}`;
48-
return kubeConfigWithServerUrl(this.kubeConfig, serverUrl);
51+
return kubeConfigWithServerUrl(this.rawKubeConfig, serverUrl);
52+
}
53+
54+
public getAliasedKubeConfig(networkAlias: string) {
55+
const serverUrl = `https://${networkAlias}:${KUBE_SECURE_PORT}`;
56+
return kubeConfigWithServerUrl(this.rawKubeConfig, serverUrl);
4957
}
5058
}
5159

@@ -73,6 +81,6 @@ async function extractFromTarStream(tarStream: NodeJS.ReadableStream, entryName:
7381
return extracted;
7482
}
7583

76-
export function kubeConfigWithServerUrl(kubeConfig: string, server: string): string {
84+
function kubeConfigWithServerUrl(kubeConfig: string, server: string): string {
7785
return kubeConfig.replace(/server:\s?[:/.\d\w]+/, `server: ${server}`);
7886
}

0 commit comments

Comments
 (0)