Skip to content

Commit 7bd491f

Browse files
dajudgedajudge
authored
Merge pull request #3 from dajudge/apiserver-only-container
Add ApiServerContainer
2 parents b5ec58c + 7cdb0bd commit 7bd491f

File tree

13 files changed

+778
-3
lines changed

13 files changed

+778
-3
lines changed

‎README.md‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,33 @@ public class SomeKubernetesTest {
4444
public static final KindContainer KUBE = new KindContainer();
4545

4646
@Test
47-
public void test_something() {
47+
public void verify_node_is_present() {
4848
// Do something useful with the fabric8 client it returns!
49-
System.out.println(KUBE.client());
49+
try(final KuberntesClient client = KUBE.client()) {
50+
assertEquals(1, client.nodes().list().getItems().size());
51+
}
52+
}
53+
}
54+
```
55+
56+
### API-Server-only testing
57+
If you don't need a full-fledged Kubernetes distribution for your testing, using the `ApiServerContainer`
58+
might be an option for you that shaves off a lot of the startup overhead of the `KindContainer`. The
59+
`ApiServerContainer` only starts an etcd instance and a Kubernetes API-Server, can be more than enough
60+
e.g. if all you want to test is if your custom operator handles it's CRDs properly or creates the required
61+
objects in the control plane.
62+
63+
```java
64+
public class SomeControlPlaneTest {
65+
@ClassRule
66+
public static final ApiServerContainer KUBE = new ApiServerContainer();
67+
68+
@Test
69+
public void verify_no_node_is_present() {
70+
// Do something useful with the fabric8 client it returns!
71+
try(final KuberntesClient client = KUBE.client()) {
72+
assertTrue(client.nodes().list().getItems().isEmpty());
73+
}
5074
}
5175
}
5276
```

‎src/main/java/com/dajudge/kindcontainer/ApiServerContainer.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
Copyright 2020-2021 Alex Stockinger
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package com.dajudge.kindcontainer;
17+
18+
import com.dajudge.kindcontainer.pki.CertAuthority;
19+
import com.dajudge.kindcontainer.pki.KeyStoreWrapper;
20+
import com.github.dockerjava.api.command.InspectContainerResponse;
21+
import com.github.dockerjava.api.model.ContainerNetwork;
22+
import io.fabric8.kubernetes.client.Config;
23+
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
24+
import io.fabric8.kubernetes.client.KubernetesClient;
25+
import org.testcontainers.shaded.org.bouncycastle.asn1.x509.GeneralName;
26+
27+
import java.util.Base64;
28+
import java.util.Map;
29+
30+
import static com.dajudge.kindcontainer.Utils.writeAsciiFile;
31+
import static java.lang.String.format;
32+
import static java.nio.charset.StandardCharsets.US_ASCII;
33+
import static java.util.Arrays.asList;
34+
import static org.testcontainers.utility.MountableFile.forClasspathResource;
35+
36+
public class ApiServerContainer extends BusyBoxContainer<ApiServerContainer> {
37+
private static final String API_SERVER_IMAGE = "k8s.gcr.io/kube-apiserver:v1.21.1";
38+
private static final String PKI_BASEDIR = "/etc/kubernetes/pki";
39+
private static final String ETCD_PKI_BASEDIR = PKI_BASEDIR + "/etcd";
40+
private static final String ETCD_CLIENT_KEY = ETCD_PKI_BASEDIR + "/etcd/apiserver-client.key";
41+
private static final String ETCD_CLIENT_CERT = ETCD_PKI_BASEDIR + "/etcd/apiserver-client.crt";
42+
private static final String ETCD_CLIENT_CA = ETCD_PKI_BASEDIR + "/etcd/ca.crt";
43+
private static final String API_SERVER_CA = PKI_BASEDIR + "/ca.crt";
44+
private static final String API_SERVER_CERT = PKI_BASEDIR + "/apiserver.crt";
45+
private static final String API_SERVER_KEY = PKI_BASEDIR + "/apiserver.key";
46+
private static final String API_SERVER_PUBKEY = PKI_BASEDIR + "/apiserver.pub";
47+
private static final String DOCKER_BASE_PATH = "/docker";
48+
private static final String RUN_SCRIPT_PATH = DOCKER_BASE_PATH + "/run-apiserver.sh";
49+
private static final String ENTRYPOINT_PATH = DOCKER_BASE_PATH + "/entrypoint-etcd.sh";
50+
private static final String IP_ADDRESS_PATH = DOCKER_BASE_PATH + "/ip.txt";
51+
private static final String ETCD_HOSTNAME_PATH = DOCKER_BASE_PATH + "/etcd.txt";
52+
private final CertAuthority apiServerCa = new CertAuthority(System::currentTimeMillis, "CN=API Server CA");
53+
private final EtcdContainer etcd;
54+
private final Config config = Config.empty();
55+
56+
public ApiServerContainer() {
57+
super(API_SERVER_IMAGE);
58+
etcd = new EtcdContainer();
59+
this
60+
.withCreateContainerCmdModifier(cmd -> {
61+
cmd.withEntrypoint(ENTRYPOINT_PATH);
62+
cmd.withCmd(RUN_SCRIPT_PATH);
63+
})
64+
.withEnv("ETCD_CLIENT_KEY", ETCD_CLIENT_KEY)
65+
.withEnv("ETCD_CLIENT_CERT", ETCD_CLIENT_CERT)
66+
.withEnv("ETCD_CLIENT_CA", ETCD_CLIENT_CA)
67+
.withEnv("API_SERVER_CA", API_SERVER_CA)
68+
.withEnv("API_SERVER_CERT", API_SERVER_CERT)
69+
.withEnv("API_SERVER_KEY", API_SERVER_KEY)
70+
.withEnv("API_SERVER_PUBKEY", API_SERVER_PUBKEY)
71+
.withEnv("IP_ADDRESS_PATH", IP_ADDRESS_PATH)
72+
.withEnv("ETCD_HOSTNAME_PATH", ETCD_HOSTNAME_PATH)
73+
.withCopyFileToContainer(forClasspathResource("scripts/entrypoint-apiserver.sh", 755), ENTRYPOINT_PATH)
74+
.withCopyFileToContainer(forClasspathResource("scripts/run-apiserver.sh", 755), RUN_SCRIPT_PATH)
75+
.withExposedPorts(6443);
76+
}
77+
78+
public KubernetesClient getClient() {
79+
return new DefaultKubernetesClient(config);
80+
}
81+
82+
@Override
83+
protected void containerIsStarting(final InspectContainerResponse containerInfo) {
84+
etcd.start();
85+
try {
86+
final String apiServerIpAddress = getApiServerIpAddress();
87+
final KeyStoreWrapper apiServerKeyPair = apiServerCa.newKeyPair("O=system:masters,CN=kubernetes-admin", asList(
88+
new GeneralName(GeneralName.iPAddress, apiServerIpAddress),
89+
new GeneralName(GeneralName.dNSName, "localhost")
90+
));
91+
final KeyStoreWrapper etcdClientKeyPair = etcd.newClientKeypair("CN=API Server");
92+
writeAsciiFile(this, etcdClientKeyPair.getCertificatePem(), ETCD_CLIENT_CERT);
93+
writeAsciiFile(this, etcdClientKeyPair.getPrivateKeyPem(), ETCD_CLIENT_KEY);
94+
writeAsciiFile(this, etcd.getCaCertificatePem(), ETCD_CLIENT_CA);
95+
writeAsciiFile(this, apiServerKeyPair.getCertificatePem(), API_SERVER_CERT);
96+
writeAsciiFile(this, apiServerKeyPair.getPrivateKeyPem(), API_SERVER_KEY);
97+
writeAsciiFile(this, apiServerKeyPair.getPublicKeyPem(), API_SERVER_PUBKEY);
98+
writeAsciiFile(this, apiServerCa.getCaKeyStore().getCertificatePem(), API_SERVER_CA);
99+
writeAsciiFile(this, apiServerIpAddress, IP_ADDRESS_PATH);
100+
writeAsciiFile(this, etcd.getEtcdIpAddress(), ETCD_HOSTNAME_PATH);
101+
config.setCaCertData(base64(apiServerCa.getCaKeyStore().getCertificatePem()));
102+
config.setClientCertData(base64(apiServerKeyPair.getCertificatePem()));
103+
config.setClientKeyData(base64(apiServerKeyPair.getPrivateKeyPem()));
104+
config.setMasterUrl(format("https://%s:%d", getContainerIpAddress(), getMappedPort(6443)));
105+
config.setConnectionTimeout(10000);
106+
config.setRequestTimeout(60000);
107+
} catch (final RuntimeException e) {
108+
etcd.close();
109+
throw e;
110+
}
111+
}
112+
113+
private String base64(final String str) {
114+
return Base64.getEncoder().encodeToString(str.getBytes(US_ASCII));
115+
}
116+
117+
public String getApiServerIpAddress() {
118+
final Map<String, ContainerNetwork> networks = getContainerInfo().getNetworkSettings().getNetworks();
119+
return networks.values().iterator().next().getIpAddress();
120+
}
121+
122+
@Override
123+
public void stop() {
124+
super.stop();
125+
etcd.stop();
126+
}
127+
}

‎src/main/java/com/dajudge/kindcontainer/BusyBoxContainer.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
Copyright 2020-2021 Alex Stockinger
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package com.dajudge.kindcontainer;
17+
18+
import org.testcontainers.containers.GenericContainer;
19+
import org.testcontainers.images.builder.ImageFromDockerfile;
20+
import org.testcontainers.images.builder.dockerfile.statement.SingleArgumentStatement;
21+
22+
import java.util.List;
23+
24+
import static java.lang.String.format;
25+
import static java.util.Arrays.asList;
26+
import static java.util.stream.Collectors.joining;
27+
28+
abstract class BusyBoxContainer<T extends GenericContainer<T>> extends GenericContainer<T> {
29+
30+
protected static final String[] COMMANDS = {"sleep", "ls", "cat", "grep", "awk", "nc", "bash", "sh"};
31+
32+
protected BusyBoxContainer(final String image) {
33+
super(buildImageWithBusybox(image, COMMANDS));
34+
}
35+
36+
private static String linkBusyBox(final List<String> sources) {
37+
return sources.stream()
38+
.map(s -> format("/bin/ln -s /bin/busybox /tmp/busybox/%s", s))
39+
.collect(joining(" && "));
40+
}
41+
42+
private static ImageFromDockerfile buildImageWithBusybox(final String image, final String[] commands) {
43+
return new ImageFromDockerfile().withDockerfileFromBuilder(builder -> builder
44+
.withStatement(new SingleArgumentStatement("FROM", "busybox as builder"))
45+
.run("mkdir -p /tmp/busybox")
46+
.run(linkBusyBox(asList(commands)))
47+
.withStatement(new SingleArgumentStatement("FROM", image))
48+
.withStatement(new SingleArgumentStatement("COPY", "--from=builder /bin/busybox /bin/busybox"))
49+
.withStatement(new SingleArgumentStatement("COPY", "--from=builder /tmp/busybox/* /bin/"))
50+
);
51+
}
52+
}

‎src/main/java/com/dajudge/kindcontainer/EtcdContainer.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
Copyright 2020-2021 Alex Stockinger
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package com.dajudge.kindcontainer;
17+
18+
import com.dajudge.kindcontainer.pki.CertAuthority;
19+
import com.dajudge.kindcontainer.pki.KeyStoreWrapper;
20+
import com.github.dockerjava.api.command.InspectContainerResponse;
21+
import com.github.dockerjava.api.model.ContainerNetwork;
22+
import org.testcontainers.shaded.org.bouncycastle.asn1.x509.GeneralName;
23+
24+
import java.util.Map;
25+
26+
import static com.dajudge.kindcontainer.Utils.writeAsciiFile;
27+
import static java.util.Collections.emptyList;
28+
import static java.util.Collections.singletonList;
29+
import static org.testcontainers.utility.MountableFile.forClasspathResource;
30+
31+
class EtcdContainer extends BusyBoxContainer<EtcdContainer> {
32+
private static final String DOCKER_BASE_PATH = "/docker";
33+
private static final String RUN_SCRIPT_PATH = DOCKER_BASE_PATH + "/run-etcd.sh";
34+
private static final String ENTRYPOINT_PATH = DOCKER_BASE_PATH + "/entrypoint-etcd.sh";
35+
private static final String SERVER_CERT_PATH = DOCKER_BASE_PATH + "/server.crt";
36+
private static final String SERVER_KEY_PATH = DOCKER_BASE_PATH + "/server.key";
37+
private static final String SERVER_CACERTS_PATH = DOCKER_BASE_PATH + "/ca.crt";
38+
private static final String IP_ADDRESS_PATH = DOCKER_BASE_PATH + "/ip.txt";
39+
private static final int ETCD_PORT = 2379;
40+
private static final String ETCD_IMAGE = "k8s.gcr.io/etcd:3.4.13-0";
41+
private final CertAuthority etcdCa = new CertAuthority(System::currentTimeMillis, "CN=etcd CA");
42+
43+
EtcdContainer() {
44+
super(ETCD_IMAGE);
45+
this
46+
.withCreateContainerCmdModifier(cmd -> {
47+
cmd.withEntrypoint(ENTRYPOINT_PATH);
48+
cmd.withCmd(RUN_SCRIPT_PATH);
49+
})
50+
.withEnv("SERVER_CERT_PATH", SERVER_CERT_PATH)
51+
.withEnv("SERVER_KEY_PATH", SERVER_KEY_PATH)
52+
.withEnv("SERVER_CACERTS_PATH", SERVER_CACERTS_PATH)
53+
.withEnv("IP_ADDRESS_PATH", IP_ADDRESS_PATH)
54+
.withCopyFileToContainer(forClasspathResource("scripts/entrypoint-etcd.sh", 755), ENTRYPOINT_PATH)
55+
.withCopyFileToContainer(forClasspathResource("scripts/run-etcd.sh", 755), RUN_SCRIPT_PATH)
56+
.withExposedPorts(ETCD_PORT);
57+
}
58+
59+
@Override
60+
protected void containerIsStarting(final InspectContainerResponse containerInfo) {
61+
final String etcdIpAddress = getEtcdIpAddress();
62+
final KeyStoreWrapper etcdKeypair = etcdCa.newKeyPair(
63+
"CN=etcd",
64+
singletonList(new GeneralName(GeneralName.iPAddress, etcdIpAddress))
65+
);
66+
writeAsciiFile(this, etcdKeypair.getCertificatePem(), SERVER_CERT_PATH);
67+
writeAsciiFile(this, etcdKeypair.getPrivateKeyPem(), SERVER_KEY_PATH);
68+
writeAsciiFile(this, etcdCa.getCaKeyStore().getCertificatePem(), SERVER_CACERTS_PATH);
69+
writeAsciiFile(this, etcdIpAddress, IP_ADDRESS_PATH);
70+
}
71+
72+
public String getEtcdIpAddress() {
73+
final Map<String, ContainerNetwork> networks = getContainerInfo().getNetworkSettings().getNetworks();
74+
return networks.values().iterator().next().getIpAddress();
75+
}
76+
77+
public KeyStoreWrapper newClientKeypair(final String dn) {
78+
return etcdCa.newKeyPair(dn, emptyList());
79+
}
80+
81+
protected String getCaCertificatePem() {
82+
return etcdCa.getCaKeyStore().getCertificatePem();
83+
}
84+
}

‎src/main/java/com/dajudge/kindcontainer/Utils.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,17 @@
1818

1919
import org.slf4j.Logger;
2020
import org.slf4j.LoggerFactory;
21+
import org.testcontainers.containers.GenericContainer;
22+
import org.testcontainers.images.builder.Transferable;
2123

2224
import java.io.ByteArrayOutputStream;
2325
import java.io.IOException;
2426
import java.io.InputStream;
25-
import java.util.function.Function;
2627
import java.util.function.Supplier;
2728

2829
import static java.lang.System.currentTimeMillis;
2930
import static java.lang.Thread.sleep;
31+
import static java.nio.charset.StandardCharsets.US_ASCII;
3032
import static java.nio.charset.StandardCharsets.UTF_8;
3133

3234
final class Utils {
@@ -86,4 +88,8 @@ static <T, E extends Exception> T waitUntilNotNull(
8688
public static String indent(final String prefix, final String string) {
8789
return string.replaceAll("(?m)^", prefix);
8890
}
91+
92+
static void writeAsciiFile(final GenericContainer<?> container, final String text, final String path) {
93+
container.copyFileToContainer(Transferable.of(text.getBytes(US_ASCII)), path);
94+
}
8995
}

0 commit comments

Comments
 (0)