Skip to content

Commit 193792d

Browse files
migesokmigesok
committed
Add integration tests for K8sDnsNameResolver
Resolves #2
1 parent 410675a commit 193792d

File tree

14 files changed

+957
-11
lines changed

14 files changed

+957
-11
lines changed

‎build.sbt‎

Lines changed: 76 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import Dependencies.*
2+
import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
23
import sbt.*
34

45
ThisBuild / organization := "com.evolution.jgrpc.tools"
@@ -26,27 +27,53 @@ ThisBuild / scmInfo := Some(ScmInfo(
2627
// not sure if bincompat check works for Java code, put it here just in case
2728
ThisBuild / versionPolicyIntention := Compatibility.BinaryCompatible
2829

29-
// this is a Java project, setting a fixed Scala version just in case
30-
ThisBuild / scalaVersion := "2.13.16"
31-
3230
// setting pure-Java module build settings
3331
ThisBuild / crossPaths := false // drop off Scala suffix from artifact names.
3432
ThisBuild / autoScalaLibrary := false // exclude scala-library from dependencies
3533
ThisBuild / javacOptions := Seq("-source", "17", "-target", "17", "-Werror", "-Xlint:all")
3634
ThisBuild / doc / javacOptions := Seq("-source", "17", "-Xdoclint:all", "-Werror")
37-
38-
// common test dependencies:
39-
ThisBuild / libraryDependencies ++= Seq(
40-
// to be able to run JUnit 5+ tests:
41-
"com.github.sbt.junit" % "jupiter-interface" % JupiterKeys.jupiterVersion.value,
42-
Slf4j.simple,
43-
).map(_ % Test)
35+
ThisBuild / scalaVersion := "2.13.18"
36+
ThisBuild / scalacOptions ++= Seq(
37+
"-release:17",
38+
"-deprecation",
39+
"-Xsource:3",
40+
)
4441

4542
// common compile dependencies:
4643
ThisBuild / libraryDependencies ++= Seq(
4744
jspecify, // JSpecify null-check annotations
4845
)
4946

47+
def asJavaPublishedModule(p: Project): Project = {
48+
p.settings(
49+
// common test dependencies for Java modules:
50+
libraryDependencies ++= Seq(
51+
// to be able to run JUnit 5+ tests:
52+
"com.github.sbt.junit" % "jupiter-interface" % JupiterKeys.jupiterVersion.value,
53+
Slf4j.simple,
54+
).map(_ % Test),
55+
)
56+
}
57+
58+
def asScalaIntegrationTestModule(p: Project): Project = {
59+
p.disablePlugins(JupiterPlugin) // using scalatest instead
60+
.settings(
61+
publish / skip := true,
62+
autoScalaLibrary := true, // int tests are written in Scala, returning scala-library dependency
63+
Test / parallelExecution := false, // disable parallel execution between test suites
64+
Test / fork := true, // disable parallel execution between modules
65+
// tests take a long time to run, better to see the process in real time
66+
Test / logBuffered := false,
67+
// disable scaladoc generation to avoid dealing with annoying warnings
68+
Compile / doc / sources := Seq.empty,
69+
// common test dependencies for Scala int test modules:
70+
libraryDependencies ++= Seq(
71+
scalatest,
72+
Slf4j.simple,
73+
).map(_ % Test),
74+
)
75+
}
76+
5077
lazy val root = project.in(file("."))
5178
.settings(
5279
name := "grpc-java-tools-root",
@@ -55,9 +82,11 @@ lazy val root = project.in(file("."))
5582
)
5683
.aggregate(
5784
k8sDnsNameResolver,
85+
k8sDnsNameResolverIt,
5886
)
5987

6088
lazy val k8sDnsNameResolver = project.in(file("k8s-dns-name-resolver"))
89+
.configure(asJavaPublishedModule)
6190
.settings(
6291
name := "k8s-dns-name-resolver",
6392
description := "Evolution grpc-java tools - DNS-based name resolver for Kubernetes services",
@@ -68,6 +97,43 @@ lazy val k8sDnsNameResolver = project.in(file("k8s-dns-name-resolver"))
6897
),
6998
)
7099

100+
lazy val k8sDnsNameResolverIt = project.in(file("k8s-dns-name-resolver-it"))
101+
.configure(asScalaIntegrationTestModule)
102+
// the module builds its own test app docker container
103+
.enablePlugins(JavaAppPackaging, DockerPlugin)
104+
.settings(
105+
name := "k8s-dns-name-resolver-it",
106+
description := "Evolution grpc-java tools - DNS-based name resolver for Kubernetes services - integration tests",
107+
Compile / PB.targets := Seq(
108+
scalapb.gen() -> (Compile / sourceManaged).value / "scalapb",
109+
),
110+
dockerBaseImage := "amazoncorretto:17-alpine",
111+
dockerCommands ++= Seq(
112+
// root rights are needed to install additional packages, and also test client needs it
113+
// to manipulate its DNS settings
114+
Cmd("USER", "root"),
115+
// bash is needed for testcontainers log watching logic
116+
// lsof and coredns are needed for the integration test logic
117+
ExecCmd("RUN", "apk", "add", "--no-cache", "bash", "lsof", "coredns"),
118+
),
119+
dockerExposedPorts := Seq(9000), // Should match the test app GRPC server port.
120+
// The int test here needs the test app docker container staged before running the code.
121+
// It's then used in docker compose inside testcontainers.
122+
test := {
123+
(Docker / stage).value
124+
(Test / test).value
125+
},
126+
libraryDependencies ++= Seq(
127+
Slf4j.simple,
128+
commonsLang3,
129+
"io.grpc" % "grpc-netty" % scalapb.compiler.Version.grpcJavaVersion,
130+
"com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion,
131+
Testcontainers.core % Test,
132+
),
133+
).dependsOn(
134+
k8sDnsNameResolver,
135+
)
136+
71137
addCommandAlias("fmt", "all scalafmtAll scalafmtSbt javafmtAll")
72138
addCommandAlias(
73139
"build",

‎k8s-dns-name-resolver-it/src/main/protobuf/test_svc.proto‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
syntax = "proto2";
2+
3+
package k8sdns.it;
4+
5+
service TestSvc {
6+
rpc GetId (GetIdRequest) returns (GetIdReply) {}
7+
}
8+
9+
message GetIdRequest {}
10+
11+
message GetIdReply {
12+
required int32 id = 1;
13+
}

‎k8s-dns-name-resolver-it/src/main/resources/simplelogger.properties‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Configure slf4j-simple to have concise output for tests
2+
# Supported settings: https://www.slf4j.org/api/org/slf4j/simple/SimpleLogger.html
3+
org.slf4j.simpleLogger.logFile=System.out
4+
org.slf4j.simpleLogger.showDateTime=true
5+
org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS
6+
org.slf4j.simpleLogger.showThreadName=false
7+
org.slf4j.simpleLogger.showShortLogName=true

‎k8s-dns-name-resolver-it/src/main/scala/com/evolution/jgrpc/tools/k8sdns/it/TestApp.scala‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package com.evolution.jgrpc.tools.k8sdns.it
2+
3+
/**
4+
* `K8sDnsNameResolver` integration test service app entrypoint.
5+
*
6+
* Depending on the run mode environment variable value could either work as
7+
* [[TestServer]] or [[TestClient]].
8+
*/
9+
object TestApp extends App {
10+
private val runModeEnvVarName = "TEST_SVC_RUN_MODE"
11+
private val instanceIdVarName = "TEST_SVC_INSTANCE_ID"
12+
13+
sys.env.get(runModeEnvVarName) match {
14+
case None =>
15+
sys.error(s"missing environment variable: $runModeEnvVarName")
16+
case Some("server") =>
17+
runServer()
18+
case Some("client") =>
19+
runClient()
20+
case Some(unexpectedRunMode) =>
21+
sys.error(s"unexpected run mode: $unexpectedRunMode")
22+
}
23+
24+
private def runServer(): Unit = {
25+
val instanceId = sys.env.getOrElse(
26+
instanceIdVarName,
27+
sys.error(s"missing environment variable: $instanceIdVarName"),
28+
).toInt
29+
30+
new TestServer(instanceId = instanceId).run()
31+
}
32+
33+
private def runClient(): Unit = {
34+
new TestClient().run()
35+
}
36+
}

‎k8s-dns-name-resolver-it/src/main/scala/com/evolution/jgrpc/tools/k8sdns/it/TestAppShared.scala‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
package com.evolution.jgrpc.tools.k8sdns.it
2+
3+
import java.nio.file.*
4+
5+
/**
6+
* Common things shared between the `K8sDnsNameResolver` integration test code and the
7+
* test service code
8+
*
9+
* @see
10+
* [[TestApp]]
11+
*/
12+
object TestAppShared {
13+
14+
/**
15+
* [[TestApp]] GRPC server port
16+
*/
17+
val ServerPort: Int = 9000
18+
19+
/**
20+
* Docker compose service names for [[TestApp]] containers.
21+
*
22+
* The names here should match the ones used in the
23+
* `src/test/resources/docker/compose-test.yml` file.
24+
*/
25+
object TestAppSvcNames {
26+
27+
/**
28+
* First [[TestApp]] service in a [[TestServer]] mode
29+
*/
30+
val Server1: String = "test-server1"
31+
32+
/**
33+
* Second [[TestApp]] service in a [[TestServer]] mode
34+
*/
35+
val Server2: String = "test-server2"
36+
37+
/**
38+
* [[TestApp]] service in a [[TestClient]] mode
39+
*/
40+
val Client: String = "test-client"
41+
}
42+
43+
/**
44+
* `K8sDnsNameResolver` integration test watches for these [[TestApp]] log messages in
45+
* the stdout.
46+
*/
47+
object TestAppSpecialLogMsgs {
48+
49+
/**
50+
* [[TestApp]] docker container has been started and ready to proceed with the test
51+
*/
52+
val Ready: String = "TEST CONTAINER READY"
53+
54+
/**
55+
* [[TestApp]] in the [[TestClient]] mode died prematurely, all the tests should be
56+
* aborted
57+
*/
58+
val ClientPrematureDeath: String = "TEST CLIENT PANIC"
59+
60+
/**
61+
* [[TestApp]] in the [[TestClient]] mode completed a requested test case successfully
62+
*
63+
* @see
64+
* [[TestClientControl]] for how to request a test case execution
65+
*/
66+
val ClientTestCaseSuccess: String = "TEST SUCCESS"
67+
68+
/**
69+
* [[TestApp]] in the [[TestClient]] mode ran a requested test case and got a failure
70+
*
71+
* @see
72+
* [[TestClientControl]] for how to request a test case execution
73+
*/
74+
val ClientTestCaseFailed: String = "TEST FAILED"
75+
}
76+
77+
/**
78+
* Defines the way to send commands to the [[TestApp]] container in the [[TestClient]]
79+
* mode:
80+
* - create an empty file in the [[CmdDirPath]] directory on the container - the name
81+
* of the file is the command name
82+
* - the [[TestClient]] code deletes the file and queues the command for execution
83+
* - commands are executed on the [[TestClient]] one-by-one
84+
* - monitor [[TestClient]] container stdout for the command progress - see
85+
* [[TestAppSpecialLogMsgs]]
86+
*
87+
* Currently supported commands:
88+
* - [[RunTestCaseCmdFileName]] for running [[TestClientTestCase]]
89+
*/
90+
object TestClientControl {
91+
92+
/**
93+
* Directory which [[TestApp]] in the [[TestClient]] mode uses for receiving commands
94+
*
95+
* @see
96+
* [[TestClientControl]]
97+
*/
98+
val CmdDirPath: Path = Paths.get("/tmp/test-client-control")
99+
100+
/**
101+
* [[TestClientControl]] command for running [[TestClientTestCase]].
102+
*/
103+
object RunTestCaseCmdFileName {
104+
private val fileNamePrefix = ".run-test-case-"
105+
106+
/**
107+
* Creates a [[TestClientControl]] command file name for running the given
108+
* [[TestClientTestCase]]
109+
*/
110+
def apply(testCase: TestClientTestCase): String = {
111+
s"$fileNamePrefix${ testCase.name }"
112+
}
113+
114+
/**
115+
* Matches [[TestClientControl]] command file name which runs a
116+
* [[TestClientTestCase]]
117+
*/
118+
def unapply(fileName: String): Option[TestClientTestCase] = {
119+
if (fileName.startsWith(fileNamePrefix)) {
120+
val testCaseName = fileName.drop(fileNamePrefix.length)
121+
TestClientTestCase.values.find(_.name == testCaseName)
122+
} else {
123+
None
124+
}
125+
}
126+
}
127+
}
128+
129+
/**
130+
* Test case to run on [[TestClient]].
131+
*
132+
* @see
133+
* [[TestClientControl.RunTestCaseCmdFileName]]
134+
*/
135+
sealed abstract class TestClientTestCase extends Product {
136+
final def name: String = productPrefix
137+
}
138+
object TestClientTestCase {
139+
val values: Vector[TestClientTestCase] = Vector(
140+
DiscoverNewPod,
141+
DnsFailureRecover,
142+
)
143+
144+
/**
145+
* [[TestClient]] test case verifying that `K8sDnsNameResolver` live pod discovery
146+
* works.
147+
*
148+
* Test steps overview:
149+
* - point the service host DNS records to one server container
150+
* - create a GRPC client, check that it sees only the first server
151+
* - add the second server to the DNS records
152+
* - check that after the configured reload TTL, the client sees both servers
153+
*/
154+
case object DiscoverNewPod extends TestClientTestCase
155+
156+
/**
157+
* [[TestClient]] test case verifying that `K8sDnsNameResolver` recovers after a DNS
158+
* call failure.
159+
*
160+
* Test steps overview:
161+
* - point the service host DNS records to one server container
162+
* - create a GRPC client, check that it sees only the first server
163+
* - stop the DNS server, wait until the client gets a DNS error
164+
* - start the DNS server back again, with 2 servers in the records
165+
* - check that after the configured reload TTL, the client sees both servers
166+
*/
167+
case object DnsFailureRecover extends TestClientTestCase
168+
}
169+
}

0 commit comments

Comments
 (0)