feat: resolve docker file to canonical format (#2029)

* feat: resolve docker file to canonical format

* add .env to each sample

* docker-compose file must be valid

Author: octonato

devtools/src/main/scala/kalix/devtools/impl/DockerComposeUtils.scala

@@ -20,6 +20,7 @@ import java.io.File
 import java.nio.file.Files
 import java.nio.file.Paths
 
+import scala.collection.mutable
 import scala.io.Source
 import scala.jdk.CollectionConverters._
 import scala.sys.process._
@@ -28,19 +29,14 @@ import com.typesafe.config.Config
 
 object DockerComposeUtils {
 
-  def apply(file: String): DockerComposeUtils = DockerComposeUtils(file, Map.empty)
-
   def fromConfig(config: Config): Option[DockerComposeUtils] =
     Option(config.getString("kalix.dev-mode.docker-compose-file"))
       .filter(_.trim.toLowerCase != "none")
       .filter { file => new File(sys.props("user.dir"), file).exists() }
-      .map { file => DockerComposeUtils(file, sys.env) }
+      .map { file => DockerComposeUtils(file) }
 }
 
-case class DockerComposeUtils(file: String, envVar: Map[String, String]) {
-
-  // mostly for using from Java code
-  def this(file: String) = this(file, Map.empty)
+case class DockerComposeUtils(file: String) {
 
   @volatile private var started = false
 
@@ -61,12 +57,10 @@ case class DockerComposeUtils(file: String, envVar: Map[String, String]) {
   // we will need to iterate over it more than once
   private lazy val lines: Seq[String] =
     if (Files.exists(Paths.get(file))) {
-      val src = Source.fromFile(file)
-      try {
-        src.getLines().toList
-      } finally {
-        src.close()
-      }
+      val collectedLines = mutable.Buffer.empty[String]
+      val processLogger = ProcessLogger(out => collectedLines.append(out))
+      Process(s"docker-compose -f $file config", None).!(processLogger)
+      collectedLines.toSeq // to immutable Seq
     } else {
       Seq.empty
     }
@@ -106,12 +100,7 @@ case class DockerComposeUtils(file: String, envVar: Map[String, String]) {
     else 0
 
   def userFunctionPort: Int =
-    envVar
-      .get("USER_SERVICE_PORT")
-      .orElse(envVar.get("USER_FUNCTION_PORT")) // legacy name
-      .map(_.toInt)
-      .orElse(userFunctionPortFromFile)
-      .getOrElse(8080)
+    userFunctionPortFromFile.getOrElse(8080)
 
   private def userFunctionPortFromFile: Option[Int] =
     lines.collectFirst { case UserServicePortExtractor(port) => port }

devtools/src/test/scala/kalix/devtools/impl/DevModeSettingsSpec.scala

@@ -40,8 +40,8 @@ object DevModeSettingsSpec {
       |        -Dkalix.dev-mode.service-port-mappings.foo=9001
       |        -Dkalix.dev-mode.service-port-mappings.bar=9002
       |        -Dkalix.dev-mode.service-port-mappings.baz=host.docker.internal:9003
-      |      USER_SERVICE_HOST:${USER_SERVICE_HOST:-host.docker.internal}
-      |      USER_SERVICE_PORT:${USER_SERVICE_PORT:-8081}
+      |      USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
+      |      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8081}
       |""".stripMargin
 }
 

devtools/src/test/scala/kalix/devtools/impl/DockerComposeTestFile.scala

@@ -22,9 +22,24 @@ import java.io.FileWriter
 
 object DockerComposeTestFile {
 
-  def createTmpFile(fileContent: String): String = {
+  def createTmpFile(fileContent: String, env: Map[String, String] = Map.empty): String = {
     // write docker-compose.yml to a temporary file
     val userDir = sys.props("user.dir")
+
+    val envFile = new File(new File(userDir, "target"), ".env")
+    // if previous exist, we should delete it
+    if (envFile.exists()) envFile.delete()
+
+    // create .env file if needed
+    if (env.nonEmpty) {
+      envFile.deleteOnExit()
+      val envBuff = new BufferedWriter(new FileWriter(envFile))
+      env.foreach { case (key, value) =>
+        envBuff.write(s"$key=$value")
+      }
+      envBuff.close()
+    }
+
     val dockerComposeFile = File.createTempFile("docker-compose-", ".yml", new File(userDir, "target"))
     dockerComposeFile.deleteOnExit()
     val bw = new BufferedWriter(new FileWriter(dockerComposeFile))

devtools/src/test/scala/kalix/devtools/impl/DockerComposeUtilsSpec.scala

@@ -35,7 +35,7 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
       |version: "3"
       |services:
       |  kalix-runtime:
-      |    image: gcr.io/kalix-public/kalix-runtime:1.1.27
+      |    image: gcr.io/kalix-public/kalix-runtime:1.1.31
       |    ports:
       |      - "9000:9000"
       |    extra_hosts:
@@ -46,8 +46,8 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
       |        -Dlogback.configurationFile=logback-dev-mode.xml
       |        -Dkalix.dev-mode.service-port-mappings.foo=9001
       |        -Dkalix.dev-mode.service-port-mappings.bar=9002
-      |      USER_SERVICE_HOST:${USER_SERVICE_HOST:-host.docker.internal}
-      |      USER_SERVICE_PORT:${USER_SERVICE_PORT:-8081}
+      |      USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
+      |      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8081}
       |""".stripMargin
 
   "DockerComposeUtils" should {
@@ -60,8 +60,8 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
 
     "favor USER_SERVICE_PORT env var if set " in {
       val envVar = Map("USER_SERVICE_PORT" -> "8082")
-      val dockerComposeFile = createTmpFile(defaultFile)
-      val dockerComposeUtils = DockerComposeUtils(dockerComposeFile, envVar)
+      val dockerComposeFile = createTmpFile(defaultFile, envVar)
+      val dockerComposeUtils = DockerComposeUtils(dockerComposeFile)
       dockerComposeUtils.userFunctionPort shouldBe 8082
     }
 
@@ -71,30 +71,31 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
           |version: "3"
           |services:
           |  kalix-runtime:
-          |    image: gcr.io/kalix-public/kalix-runtime:1.1.27
+          |    image: gcr.io/kalix-public/kalix-runtime:1.1.31
           |    ports:
           |      - "9000:9000"
           |    extra_hosts:
           |      - "host.docker.internal:host-gateway"
           |""".stripMargin
+
       val dockerComposeFile = createTmpFile(fileContent)
       val dockerComposeUtils = DockerComposeUtils(dockerComposeFile)
       dockerComposeUtils.userFunctionPort shouldBe 8080
     }
 
-    "be able read user function port when env var is not used" in {
+    "be able to read user service port when env var is not used" in {
       val fileWithoutEnvVar =
         """
           |version: "3"
           |services:
           |  kalix-runtime:
-          |    image: gcr.io/kalix-public/kalix-runtime:1.1.27
+          |    image: gcr.io/kalix-public/kalix-runtime:1.1.31
           |    ports:
           |      - "9000:9000"
           |    extra_hosts:
           |      - "host.docker.internal:host-gateway"
           |    environment:
-          |      USER_SERVICE_PORT:8081
+          |      USER_SERVICE_PORT: 8081
           |""".stripMargin
 
       val dockerComposeFile = createTmpFile(fileWithoutEnvVar)
@@ -120,7 +121,7 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
       val extraProxy =
         """
           |  kalix-runtime-2:
-          |    image: gcr.io/kalix-public/kalix-runtime:1.1.27
+          |    image: gcr.io/kalix-public/kalix-runtime:1.1.31
           |    ports:
           |      - "9000:9000"
           |    extra_hosts:
@@ -129,8 +130,8 @@ class DockerComposeUtilsSpec extends AnyWordSpec with Matchers with OptionValues
           |      JAVA_TOOL_OPTIONS: >
           |        -Dconfig.resource=dev-mode.conf
           |        -Dlogback.configurationFile=logback-dev-mode.xml
-          |      USER_SERVICE_HOST:${USER_SERVICE_HOST:-host.docker.internal}
-          |      USER_SERVICE_PORT:${USER_SERVICE_PORT:-8082}
+          |      USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
+          |      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8082}
           |""".stripMargin
 
       val dockerComposeFile = createTmpFile(defaultFile + extraProxy)

samples/java-protobuf-customer-registry-kafka-quickstart/.env

@@ -0,0 +1 @@
+USER_SERVICE_PORT=8080

samples/java-protobuf-customer-registry-kafka-quickstart/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       JAVA_TOOL_OPTIONS: >
         -Dkalix.proxy.eventing.support=kafka
       USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
-      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8080}
+      USER_SERVICE_PORT: ${USER_SERVICE_PORT}
       # configuring kafka broker used for eventing
       BROKER_SERVERS: kafka:29092
 

samples/java-protobuf-customer-registry-quickstart/.env

@@ -0,0 +1 @@
+USER_SERVICE_PORT=8080

samples/java-protobuf-customer-registry-quickstart/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       # jvm -D properties can be added under this environment map (note: remove this comment when adding properties)
 
       USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
-      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8080}
+      USER_SERVICE_PORT: ${USER_SERVICE_PORT}
       #PUBSUB_EMULATOR_HOST: gcloud-pubsub-emulator
   #gcloud-pubsub-emulator:
   #  image: gcr.io/google.com/cloudsdktool/cloud-sdk:341.0.0

samples/java-protobuf-customer-registry-views-quickstart/.env

@@ -0,0 +1 @@
+USER_SERVICE_PORT=8080

samples/java-protobuf-customer-registry-views-quickstart/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       # jvm -D properties can be added under this environment map (note: remove this comment when adding properties)
 
       USER_SERVICE_HOST: ${USER_SERVICE_HOST:-host.docker.internal}
-      USER_SERVICE_PORT: ${USER_SERVICE_PORT:-8080}
+      USER_SERVICE_PORT: ${USER_SERVICE_PORT}
       #PUBSUB_EMULATOR_HOST: gcloud-pubsub-emulator
   #gcloud-pubsub-emulator:
   #  image: gcr.io/google.com/cloudsdktool/cloud-sdk:341.0.0