diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index b91c239b6abe7..bab7167112354 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -26,6 +26,7 @@ spark-kubernetes-integration-tests_2.11 + 1.7.24 kubernetes-integration-tests jar @@ -42,6 +43,16 @@ spark-core_${scala.binary.version} ${project.version} + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + test + + + log4j + log4j + org.apache.spark spark-kubernetes_${scala.binary.version} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 50ee46c93b592..c6f757b9908cb 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -38,6 +38,13 @@ import org.apache.spark.deploy.k8s.submit.{Client, ClientArguments, JavaMainAppR import org.apache.spark.launcher.SparkLauncher import org.apache.spark.util.Utils +/** + * This is a long running Suite. + * + * Console output will be empty while docker images are built. + * + * More logs is printed in ./target/integration-tests.log + */ private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter { import KubernetesSuite._ private val testBackend = IntegrationTestBackendFactory.getTestBackend() diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala new file mode 100644 index 0000000000000..459c0a4138b86 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import org.apache.log4j.{Logger, LogManager, Priority} + +trait Logging { + + private val log: Logger = LogManager.getLogger(this.getClass) + + protected def logDebug(msg: => String) = if (log.isDebugEnabled) log.debug(msg) + + protected def logInfo(msg: => String) = if (log.isInfoEnabled) log.info(msg) + + protected def logWarning(msg: => String) = if (log.isEnabledFor(Priority.WARN)) log.warn(msg) + + protected def logWarning(msg: => String, throwable: Throwable) = + if (log.isEnabledFor(Priority.WARN)) log.warn(msg, throwable) + + protected def logError(msg: => String) = if (log.isEnabledFor(Priority.ERROR)) log.error(msg) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/LoggingBuildHandler.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/LoggingBuildHandler.scala new file mode 100644 index 0000000000000..164ed08028613 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/LoggingBuildHandler.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.docker + +import com.spotify.docker.client.ProgressHandler +import com.spotify.docker.client.exceptions.DockerException +import com.spotify.docker.client.messages.ProgressMessage +import org.apache.spark.deploy.k8s.integrationtest.Logging + +class LoggingBuildHandler() extends ProgressHandler with Logging { + @throws[DockerException] + def progress(message: ProgressMessage) { + if (message.error != null) throw new DockerException(message.toString) + else if (message.status == null) { + logInfo(s"build: $message") + } + } +} \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/SparkDockerImageBuilder.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/SparkDockerImageBuilder.scala index cf2766d81859f..156fb954f868e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/SparkDockerImageBuilder.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/SparkDockerImageBuilder.scala @@ -21,17 +21,13 @@ import java.net.URI import java.nio.file.Paths import scala.collection.JavaConverters._ - -import com.spotify.docker.client.{DefaultDockerClient, DockerCertificates, LoggingBuildHandler} +import com.spotify.docker.client.{DefaultDockerClient, DockerCertificates} import org.apache.http.client.utils.URIBuilder +import org.apache.spark.deploy.k8s.integrationtest.Logging import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.{Minutes, Seconds, Span} - -import org.apache.spark.internal.Logging import org.apache.spark.util.RedirectThread - - private[spark] class SparkDockerImageBuilder (private val dockerEnv: Map[String, String]) extends Logging{ @@ -113,10 +109,13 @@ private[spark] class SparkDockerImageBuilder } private def buildImage(name: String, dockerFile: String): Unit = { + logInfo(s"Start building docker image $name from Dockerfile $dockerFile") dockerClient.build( DOCKER_BUILD_PATH, name, dockerFile, new LoggingBuildHandler()) + logInfo(s"Built $name docker image") } + }