working hadoop cluster

Author: ifilonenko

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/hadoopsteps/HadoopKerberosKeytabResolverStep.scala

@@ -59,7 +59,7 @@ private[spark] class HadoopKerberosKeytabResolverStep(
       }
     // In the case that keytab is not specified we will read from Local Ticket Cache
     val jobUserUGI = maybeJobUserUGI.getOrElse(UserGroupInformation.getCurrentUser)
-    logInfo("Primary group name: jobUserUGI.getPrimaryGroupName")
+    logInfo(s"Primary group name: ${jobUserUGI.getPrimaryGroupName}")
     val credentials: Credentials = jobUserUGI.getCredentials
     val credentialsManager = newHadoopTokenManager(submissionSparkConf, hadoopConf)
     var renewalTime = Long.MaxValue

resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/kerberos-test/Dockerfile

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM spark-base
+
+# If this docker file is being used in the context of building your images from a Spark distribution, the docker build
+# command should be invoked from the top level directory of the Spark distribution. E.g.:
+# docker build -t kerberos-test:latest -f dockerfiles/kerberos-test/Dockerfile .
+
+COPY examples /opt/spark/examples
+RUN apk add --no-cache --update krb5 krb5-libs
+COPY hconf/krb5.conf /etc/krb5.conf
+COPY test-scripts/test-env.sh /
+
+CMD /opt/spark/bin/spark-submit \
+      --deploy-mode cluster \
+      --class ${CLASS_NAME} \
+      --master k8s://${MASTER_URL} \
+      --kubernetes-namespace ${NAMESPACE} \
+      --conf spark.executor.instances=1 \
+      --conf spark.app.name=spark-hdfs \
+      --conf spark.kubernetes.driver.docker.image=spark-driver:latest \
+      --conf spark.kubernetes.executor.docker.image=spark-executor:latest \
+      --conf spark.kubernetes.initcontainer.docker.image=spark-init:latest \
+      --conf spark.kubernetes.kerberos=true \
+      --conf spark.kubernetes.kerberos.keytab=/var/keytabs/hdfs.keytab \
+      --conf spark.kubernetes.kerberos.principal=hdfs/nn.${NAMESPACE}[email protected] \
+      --conf spark.kubernetes.driver.labels=spark-app-locator=${APP_LOCATOR_LABEL} \
+      --files local:///etc/hadoop/core-site.xml,local:///etc/hadoop/hdfs-site.xml,local:///etc/hadoop/yarn-site.xml \
+      ${SUBMIT_RESOURCE} \
+      hdfs://nn.${NAMESPACE}.svc.cluster.local:9000/user/ifilonenko/wordcount.txt

resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/HDFSTest.scala

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.kubernetes.integrationtest.jobs
+
+import org.apache.spark.sql.SparkSession
+
+private[spark] object HDFSTest{
+
+  def main(args: Array[String]): Unit = {
+    // scalastyle:off println
+    if (args.length < 1) {
+      System.err.println("Usage: HdfsTest <file>")
+      System.exit(1)
+    }
+    // scalastyle:on println
+    val spark = SparkSession
+      .builder
+      .appName("HdfsTest")
+      .getOrCreate()
+    val file = spark.read.text(args(0)).rdd
+    val mapped = file.map(s => s.length).cache()
+    // scalastyle:off println
+    for (iter <- 1 to 10) {
+      val start = System.currentTimeMillis()
+      for (x <- mapped) { x + 2 }
+      val end = System.currentTimeMillis()
+      println("Iteration " + iter + " took " + (end-start) + " ms")
+    }
+    // scalastyle:on println
+    spark.stop()
+  }
+}

resource-managers/kubernetes/integration-tests/kerberos-yml/data-populator-deployment.yml

@@ -18,6 +18,7 @@ spec:
         name: data-populator
         image: ifilonenko/hadoop-base:latest
         imagePullPolicy: IfNotPresent
+        runAsNonRoot: false
         volumeMounts:
         - mountPath: /var/keytabs
           name: data-populator-keytab

resource-managers/kubernetes/integration-tests/kerberos-yml/dn1-deployment.yml

@@ -18,6 +18,7 @@ spec:
         name: dn1
         image: ifilonenko/hadoop-base:latest
         imagePullPolicy: IfNotPresent
+        runAsNonRoot: false
         volumeMounts:
         - mountPath: /var/keytabs
           name: dn1-keytab

resource-managers/kubernetes/integration-tests/kerberos-yml/kerberos-deployment.yml

@@ -18,6 +18,7 @@ spec:
         name: kerberos
         image: ifilonenko/hadoop-base:latest
         imagePullPolicy: IfNotPresent
+        runAsNonRoot: false
         volumeMounts:
         - mountPath: /var/keytabs
           name: kerb-keytab

resource-managers/kubernetes/integration-tests/kerberos-yml/kerberos-test.yml

@@ -0,0 +1,23 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kerberos-test
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: kerberos-test
+    spec:
+      containers:
+      - name: kerberos-test
+        image: kerberos-test:latest
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - mountPath: /var/keytabs
+          name: kerberos-test-keytab
+      restartPolicy: Always
+      volumes:
+      - name: kerberos-test-keytab
+        persistentVolumeClaim:
+          claimName: server-keytab

resource-managers/kubernetes/integration-tests/kerberos-yml/namenode-hadoop-pv.yml

@@ -4,6 +4,7 @@ metadata:
   name: nn-hadoop
   labels:
     type: local
+    job: kerberostest
 spec:
   capacity:
     storage: 10Gi

resource-managers/kubernetes/integration-tests/kerberos-yml/nn-deployment.yml

@@ -20,6 +20,7 @@ spec:
         - containerPort: 9000
         image: ifilonenko/hadoop-base:latest
         imagePullPolicy: IfNotPresent
+        runAsNonRoot: false
         volumeMounts:
         - mountPath: /var/keytabs
           name: nn-keytab

resource-managers/kubernetes/integration-tests/kerberos-yml/server-keytab-pv.yml

@@ -4,6 +4,7 @@ metadata:
   name: server-keytab
   labels:
     type: local
+    job: kerberostest
 spec:
   capacity:
     storage: 10Gi