Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.

Commit 50c8fbf

Browse files
ifilonenkoifilonenko
committed
Added current, BROKEN, integration test environment for review
1 parent 3f1c567 commit 50c8fbf

File tree

26 files changed

+816
-199
lines changed

26 files changed

+816
-199
lines changed

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/hadoopsteps/HadoopKerberosKeytabResolverStep.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ private[spark] class HadoopKerberosKeytabResolverStep(
4141
override def configureContainers(hadoopConfigSpec: HadoopConfigSpec): HadoopConfigSpec = {
4242
// FIXME: Pass down hadoopConf so you can call sc.hadoopConfiguration
4343
val hadoopConf = SparkHadoopUtil.get.newConfiguration(submissionSparkConf)
44+
logInfo(s"Hadoop Configuration: ${hadoopConf.toString}")
4445
if (!UserGroupInformation.isSecurityEnabled) logError("Hadoop not configuration with Kerberos")
4546
val maybeJobUserUGI =
4647
for {
@@ -58,6 +59,7 @@ private[spark] class HadoopKerberosKeytabResolverStep(
5859
}
5960
// In the case that keytab is not specified we will read from Local Ticket Cache
6061
val jobUserUGI = maybeJobUserUGI.getOrElse(UserGroupInformation.getCurrentUser)
62+
logInfo("Primary group name: jobUserUGI.getPrimaryGroupName")
6163
val credentials: Credentials = jobUserUGI.getCredentials
6264
val credentialsManager = newHadoopTokenManager(submissionSparkConf, hadoopConf)
6365
var renewalTime = Long.MaxValue

‎resource-managers/kubernetes/integration-tests/kerberos-yml/data-populator-deployment.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: data-populator
5+
spec:
6+
replicas: 1
7+
template:
8+
metadata:
9+
annotations:
10+
pod.beta.kubernetes.io/hostname: data-populator
11+
labels:
12+
name: hdfs-data-populator
13+
kerberosService: data-populator
14+
spec:
15+
containers:
16+
- command:
17+
- /populate-data.sh
18+
name: data-populator
19+
image: ifilonenko/hadoop-base:latest
20+
imagePullPolicy: IfNotPresent
21+
volumeMounts:
22+
- mountPath: /var/keytabs
23+
name: data-populator-keytab
24+
restartPolicy: Always
25+
volumes:
26+
- name: data-populator-keytab
27+
persistentVolumeClaim:
28+
claimName: server-keytab

‎resource-managers/kubernetes/integration-tests/kerberos-yml/data-populator-service.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
annotations:
5+
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
6+
labels:
7+
kerberosService: data-populator
8+
name: data-populator
9+
spec:
10+
clusterIP: None
11+
ports:
12+
- protocol: TCP
13+
port: 55555
14+
targetPort: 0
15+
selector:
16+
kerberosService: data-populator

‎resource-managers/kubernetes/integration-tests/kerberos-yml/dn1-deployment.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: dn1
5+
spec:
6+
replicas: 1
7+
template:
8+
metadata:
9+
annotations:
10+
pod.beta.kubernetes.io/hostname: dn1
11+
labels:
12+
name: hdfs-dn1
13+
kerberosService: dn1
14+
spec:
15+
containers:
16+
- command:
17+
- /start-datanode.sh
18+
name: dn1
19+
image: ifilonenko/hadoop-base:latest
20+
imagePullPolicy: IfNotPresent
21+
volumeMounts:
22+
- mountPath: /var/keytabs
23+
name: dn1-keytab
24+
restartPolicy: Always
25+
volumes:
26+
- name: dn1-keytab
27+
persistentVolumeClaim:
28+
claimName: server-keytab

‎resource-managers/kubernetes/integration-tests/kerberos-yml/dn1-service.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
annotations:
5+
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
6+
labels:
7+
kerberosService: dn1
8+
name: dn1
9+
spec:
10+
clusterIP: None
11+
ports:
12+
- protocol: TCP
13+
port: 55555
14+
targetPort: 0
15+
selector:
16+
kerberosService: dn1

‎resource-managers/kubernetes/integration-tests/kerberos-yml/kerberos-deployment.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: kerberos
5+
spec:
6+
replicas: 1
7+
template:
8+
metadata:
9+
annotations:
10+
pod.beta.kubernetes.io/hostname: kerberos
11+
labels:
12+
name: hdfs-kerberos
13+
kerberosService: kerberos
14+
spec:
15+
containers:
16+
- command:
17+
- /start-kdc.sh
18+
name: kerberos
19+
image: ifilonenko/hadoop-base:latest
20+
imagePullPolicy: IfNotPresent
21+
volumeMounts:
22+
- mountPath: /var/keytabs
23+
name: kerb-keytab
24+
restartPolicy: Always
25+
volumes:
26+
- name: kerb-keytab
27+
persistentVolumeClaim:
28+
claimName: server-keytab

‎resource-managers/kubernetes/integration-tests/kerberos-yml/kerberos-service.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
annotations:
5+
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
6+
labels:
7+
kerberosService: kerberos
8+
name: kerberos
9+
spec:
10+
clusterIP: None
11+
ports:
12+
- protocol: TCP
13+
port: 55555
14+
targetPort: 0
15+
selector:
16+
kerberosService: kerberos

‎resource-managers/kubernetes/integration-tests/kerberos-yml/namenode-hadoop-pv.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
kind: PersistentVolume
2+
apiVersion: v1
3+
metadata:
4+
name: nn-hadoop
5+
labels:
6+
type: local
7+
spec:
8+
capacity:
9+
storage: 10Gi
10+
accessModes:
11+
- ReadWriteOnce
12+
hostPath:
13+
path: "/tmp/nn"

‎resource-managers/kubernetes/integration-tests/kerberos-yml/namenode-hadoop.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: PersistentVolumeClaim
3+
metadata:
4+
name: nn-hadoop
5+
spec:
6+
accessModes:
7+
- ReadWriteOnce
8+
resources:
9+
requests:
10+
storage: 100Mi

‎resource-managers/kubernetes/integration-tests/kerberos-yml/nn-deployment.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: nn
5+
spec:
6+
replicas: 1
7+
template:
8+
metadata:
9+
annotations:
10+
pod.beta.kubernetes.io/hostname: nn
11+
labels:
12+
name: hdfs-nn
13+
kerberosService: nn
14+
spec:
15+
containers:
16+
- command:
17+
- /start-namenode.sh
18+
name: nn
19+
ports:
20+
- containerPort: 9000
21+
image: ifilonenko/hadoop-base:latest
22+
imagePullPolicy: IfNotPresent
23+
volumeMounts:
24+
- mountPath: /var/keytabs
25+
name: nn-keytab
26+
restartPolicy: Always
27+
volumes:
28+
- name: nn-keytab
29+
persistentVolumeClaim:
30+
claimName: server-keytab
31+
- name: nn-hadoop
32+
persistentVolumeClaim:
33+
claimName: nn-hadoop

0 commit comments

Comments
 (0)