Use a common configmap and remove duplicates (#43)

* Add krb5-server chart

* krb5-server runs

* Support multiple test cases

* Add cases

* Support skip cleaning up

* Kerberos test sets up config and keytabs

* Fix bugs

* Fix typo

* Enable kerberos for journal nodes

* Fix a bug

* Fix a bug

* Specify journalnode princiapls in the namenode config

* Test client use kerberos

* Use nn0 pod for hdfs admin commands

* Run test cases back to back

* Describe pod when tests fail

* Debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* Fix typo

* debug travis

* debug travis

* Do mount --make-share for travis

* Works end-to-end even in Travis

* Document the krb5-server chart

* Minor clean up

* Clean up

* Use -k option

* Add a chart for common configmap

* Remove vim file

* Switch charts to use the new configmap

* Fix a minor bug

* Fix a typo

* Fix a bug

* Disable service level RPC authorization

* Add group mapping for the hdfs user

* Remove redundant license notices

* CI clean up the hdfs-config chart

* Remove redundant zookeeper param in CI

* Test multiple datanode data dirs

* Test custom Hadoop config values in CI

* Update README.md's

* Cover non-HA in CI

* Address review comment

Author: kimoonkim

.travis.yml

@@ -3,6 +3,7 @@ sudo: required
 env:
   - CASES=_basic.sh
   - CASES=_kerberos.sh
+  - CASES=_single-namenode.sh
 
 before_script:
 # Required for K8s v1.10.x. See

charts/README.md

@@ -5,7 +5,8 @@ title: HDFS charts
 
 # HDFS charts
 Helm charts for launching HDFS daemons in a K8s cluster. Note that the HDFS
-charts are currently being heavily revised and are subject to change.
+charts are currently in pre-alpha quality. They are also being heavily revised
+and are subject to change.
 
 # Prerequisite
 
@@ -26,8 +27,8 @@ The HDFS daemons should be launched in the following order.
      [hdfs-datanode-k8s/README.md](hdfs-datanode-k8s/README.md)
      for how to launch.
 
-Kerberos is supported. See the `kerberosEnabled` option in the namenode and
-datanode charts.
+Kerberos is supported in the HA setup with the `kerberosEnabled` option.
+For more details, see the namenode `README.md`.
 
 There is also a HDFS client chart `hdfs-client` that can be convenient for
 testing.

charts/hdfs-client/templates/client-deployment.yaml

@@ -1,18 +1,3 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 apiVersion: apps/v1
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -34,47 +19,17 @@ spec:
         - name: hdfs-client
           image: uhopper/hadoop:2.7.2
           env:
-            # The following env vars are listed according to low-to-high precedence order.
-            # i.e. Whoever comes last will override the earlier value of the same variable.
-            {{- if .Values.kerberosEnabled }}
-            - name: CORE_CONF_hadoop_security_authentication
-              value: kerberos
-            - name: CORE_CONF_hadoop_security_authorization
-              value: "true"
-            - name: CORE_CONF_hadoop_rpc_protection
-              value: privacy
-            {{- end }}
-            {{- range $key, $value := .Values.customHadoopConfig }}
-            - name: {{ $key | quote }}
-              value: {{ $value | quote }}
-            {{- end }}
-            {{- if .Values.namenodeHAEnabled }}
-            - name: CORE_CONF_fs_defaultFS
-              value: hdfs://hdfs-k8s
-            - name: HDFS_CONF_dfs_nameservices
-              value: hdfs-k8s
-            - name: HDFS_CONF_dfs_ha_namenodes_hdfs___k8s
-              value: nn0,nn1
-            - name: HDFS_CONF_dfs_namenode_rpc___address_hdfs___k8s_nn0
-              value: hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:8020
-            - name: HDFS_CONF_dfs_namenode_rpc___address_hdfs___k8s_nn1
-              value: hdfs-namenode-1.hdfs-namenode.default.svc.cluster.local:8020
-            - name: HDFS_CONF_dfs_namenode_http___address_hdfs___k8s_nn0
-              value: hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:50070
-            - name: HDFS_CONF_dfs_namenode_http___address_hdfs___k8s_nn1
-              value: hdfs-namenode-1.hdfs-namenode.default.svc.cluster.local:50070
-            - name: HDFS_CONF_dfs_client_failover_proxy_provider_hdfs___k8s
-              value: org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
-            {{- else }}
-            - name: CORE_CONF_fs_defaultFS
-              value: hdfs://hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:8020
-            {{- end }}
+            - name: HADOOP_CUSTOM_CONF_DIR
+              value: /etc/hadoop-custom-conf
             - name: MULTIHOMED_NETWORK
               value: "0"
           command: ['/bin/sh', '-c']
           args:
             - /entrypoint.sh /usr/bin/tail -f /var/log/dmesg
           volumeMounts:
+            - name: hdfs-config
+              mountPath: /etc/hadoop-custom-conf
+              readOnly: true
             {{- if .Values.kerberosEnabled }}
             - name: kerberos-config
               mountPath: /etc/krb5.conf
@@ -83,6 +38,9 @@ spec:
             {{- end }}
       restartPolicy: Always
       volumes:
+        - name: hdfs-config
+          configMap:
+            name: {{ .Values.hdfsConfigMap }}
         {{- if .Values.kerberosEnabled }}
         - name: kerberos-config
           configMap:

charts/hdfs-client/values.yaml

@@ -15,30 +15,17 @@
 
 # Default values for template variables.
 
-# Custom hadoop config keys passed through env variables to hadoop uhopper images.
-# See https://hub.docker.com/r/uhopper/hadoop/ to get more details
-# Please note that these are not hadoop env variables, but docker env variables that
-# will be transformed into hadoop config keys
-# HDFS_CONF_dfs_datanode_data_dir and CORE_CONF_fs_defaultFS need special handling and
-# they're already set by the chart so any value coming from below config will be ignored
-customHadoopConfig: {}
-  # Set variables through a hash where env variable is the key, e.g.
-  # HDFS_CONF_dfs_datanode_use_datanode_hostname: "false"
+# Name of the k8s config map containing Hadoop config files such as core-site.xml
+# and hdfs-site.xml.
+hdfsConfigMap: hdfs-config
 
 # Whether or not Kerberos support is enabled.
 kerberosEnabled: false
 
-# Required to be non-empty if Kerberos is enabled. Specify your Kerberos realm name.
-# This should match the realm name in your Kerberos config file.
-kerberosRealm: ""
-
 # Effective only if Kerberos is enabled. Name of the k8s config map containing
 # the kerberos config file.
 kerberosConfigMap: kerberos-config
 
 # Effective only if Kerberos is enabled. Name of the kerberos config file inside
 # the config map.
 kerberosConfigFileName: krb5.conf
-
-# Whether or not to expect namenodes in the HA setup.
-namenodeHAEnabled: true

charts/hdfs-config-k8s/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/hdfs-config-k8s/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for configuring HDFS on Kubernetes
+name: hdfs-config-k8s
+version: 0.1.0

charts/hdfs-config-k8s/templates/_helpers.tpl

@@ -0,0 +1,64 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "hdfs-config-k8s.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "hdfs-config-k8s.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hdfs-config-k8s.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the kerberos principal suffix for core HDFS services
+*/}}
+{{- define "hdfs-principal" -}}
+{{- printf "hdfs/_HOST@%s" .Values.kerberosRealm -}}
+{{- end -}}
+
+{{/*
+Create the kerberos principal for HTTP services
+*/}}
+{{- define "http-principal" -}}
+{{- printf "HTTP/_HOST@%s" .Values.kerberosRealm -}}
+{{- end -}}
+
+{{/*
+Create the datanode data dir list.  The below uses two loops to make sure the
+last item does not have comma. It uses index 0 for the last item since that is
+the only special index that helm template gives us.
+*/}}
+{{- define "datanode-data-dirs" -}}
+{{- range $index, $path := .Values.dataNodeHostPath -}}
+  {{- if ne $index 0 -}}
+    /hadoop/dfs/data/{{ $index }},
+  {{- end -}}
+{{- end -}}
+{{- range $index, $path := .Values.dataNodeHostPath -}}
+  {{- if eq $index 0 -}}
+    /hadoop/dfs/data/{{ $index }}
+  {{- end -}}
+{{- end -}}
+{{- end -}}