Merge pull request #34 from pndaproject/PNDA-3330

Change to use a default application user instead of the hdfs user.

Author: trsmith2

CHANGELOG.md

@@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 ### Added:
+- PNDA-3330: Change to use a default application user instead of the hdfs user.
 - PNDA-2445: Support for Hortonworks HDP
 
 ## [0.4.0] 2017-05-23

api/src/main/resources/plugins/base_creator.py

@@ -36,6 +36,7 @@
 import json
 import string
 import collections
+import getpass
 import requests
 import hbase_descriptor
 import opentsdb_descriptor
@@ -166,6 +167,7 @@ def _instantiate_properties(self, application_name, component, property_override
         props['component_name'] = component['component_name']
         props['component_job_name'] = '%s-%s-job' % (props['component_application'], props['component_name'])
         props['component_hdfs_root'] = '/user/%s/%s' % (application_name, component['component_name'])
+        props['application_user'] = self._get_application_user()
         return props
 
     def _fill_properties(self, local_file, props):
@@ -327,3 +329,10 @@ def _find_yarn_app_info(self, all_yarn_applications, job_name):
                     if result is None or self._get_yarn_start_time(app) > self._get_yarn_start_time(result):
                         result = app
         return result
+
+    def _get_application_user(self):
+        application_user = getpass.getuser()
+        # if running as root, make sure to start the application under a different user.
+        if application_user == 'root':
+            application_user = self._environment['application_default_user']
+        return application_user

api/src/main/resources/plugins/jupyter.py

@@ -67,11 +67,14 @@ def create_component(self, staged_component_path, application_name, component, p
         key_file = self._environment['cluster_private_key']
         root_user = self._environment['cluster_root_user']
         target_host = self._environment['jupyter_host']
+        application_user = properties['application_user']
         delete_commands = []
 
         mkdircommands = []
-        remote_component_tmp_path = '%s/%s/%s' % ('/tmp/%s' % self._namespace, application_name, component['component_name'])
+        remote_component_tmp_path = '%s/%s/%s/%s' % ('/tmp/%s' % self._namespace, application_user, application_name, component['component_name'])
+        remote_notebook_path = '/home/%s/%s' % (application_user, self._environment['jupyter_notebook_directory'])
         mkdircommands.append('mkdir -p %s' % remote_component_tmp_path)
+        mkdircommands.append('sudo -u %s mkdir -p %s' % (application_user, remote_notebook_path))
         deployer_utils.exec_ssh(target_host, root_user, key_file, mkdircommands)
 
         file_list = component['component_detail']
@@ -82,7 +85,7 @@ def create_component(self, staged_component_path, application_name, component, p
                 os.system("scp -i %s -o StrictHostKeyChecking=no %s/%s %s@%s:%s" %
                           (key_file, staged_component_path, file_name, root_user, target_host, remote_component_tmp_path))
 
-                remote_component_install_path = '%s/%s_%s' % (self._environment['jupyter_notebook_directory'], application_name, file_name)
+                remote_component_install_path = '%s/%s_%s' % (remote_notebook_path, application_name, file_name)
                 deployer_utils.exec_ssh(
                     target_host, root_user, key_file,
                     ['sudo mv %s %s' % (remote_component_tmp_path + '/*.ipynb', remote_component_install_path)])

api/src/main/resources/plugins/oozie.py

@@ -52,19 +52,19 @@ def destroy_component(self, application_name, create_data):
             application_name,
             json.dumps(create_data))
         # terminate oozie jobs
-        self._kill_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
+        self._kill_oozie(create_data['job_handle'], create_data['application_user'])
 
         # delete component from hdfs
         remote_path = create_data['component_hdfs_root'][1:]
         self._hdfs_client.remove(remote_path, recursive=True)
 
     def start_component(self, application_name, create_data):
         logging.debug("start_component: %s %s", application_name, json.dumps(create_data))
-        self._start_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
+        self._start_oozie(create_data['job_handle'], create_data['application_user'])
 
     def stop_component(self, application_name, create_data):
         logging.debug("stop_component: %s %s", application_name, json.dumps(create_data))
-        self._stop_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
+        self._stop_oozie(create_data['job_handle'], create_data['application_user'])
 
     def create_component(self, staged_component_path, application_name, component, properties):
         logging.debug(
@@ -89,7 +89,7 @@ def create_component(self, staged_component_path, application_name, component, p
         properties['deployment_end'] = end.strftime("%Y-%m-%dT%H:%MZ")
 
         # insert required oozie properties
-        properties['user.name'] = self.OOZIE_USER_NAME
+        properties['user.name'] = properties['application_user']
         # Oozie ShareLib - supports actions
         properties['oozie.use.system.libpath'] = 'true'
         # platform shared libs e.g. hbase
@@ -104,12 +104,14 @@ def create_component(self, staged_component_path, application_name, component, p
         properties[def_path] = '%s/%s' % (self._environment['name_node'], remote_path)
 
         # deploy everything to various hadoop services
-        undeploy = self._deploy_to_hadoop(properties, staged_component_path, remote_path)
+        undeploy = self._deploy_to_hadoop(properties, staged_component_path, remote_path, properties['application_user'])
 
         # return something that can be used to undeploy later
-        return {'job_handle': undeploy['id'], 'component_hdfs_root': properties['component_hdfs_root']}
+        return {'job_handle': undeploy['id'],
+                'component_hdfs_root': properties['component_hdfs_root'],
+                'application_user': properties['application_user']}
 
-    def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, exclude=None):
+    def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, application_user, exclude=None):
         if exclude is None:
             exclude = []
         exclude.extend(['hdfs.json',
@@ -127,7 +129,7 @@ def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, excl
 
         # submit to oozie
         result = self._submit_oozie(properties)
-        self._stop_oozie(result['id'], self.OOZIE_USER_NAME)
+        self._stop_oozie(result['id'], application_user)
 
         return result
 

api/src/main/resources/plugins/sparkStreaming.py

@@ -27,7 +27,6 @@
 import logging
 import platform
 from shutil import copy
-
 import deployer_utils
 from plugins.base_creator import Creator
 

api/src/main/resources/plugins/systemd.service.py.tpl

@@ -3,7 +3,7 @@ Description=PNDA Application: ${component_application}-${component_name}
 
 [Service]
 Type=simple
-User=hdfs
+User=${application_user}
 WorkingDirectory=/opt/${environment_namespace}/${component_application}/${component_name}/
 ExecStartPre=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 ExecStopPost=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py

api/src/main/resources/plugins/systemd.service.tpl

@@ -3,7 +3,7 @@ Description=PNDA Application: ${component_application}-${component_name}
 
 [Service]
 Type=simple
-User=hdfs
+User=${application_user}
 WorkingDirectory=/opt/${environment_namespace}/${component_application}/${component_name}/
 ExecStartPre=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 ExecStopPost=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py

api/src/main/resources/plugins/upstart.conf.py.tpl

@@ -5,4 +5,4 @@ respawn limit unlimited
 pre-start exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 pre-stop exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 chdir /opt/${environment_namespace}/${component_application}/${component_name}/
-exec sudo -u hdfs spark-submit --driver-java-options "-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties" --conf 'spark.executor.extraJavaOptions=-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties' --name '${component_job_name}' --master yarn-cluster --py-files application.properties,${component_py_files} ${component_spark_submit_args} ${component_main_py}
+exec sudo -u ${application_user} spark-submit --driver-java-options "-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties" --conf 'spark.executor.extraJavaOptions=-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties' --name '${component_job_name}' --master yarn-cluster --py-files application.properties,${component_py_files} ${component_spark_submit_args} ${component_main_py}

api/src/main/resources/plugins/upstart.conf.tpl

@@ -6,4 +6,4 @@ pre-start exec /opt/${environment_namespace}/${component_application}/${componen
 pre-stop exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 env programDir=/opt/${environment_namespace}/${component_application}/${component_name}/
 chdir /opt/${environment_namespace}/${component_application}/${component_name}/
-exec sudo -u hdfs spark-submit --driver-java-options "-Dlog4j.configuration=file:///${programDir}log4j.properties" --class ${component_main_class} --name '${component_job_name}' --master yarn-cluster --files log4j.properties ${component_spark_submit_args} ${component_main_jar}
+exec sudo -u ${application_user} spark-submit --driver-java-options "-Dlog4j.configuration=file:///${programDir}log4j.properties" --class ${component_main_class} --name '${component_job_name}' --master yarn-cluster --files log4j.properties ${component_spark_submit_args} ${component_main_jar}