Revert "Change to use a default application user instead of the hdfs user."

Author: jeclarke

CHANGELOG.md

@@ -3,7 +3,6 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 ### Added:
-- PNDA-3330: Change to use a default application user instead of the hdfs user.
 - PNDA-2445: Support for Hortonworks HDP
 
 ## [0.4.0] 2017-05-23

api/src/main/resources/plugins/base_creator.py

@@ -39,7 +39,6 @@
 import requests
 import hbase_descriptor
 import opentsdb_descriptor
-import getpass
 from deployer_utils import HDFS
 
 
@@ -167,7 +166,6 @@ def _instantiate_properties(self, application_name, component, property_override
         props['component_name'] = component['component_name']
         props['component_job_name'] = '%s-%s-job' % (props['component_application'], props['component_name'])
         props['component_hdfs_root'] = '/user/%s/%s' % (application_name, component['component_name'])
-        props['application_user'] = self._get_application_user()
         return props
 
     def _fill_properties(self, local_file, props):
@@ -329,10 +327,3 @@ def _find_yarn_app_info(self, all_yarn_applications, job_name):
                     if result is None or self._get_yarn_start_time(app) > self._get_yarn_start_time(result):
                         result = app
         return result
-
-    def _get_application_user(self):
-        application_user = getpass.getuser()
-        # if running as root, make sure to start the application under a different user. 
-        if application_user == 'root':
-            application_user = self._environment['application_default_user']
-        return application_user

api/src/main/resources/plugins/jupyter.py

@@ -67,14 +67,11 @@ def create_component(self, staged_component_path, application_name, component, p
         key_file = self._environment['cluster_private_key']
         root_user = self._environment['cluster_root_user']
         target_host = self._environment['jupyter_host']
-        application_user = properties['application_user']
         delete_commands = []
 
         mkdircommands = []
-        remote_component_tmp_path = '%s/%s/%s/%s' % ('/tmp/%s' % self._namespace, application_user, application_name, component['component_name'])
-        remote_notebook_path = '/home/%s/%s' % (application_user, self._environment['jupyter_notebook_directory'])
+        remote_component_tmp_path = '%s/%s/%s' % ('/tmp/%s' % self._namespace, application_name, component['component_name'])
         mkdircommands.append('mkdir -p %s' % remote_component_tmp_path)
-        mkdircommands.append('sudo -u %s mkdir -p %s' % (application_user ,remote_notebook_path))
         deployer_utils.exec_ssh(target_host, root_user, key_file, mkdircommands)
 
         file_list = component['component_detail']
@@ -85,7 +82,7 @@ def create_component(self, staged_component_path, application_name, component, p
                 os.system("scp -i %s -o StrictHostKeyChecking=no %s/%s %s@%s:%s" %
                           (key_file, staged_component_path, file_name, root_user, target_host, remote_component_tmp_path))
 
-                remote_component_install_path = '%s/%s_%s' % (remote_notebook_path, application_name, file_name)
+                remote_component_install_path = '%s/%s_%s' % (self._environment['jupyter_notebook_directory'], application_name, file_name)
                 deployer_utils.exec_ssh(
                     target_host, root_user, key_file,
                     ['sudo mv %s %s' % (remote_component_tmp_path + '/*.ipynb', remote_component_install_path)])

api/src/main/resources/plugins/oozie.py

@@ -52,19 +52,19 @@ def destroy_component(self, application_name, create_data):
             application_name,
             json.dumps(create_data))
         # terminate oozie jobs
-        self._kill_oozie(create_data['job_handle'], create_data['application_user'])
+        self._kill_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
 
         # delete component from hdfs
         remote_path = create_data['component_hdfs_root'][1:]
         self._hdfs_client.remove(remote_path, recursive=True)
 
     def start_component(self, application_name, create_data):
         logging.debug("start_component: %s %s", application_name, json.dumps(create_data))
-        self._start_oozie(create_data['job_handle'], create_data['application_user'])
+        self._start_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
 
     def stop_component(self, application_name, create_data):
         logging.debug("stop_component: %s %s", application_name, json.dumps(create_data))
-        self._stop_oozie(create_data['job_handle'], create_data['application_user'])
+        self._stop_oozie(create_data['job_handle'], self.OOZIE_USER_NAME)
 
     def create_component(self, staged_component_path, application_name, component, properties):
         logging.debug(
@@ -89,7 +89,7 @@ def create_component(self, staged_component_path, application_name, component, p
         properties['deployment_end'] = end.strftime("%Y-%m-%dT%H:%MZ")
 
         # insert required oozie properties
-        properties['user.name'] = properties['application_user']
+        properties['user.name'] = self.OOZIE_USER_NAME
         # Oozie ShareLib - supports actions
         properties['oozie.use.system.libpath'] = 'true'
         # platform shared libs e.g. hbase
@@ -104,15 +104,12 @@ def create_component(self, staged_component_path, application_name, component, p
         properties[def_path] = '%s/%s' % (self._environment['name_node'], remote_path)
 
         # deploy everything to various hadoop services
-        undeploy = self._deploy_to_hadoop(properties, staged_component_path, remote_path, properties['application_user'])
+        undeploy = self._deploy_to_hadoop(properties, staged_component_path, remote_path)
 
         # return something that can be used to undeploy later
-        return {'job_handle': undeploy['id'], 
-                'component_hdfs_root': properties['component_hdfs_root'],
-                'application_user': properties['application_user']
-        }
+        return {'job_handle': undeploy['id'], 'component_hdfs_root': properties['component_hdfs_root']}
 
-    def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, application_user, exclude=None):
+    def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, exclude=None):
         if exclude is None:
             exclude = []
         exclude.extend(['hdfs.json',
@@ -130,7 +127,7 @@ def _deploy_to_hadoop(self, properties, staged_component_path, remote_path, appl
 
         # submit to oozie
         result = self._submit_oozie(properties)
-        self._stop_oozie(result['id'], application_user)
+        self._stop_oozie(result['id'], self.OOZIE_USER_NAME)
 
         return result
 

api/src/main/resources/plugins/sparkStreaming.py

@@ -27,6 +27,7 @@
 import logging
 import platform
 from shutil import copy
+
 import deployer_utils
 from plugins.base_creator import Creator
 

api/src/main/resources/plugins/systemd.service.py.tpl

@@ -3,7 +3,7 @@ Description=PNDA Application: ${component_application}-${component_name}
 
 [Service]
 Type=simple
-User=${application_user}
+User=hdfs
 WorkingDirectory=/opt/${environment_namespace}/${component_application}/${component_name}/
 ExecStartPre=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 ExecStopPost=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py

api/src/main/resources/plugins/systemd.service.tpl

@@ -3,7 +3,7 @@ Description=PNDA Application: ${component_application}-${component_name}
 
 [Service]
 Type=simple
-User=${application_user}
+User=hdfs
 WorkingDirectory=/opt/${environment_namespace}/${component_application}/${component_name}/
 ExecStartPre=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 ExecStopPost=/opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py

api/src/main/resources/plugins/upstart.conf.py.tpl

@@ -5,4 +5,4 @@ respawn limit unlimited
 pre-start exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 pre-stop exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 chdir /opt/${environment_namespace}/${component_application}/${component_name}/
-exec sudo -u ${application_user} spark-submit --driver-java-options "-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties" --conf 'spark.executor.extraJavaOptions=-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties' --name '${component_job_name}' --master yarn-cluster --py-files application.properties,${component_py_files} ${component_spark_submit_args} ${component_main_py}
+exec sudo -u hdfs spark-submit --driver-java-options "-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties" --conf 'spark.executor.extraJavaOptions=-Dlog4j.configuration=file:///opt/${environment_namespace}/${component_application}/${component_name}/log4j.properties' --name '${component_job_name}' --master yarn-cluster --py-files application.properties,${component_py_files} ${component_spark_submit_args} ${component_main_py}

api/src/main/resources/plugins/upstart.conf.tpl

@@ -6,4 +6,4 @@ pre-start exec /opt/${environment_namespace}/${component_application}/${componen
 pre-stop exec /opt/${environment_namespace}/${component_application}/${component_name}/yarn-kill.py
 env programDir=/opt/${environment_namespace}/${component_application}/${component_name}/
 chdir /opt/${environment_namespace}/${component_application}/${component_name}/
-exec sudo -u ${application_user} spark-submit --driver-java-options "-Dlog4j.configuration=file:///${programDir}log4j.properties" --class ${component_main_class} --name '${component_job_name}' --master yarn-cluster --files log4j.properties ${component_spark_submit_args} ${component_main_jar}
+exec sudo -u hdfs spark-submit --driver-java-options "-Dlog4j.configuration=file:///${programDir}log4j.properties" --class ${component_main_class} --name '${component_job_name}' --master yarn-cluster --files log4j.properties ${component_spark_submit_args} ${component_main_jar}