Fix application deletion when app name is an HDFS username

When an application is deleted, the correct HDFS directory is deleted.
The final clean up of the application directory had not been updated
following a previous change to move the HDFS install directory under
/pnda/system.

PNDA-4128

Author: jeclarke

.gitignore

@@ -6,4 +6,5 @@ target/
 .settings/
 .coverage
 pnda-build/
-.virtualenv
+.virtualenv
+venv

api/src/main/resources/application_creator.py

@@ -84,14 +84,19 @@ def destroy_application(self, application_name, application_create_data):
 
         logging.debug("destroy_application: %s %s", application_name, application_create_data)
 
+        app_hdfs_root = None
         for component_type, component_create_data in application_create_data.iteritems():
             creator = self._load_creator(component_type)
             creator.destroy_components(application_name, component_create_data)
+            if len(component_create_data) > 0 and 'application_hdfs_root' in component_create_data[0]:
+                app_hdfs_root = component_create_data[0]['application_hdfs_root']
 
         local_path = '/opt/%s/%s/' % (self._service, application_name)
         if os.path.isdir(local_path):
             os.rmdir(local_path)
-        self._hdfs_client.remove('/user/%s' % application_name, recursive=False)
+
+        if app_hdfs_root is not None:
+            self._hdfs_client.remove(app_hdfs_root, recursive=False)
 
     def start_application(self, application_name, application_create_data):
 

api/src/main/resources/plugins/base_creator.py

@@ -166,7 +166,8 @@ def _instantiate_properties(self, application_name, user_name, component, proper
         props['component_application'] = application_name
         props['component_name'] = component['component_name']
         props['component_job_name'] = '%s-%s-job' % (props['component_application'], props['component_name'])
-        props['component_hdfs_root'] = '/pnda/system/deployment-manager/applications/%s/%s/%s' % (user_name, application_name, component['component_name'])
+        props['application_hdfs_root'] = '/pnda/system/deployment-manager/applications/%s/%s' % (user_name, application_name)
+        props['component_hdfs_root'] = '%s/%s' % (props['application_hdfs_root'], component['component_name'])
         props['application_user'] = user_name
         return props
 
@@ -245,6 +246,7 @@ def create_components(self, stage_path, application_name, user_name, components,
             self._auto_fill_app_properties(staged_component_path, merged_props)
             result = self.create_component(staged_component_path, application_name, user_name, component, merged_props)
             result['component_name'] = component_name
+            result['application_hdfs_root'] = merged_props['application_hdfs_root']
             result['component_job_name'] = merged_props['component_job_name']
             result['descriptors'] = descriptor_result
             results.append(result)

api/src/main/resources/test_application_creator.py

@@ -201,8 +201,8 @@ def json(self):
             creator.create_application('abcd', self.package_metadata, 'aname', self.property_overrides)
         print post_mock.call_args_list
         # pylint: disable=line-too-long
-        post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_queue_policy</name><value>echo dev</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_property3</name><value>3</value></property><property><name>component_property4</name><value>nine</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentA</value></property><property><name>component_job_name</name><value>aname-componentA-job</value></property><property><name>component_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname/componentA</value></property><property><name>application_user</name><value>root</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>root</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/pnda/deployment/platform</value></property><property><name>mapreduce.job.queuename</name><value>dev</value></property><property><name>oozie.wf.application.path</name><value>namenode/pnda/system/deployment-manager/applications/root/aname/componentA</value></property></configuration>', headers={'Content-Type': 'application/xml'})
-        post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_queue_policy</name><value>echo dev</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentB</value></property><property><name>component_job_name</name><value>aname-componentB-job</value></property><property><name>component_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname/componentB</value></property><property><name>application_user</name><value>root</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>root</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/pnda/deployment/platform</value></property><property><name>mapreduce.job.queuename</name><value>dev</value></property><property><name>oozie.wf.application.path</name><value>namenode/pnda/system/deployment-manager/applications/root/aname/componentB</value></property></configuration>', headers={'Content-Type': 'application/xml'})
+        post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_queue_policy</name><value>echo dev</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_property3</name><value>3</value></property><property><name>component_property4</name><value>nine</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentA</value></property><property><name>component_job_name</name><value>aname-componentA-job</value></property><property><name>application_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname</value></property><property><name>component_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname/componentA</value></property><property><name>application_user</name><value>root</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>root</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/pnda/deployment/platform</value></property><property><name>mapreduce.job.queuename</name><value>dev</value></property><property><name>oozie.wf.application.path</name><value>namenode/pnda/system/deployment-manager/applications/root/aname/componentA</value></property></configuration>', headers={'Content-Type': 'application/xml'})
+        post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_queue_policy</name><value>echo dev</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentB</value></property><property><name>component_job_name</name><value>aname-componentB-job</value></property><property><name>application_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname</value></property><property><name>component_hdfs_root</name><value>/pnda/system/deployment-manager/applications/root/aname/componentB</value></property><property><name>application_user</name><value>root</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>root</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/pnda/deployment/platform</value></property><property><name>mapreduce.job.queuename</name><value>dev</value></property><property><name>oozie.wf.application.path</name><value>namenode/pnda/system/deployment-manager/applications/root/aname/componentB</value></property></configuration>', headers={'Content-Type': 'application/xml'})
 
         put_mock.assert_any_call('oozie/v1/job/someid?action=suspend&user.name=root')