Added a jupyter creator plugin

Author: jeclarke

api/src/main/resources/application_creator.py

@@ -31,6 +31,7 @@
 import uuid
 from importlib import import_module
 from exceptiondef import FailedValidation, FailedCreation
+from deployer_utils import HDFS
 
 
 class ApplicationCreator(object):
@@ -41,6 +42,9 @@ def __init__(self, config, environment, service):
         self._service = service
         self._component_creators = {}
         self._name_regex = re.compile('')
+        self._hdfs_client = HDFS(environment['webhdfs_host'],
+                                 environment['webhdfs_port'],
+                                 'hdfs')
 
     def create_application(self, package_data, package_metadata, application_name, property_overrides):
 
@@ -76,6 +80,11 @@ def destroy_application(self, application_name, application_create_data):
             creator = self._load_creator(component_type)
             creator.destroy_components(application_name, component_create_data)
 
+        local_path = '/opt/%s/%s/' % (self._service, application_name)
+        if os.path.isdir(local_path):
+            os.rmdir(local_path)
+        self._hdfs_client.remove('/user/%s' % application_name, recursive=False)
+
     def start_application(self, application_name, application_create_data):
 
         logging.debug("start_application: %s %s", application_name, application_create_data)

api/src/main/resources/plugins/base_creator.py

@@ -37,7 +37,6 @@
 import string
 import collections
 import subprocess
-import os
 import hbase_descriptor
 import opentsdb_descriptor
 from deployer_utils import HDFS
@@ -235,10 +234,6 @@ def destroy_components(self, application_name, create_data):
         for single_component_data in create_data:
             self._destroy_optional_descriptors(single_component_data['descriptors'])
             self.destroy_component(application_name, single_component_data)
-        local_path = '/opt/%s/%s/' % (self._namespace, application_name)
-        if os.path.isdir(local_path):
-            os.rmdir(local_path)
-        self._hdfs_client.remove('/user/%s' % application_name, recursive=False)
         return None
 
     def start_components(self, application_name, start_data):

api/src/main/resources/plugins/jupyter.py

@@ -0,0 +1,91 @@
+"""
+Name:       jupyter.py
+Purpose:    Installs a jupyter notebook
+Author:     PNDA team
+
+Created:    03/10/2016
+
+Copyright (c) 2016 Cisco and/or its affiliates.
+
+This software is licensed to you under the terms of the Apache License, Version 2.0 (the "License").
+You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+The code, technical concepts, and all information contained herein, are the property of Cisco Technology, Inc.
+and/or its affiliated entities, under various laws including copyright, international treaties, patent,
+and/or contract. Any use of the material herein must be in accordance with the terms of the License.
+All rights not expressly granted by the License are reserved.
+
+Unless required by applicable law or agreed to separately in writing, software distributed under the
+License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied.
+"""
+
+# pylint: disable=C0103
+
+import json
+import os
+import logging
+
+import deployer_utils
+from plugins.base_creator import Creator
+
+
+class JupyterCreator(Creator):
+
+    def validate_component(self, component):
+        errors = []
+        file_list = component['component_detail']
+        for file_name in file_list:
+            if file_name.endswith(r'.ipynb'):
+                notebook_found = True
+
+        if not notebook_found:
+            errors.append('missing ipynb file')
+
+        return errors
+
+    def get_component_type(self):
+        return 'jupyter'
+
+    def destroy_component(self, application_name, create_data):
+        logging.debug("destroy_component: %s %s", application_name, json.dumps(create_data))
+        key_file = self._environment['cluster_private_key']
+        root_user = self._environment['cluster_root_user']
+        target_host = self._environment['jupyter_host']
+        deployer_utils.exec_ssh(target_host, root_user, key_file, create_data['delete_commands'])
+
+    def start_component(self, application_name, create_data):
+        logging.debug("start_component (nothing to do for jupyter): %s %s", application_name, json.dumps(create_data))
+
+    def stop_component(self, application_name, create_data):
+        logging.debug("stop_component (nothing to do for jupyter): %s %s", application_name, json.dumps(create_data))
+
+    def create_component(self, staged_component_path, application_name, component, properties):
+        logging.debug("create_component: %s %s %s", application_name, json.dumps(component), properties)
+
+        key_file = self._environment['cluster_private_key']
+        root_user = self._environment['cluster_root_user']
+        target_host = self._environment['jupyter_host']
+        delete_commands = []
+
+        mkdircommands = []
+        remote_component_tmp_path = '%s/%s/%s' % ('/tmp/%s' % self._namespace, application_name, component['component_name'])
+        mkdircommands.append('mkdir -p %s' % remote_component_tmp_path)
+        deployer_utils.exec_ssh(target_host, root_user, key_file, mkdircommands)
+
+        file_list = component['component_detail']
+        for file_name in file_list:
+            if file_name.endswith(r'.ipynb'):
+                self._fill_properties('%s/%s' % (staged_component_path, file_name), properties)
+                logging.debug('Copying %s/* to %s:%s', staged_component_path, target_host, remote_component_tmp_path)
+                os.system("scp -i %s -o StrictHostKeyChecking=no %s/%s %s@%s:%s" %
+                          (key_file, staged_component_path, file_name, root_user, target_host, remote_component_tmp_path))
+
+                remote_component_install_path = '%s/%s_%s' % (self._environment['jupyter_notebook_directory'], application_name, file_name)
+                deployer_utils.exec_ssh(
+                    target_host, root_user, key_file,
+                    ['sudo mv %s %s' % (remote_component_tmp_path + '/*.ipynb', remote_component_install_path)])
+                delete_commands.append('sudo rm -rf %s\n' % remote_component_install_path)
+
+        logging.debug("uninstall commands: %s", delete_commands)
+        return {'delete_commands': delete_commands}

api/src/main/resources/test_application_creator.py

@@ -338,13 +338,15 @@ def test_invalid_package(self):
         creator.validate_package('test-package-1.0.2', {"package_name": "test-package-1.0.2", "component_types": {}})
 
     # pylint: disable=unused-argument
+    @patch('deployer_utils.HDFS')
     @patch('deployer_utils.exec_ssh')
     @patch('requests.put')
     @patch('os.path.isdir')
     @patch('os.rmdir')
-    def test_destroy_application(self, rmdir_mock, isdir_mock, put_mock, exec_ssh_mock):
+    def test_destroy_application(self, rmdir_mock, isdir_mock, put_mock, exec_ssh_mock, hdfs_client_mock):
         isdir_mock.return_value = True
         creator = ApplicationCreator(self.config, self.environment, self.service)
+        creator._hdfs_client = hdfs_client_mock
         creator.destroy_application('name', self.create_data)
         print exec_ssh_mock.call_args_list
         exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', [

api/src/main/resources/test_deployer_manager.py

@@ -584,7 +584,7 @@ def test_get_environment(self):
         repository = Mock()
         package_registrar = Mock()
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -602,7 +602,7 @@ def test_list_packages(self):
         package_registrar = Mock()
         package_registrar.list_packages.return_value = expected_packages
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -633,7 +633,7 @@ def test_list_repository(self):
 
         package_registrar = Mock()
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -650,7 +650,7 @@ def test_get_unknown_package_info(self):
         package_registrar.get_package_deploy_status.return_value = None
         package_registrar.package_exists.return_value = False
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -673,7 +673,7 @@ def test_list_package_applications(self):
         package_registrar = Mock()
         application_registrar = Mock()
         application_registrar.list_applications_for_package.return_value = expected_applications
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -691,7 +691,7 @@ def test_list_applications(self):
         package_registrar = Mock()
         application_registrar = Mock()
         application_registrar.list_applications.return_value = expected_applications
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -713,7 +713,7 @@ def test_application_in_progress(self):
             'package_name': 'package_name',
             'status': ApplicationState.STARTING,
             'information': None}
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -728,7 +728,7 @@ def test_package_in_progress(self):
         repository = Mock()
         package_registrar = Mock()
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         class DeploymentManagerTester(DeploymentManager):
@@ -750,7 +750,7 @@ def test_package_not_exists(self):
         package_registrar.get_package_deploy_status.return_value = None
         package_registrar.package_exists.return_value = False
         application_registrar = Mock()
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
         dmgr = DeploymentManager(repository,
                                  package_registrar,
@@ -775,7 +775,7 @@ def test_get_application_detail(self, app_mock):
             'package_name': 'package_name',
             'status': ApplicationState.STARTED,
             'information': None}
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,
@@ -801,7 +801,7 @@ def test_fail_get_detail(self, app_mock):
             'package_name': 'package_name',
             'status': ApplicationState.NOTCREATED,
             'information': None}
-        environment = {"namespace": "some_namespace"}
+        environment = {"namespace": "some_namespace", 'webhdfs_host': 'webhdfshost', 'webhdfs_port': 'webhdfsport'}
         config = {"deployer_thread_limit": 10}
 
         dmgr = DeploymentManager(repository,