Skip to content

Commit 56a9a85

Browse files
authored
Merge pull request #9 from pndaproject/RELEASE-0.2.0
Release 0.2.0
2 parents 483c94b + 4d8a2ab commit 56a9a85

14 files changed

+164
-360
lines changed

CHANGELOG.md

Lines changed: 37 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,44 @@
1-
# Change Log
2-
All notable changes to this project will be documented in this file.
3-
4-
## [0.1.1] 2016-09-13
5-
### Changes
6-
- Improvements to documentation
7-
- Enhanced CI support
8-
9-
## [0.1.0] 2016-07-01
10-
### First version
11-
12-
## [Pre-release]
13-
14-
### Added
15-
16-
- Add hue endpoint to environment endpoints API
17-
- Application names checked to only contain alphanumeric characters (a-z A-Z 0-9 - and _) because they are used directly in file paths.
1+
# Change Log
2+
All notable changes to this project will be documented in this file.
3+
4+
## [0.2.0] 2016-10-21
5+
### Added
6+
- PNDA-2233 Jupyter notebook plugin added to deployment manager
7+
8+
## [0.1.1] 2016-09-13
9+
### Changes
10+
- Improvements to documentation
11+
- Enhanced CI support
12+
13+
## [0.1.0] 2016-07-01
14+
### First version
15+
16+
## [Pre-release]
17+
18+
### Added
19+
20+
- Add hue endpoint to environment endpoints API
21+
- Application names checked to only contain alphanumeric characters (a-z A-Z 0-9 - and _) because they are used directly in file paths.
1822
- Added ability to discover HDFS namedservices
1923
- Added information field to status reports
20-
- Using an external pacakge repository API instead of internal swift integration
21-
- Application detail API (GET /applications/<application>/detail) now returns YARN IDs assigned to the running tasks for that application.
24+
- Using an external pacakge repository API instead of internal swift integration
25+
- Application detail API (GET /applications/<application>/detail) now returns YARN IDs assigned to the running tasks for that application.
2226
- Oozie error messages are reported when querying for status of an application creation call.
2327
- Packages are validated on deployment and the error messages reported when querying for status of a package deployment call.
2428
- Added support for opentsdb.json descriptor for creating metrics when deploying applications.
2529
- Callback events are sent to the console data logger.
2630
- Application detail API now completed to return Yarn IDs for any Yarn applications associated with a PNDA application.
27-
28-
### Fixed
29-
30-
- Return IP address for webhdfs/HTTPFS endpoint instead of hostname
31-
- Timeout calls to package repository at 120 seconds.
32-
- Deploying a package that does not exist in the package repository now results in a useful error message being returned to the caller.
33-
- Fixed defect preventing '-' being used in application names.
34-
- Fix Zookeeper quorum bug issue
35-
- Improve package validation to catch packages without 3 point version numbers and where the folder inside the tar does not match the package name.
36-
- Add list of zookeeper nodes to quorum
37-
- Remove port=8020 for named service
38-
- Oozie creator plugin sets 'oozie.wf.application.path' and 'oozie.coord.application.path' to point at the folder not the xml files.
39-
- Removed some stdout printouts
40-
- Fixed bug preventing recency parameter being used on the repository/packages API.
31+
32+
### Fixed
33+
34+
- Return IP address for webhdfs/HTTPFS endpoint instead of hostname
35+
- Timeout calls to package repository at 120 seconds.
36+
- Deploying a package that does not exist in the package repository now results in a useful error message being returned to the caller.
37+
- Fixed defect preventing '-' being used in application names.
38+
- Fix Zookeeper quorum bug issue
39+
- Improve package validation to catch packages without 3 point version numbers and where the folder inside the tar does not match the package name.
40+
- Add list of zookeeper nodes to quorum
41+
- Remove port=8020 for named service
42+
- Oozie creator plugin sets 'oozie.wf.application.path' and 'oozie.coord.application.path' to point at the folder not the xml files.
43+
- Removed some stdout printouts
44+
- Fixed bug preventing recency parameter being used on the repository/packages API.

api/src/main/resources/app.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,14 @@
3131
from tornado.web import asynchronous
3232
from tornado.options import define, options
3333

34-
import repository
3534
import package_registrar
3635
import application_registrar
3736
import deployer_utils
3837
import deployment_manager
3938
from deployer_system_test import DeployerRestClientTester
4039
from exceptiondef import NotFound, ConflictingState, FailedValidation, FailedCreation
4140
from async_dispatcher import AsyncDispatcher
42-
from package_repo_rest_client import PacakgeRepoRestClient
41+
from package_repo_rest_client import PackageRepoRestClient
4342

4443
options.logging = None
4544

@@ -298,9 +297,8 @@ def main():
298297

299298
deployer_utils.fill_hadoop_env(config['environment'])
300299

301-
package_repository = PacakgeRepoRestClient(config['config']["package_repository"])
302-
repo = repository.SwiftRepository(package_repository)
303-
dm = deployment_manager.DeploymentManager(repo,
300+
package_repository = PackageRepoRestClient(config['config']["package_repository"])
301+
dm = deployment_manager.DeploymentManager(package_repository,
304302
package_registrar.HbasePackageRegistrar(
305303
config['environment']['hbase_rest_server']),
306304
application_registrar.HbaseApplicationRegistrar(

api/src/main/resources/application_creator.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import uuid
3232
from importlib import import_module
3333
from exceptiondef import FailedValidation, FailedCreation
34+
from deployer_utils import HDFS
3435

3536

3637
class ApplicationCreator(object):
@@ -41,6 +42,9 @@ def __init__(self, config, environment, service):
4142
self._service = service
4243
self._component_creators = {}
4344
self._name_regex = re.compile('')
45+
self._hdfs_client = HDFS(environment['webhdfs_host'],
46+
environment['webhdfs_port'],
47+
'hdfs')
4448

4549
def create_application(self, package_data, package_metadata, application_name, property_overrides):
4650

@@ -76,6 +80,11 @@ def destroy_application(self, application_name, application_create_data):
7680
creator = self._load_creator(component_type)
7781
creator.destroy_components(application_name, component_create_data)
7882

83+
local_path = '/opt/%s/%s/' % (self._service, application_name)
84+
if os.path.isdir(local_path):
85+
os.rmdir(local_path)
86+
self._hdfs_client.remove('/user/%s' % application_name, recursive=False)
87+
7988
def start_application(self, application_name, application_create_data):
8089

8190
logging.debug("start_application: %s %s", application_name, application_create_data)

api/src/main/resources/deployment_manager.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def _assert_package_status(self, package, required_status):
7676

7777
def list_repository(self, recency):
7878
logging.info("list_available: %s", recency)
79-
available = self._repository.list_packages(recency)
79+
available = self._repository.get_package_list(recency)
8080
return available
8181

8282
def get_package_info(self, package):
@@ -155,7 +155,7 @@ def _do_deploy():
155155
package_file = package + '.tar.gz'
156156
logging.info("deploy: %s", package)
157157
# download package:
158-
package_data = self._repository.download_package(package_file)
158+
package_data = self._repository.get_package(package_file)
159159
# put package in database:
160160
metadata = self._package_parser.get_package_metadata(package_data)
161161
self._application_creator.validate_package(package, metadata)

api/src/main/resources/dm-config.json

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
11
{
2-
"FsRepository": {
3-
"container": {
4-
"path": "/tmp/packages/"
5-
}
6-
},
72
"HDFSRegistrar": {
83
"records_path": "user/deployment/record.json",
94
"webhdfs_user": "hdfs"

api/src/main/resources/package_repo_rest_client.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from exceptiondef import NotFound
2525

2626

27-
class PacakgeRepoRestClient(object):
27+
class PackageRepoRestClient(object):
2828
def __init__(self, api_url):
2929
"""
3030
A client implementation for the package repository API

api/src/main/resources/plugins/base_creator.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import string
3838
import collections
3939
import subprocess
40-
import os
4140
import hbase_descriptor
4241
import opentsdb_descriptor
4342
from deployer_utils import HDFS
@@ -235,10 +234,6 @@ def destroy_components(self, application_name, create_data):
235234
for single_component_data in create_data:
236235
self._destroy_optional_descriptors(single_component_data['descriptors'])
237236
self.destroy_component(application_name, single_component_data)
238-
local_path = '/opt/%s/%s/' % (self._namespace, application_name)
239-
if os.path.isdir(local_path):
240-
os.rmdir(local_path)
241-
self._hdfs_client.remove('/user/%s' % application_name, recursive=False)
242237
return None
243238

244239
def start_components(self, application_name, start_data):
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
"""
2+
Name: jupyter.py
3+
Purpose: Installs a jupyter notebook
4+
Author: PNDA team
5+
6+
Created: 03/10/2016
7+
8+
Copyright (c) 2016 Cisco and/or its affiliates.
9+
10+
This software is licensed to you under the terms of the Apache License, Version 2.0 (the "License").
11+
You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
12+
13+
The code, technical concepts, and all information contained herein, are the property of Cisco Technology, Inc.
14+
and/or its affiliated entities, under various laws including copyright, international treaties, patent,
15+
and/or contract. Any use of the material herein must be in accordance with the terms of the License.
16+
All rights not expressly granted by the License are reserved.
17+
18+
Unless required by applicable law or agreed to separately in writing, software distributed under the
19+
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
20+
either express or implied.
21+
"""
22+
23+
# pylint: disable=C0103
24+
25+
import json
26+
import os
27+
import logging
28+
29+
import deployer_utils
30+
from plugins.base_creator import Creator
31+
32+
33+
class JupyterCreator(Creator):
34+
35+
def validate_component(self, component):
36+
errors = []
37+
notebook_found = False
38+
file_list = component['component_detail']
39+
for file_name in file_list:
40+
if file_name.endswith(r'.ipynb'):
41+
notebook_found = True
42+
43+
if notebook_found is False:
44+
errors.append('missing ipynb file')
45+
46+
return errors
47+
48+
def get_component_type(self):
49+
return 'jupyter'
50+
51+
def destroy_component(self, application_name, create_data):
52+
logging.debug("destroy_component: %s %s", application_name, json.dumps(create_data))
53+
key_file = self._environment['cluster_private_key']
54+
root_user = self._environment['cluster_root_user']
55+
target_host = self._environment['jupyter_host']
56+
deployer_utils.exec_ssh(target_host, root_user, key_file, create_data['delete_commands'])
57+
58+
def start_component(self, application_name, create_data):
59+
logging.debug("start_component (nothing to do for jupyter): %s %s", application_name, json.dumps(create_data))
60+
61+
def stop_component(self, application_name, create_data):
62+
logging.debug("stop_component (nothing to do for jupyter): %s %s", application_name, json.dumps(create_data))
63+
64+
def create_component(self, staged_component_path, application_name, component, properties):
65+
logging.debug("create_component: %s %s %s", application_name, json.dumps(component), properties)
66+
67+
key_file = self._environment['cluster_private_key']
68+
root_user = self._environment['cluster_root_user']
69+
target_host = self._environment['jupyter_host']
70+
delete_commands = []
71+
72+
mkdircommands = []
73+
remote_component_tmp_path = '%s/%s/%s' % ('/tmp/%s' % self._namespace, application_name, component['component_name'])
74+
mkdircommands.append('mkdir -p %s' % remote_component_tmp_path)
75+
deployer_utils.exec_ssh(target_host, root_user, key_file, mkdircommands)
76+
77+
file_list = component['component_detail']
78+
for file_name in file_list:
79+
if file_name.endswith(r'.ipynb'):
80+
self._fill_properties('%s/%s' % (staged_component_path, file_name), properties)
81+
logging.debug('Copying %s/* to %s:%s', staged_component_path, target_host, remote_component_tmp_path)
82+
os.system("scp -i %s -o StrictHostKeyChecking=no %s/%s %s@%s:%s" %
83+
(key_file, staged_component_path, file_name, root_user, target_host, remote_component_tmp_path))
84+
85+
remote_component_install_path = '%s/%s_%s' % (self._environment['jupyter_notebook_directory'], application_name, file_name)
86+
deployer_utils.exec_ssh(
87+
target_host, root_user, key_file,
88+
['sudo mv %s %s' % (remote_component_tmp_path + '/*.ipynb', remote_component_install_path)])
89+
delete_commands.append('sudo rm -rf %s\n' % remote_component_install_path)
90+
91+
logging.debug("uninstall commands: %s", delete_commands)
92+
return {'delete_commands': delete_commands}

api/src/main/resources/plugins/oozie.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def start_component(self, application_name, create_data):
6161
self._start_oozie(create_data['job_handle'])
6262

6363
def stop_component(self, application_name, create_data):
64-
logging.debug("start_component: %s %s", application_name, json.dumps(create_data))
64+
logging.debug("stop_component: %s %s", application_name, json.dumps(create_data))
6565
self._stop_oozie(create_data['job_handle'])
6666

6767
def create_component(self, staged_component_path, application_name, component, properties):

api/src/main/resources/plugins/sparkStreaming.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
2-
Name: oozie.py
2+
Name: sparkStreaming.py
33
Purpose: Submits spark streaming jobs based on application package
44
Author: PNDA team
55
@@ -57,7 +57,7 @@ def start_component(self, application_name, create_data):
5757
self._control_component(create_data['start_cmds'])
5858

5959
def stop_component(self, application_name, create_data):
60-
logging.debug("start_component: %s %s", application_name, json.dumps(create_data))
60+
logging.debug("stop_component: %s %s", application_name, json.dumps(create_data))
6161
self._control_component(create_data['stop_cmds'])
6262

6363
def _control_component(self, cmds):

0 commit comments

Comments
 (0)