Port to Python 3

Donald Hunter · Donald Hunter · commit 5fe290b5b168 · 2019-06-07T15:00:10.000+01:00
CDH support removed due to obsolete cm-api
diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,10 @@
 FROM centos:7.6.1810
 
-RUN yum install -y python-devel epel-release
-RUN yum install -y python2-pip cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-plain libffi-devel openssl-devel gcc gcc-c++
+RUN yum install -y epel-release
+RUN yum install -y python36-devel python36-pip cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-plain libffi-devel openssl-devel gcc gcc-c++
 
 COPY api/src/main/resources /deployment-manager/
 
 WORKDIR /deployment-manager
 
-RUN pip install -r requirements.txt
+RUN pip3 install -r requirements.txt
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
-NAME=deployment-manager
-VERSION=0.1.3
-REGISTRY=containers.cisco.com/donaldh
+NAME=platform-deployment-manager
+VERSION=release5.0
+REGISTRY=pnda
 
 build:	## Build the docker image
 	docker build -t "$(NAME):$(VERSION)" -f Dockerfile .
diff --git a/api/src/main/resources/authorizer_local.py b/api/src/main/resources/authorizer_local.py
@@ -57,7 +57,7 @@ def authorize(self, identity, resource, action):
                     authorize = True
                     logging.debug("authorize: %s for %s", authorize, grant_rule)
                     break
-            except KeyError, ex:
+            except KeyError as ex:
                 logging.warning("missing attribute %s", ex)
 
         if not authorize:
diff --git a/api/src/main/resources/deployer_utils.py b/api/src/main/resources/deployer_utils.py
@@ -22,7 +22,7 @@
 
 import os
 import tarfile
-import StringIO
+from io import StringIO
 import logging
 import traceback
 import time
@@ -32,18 +32,6 @@
 import spur
 from pywebhdfs.webhdfs import PyWebHdfsClient
 
-from cm_api.api_client import ApiResource
-
-
-def connect_cm(cm_api, cm_username, cm_password):
-    api = ApiResource(
-        cm_api,
-        version=6,
-        username=cm_username,
-        password=cm_password)
-    return api
-
-
 def get_nameservice(cm_host, cluster_name, service_name, user_name='admin', password='admin'):
     request_url = 'http://%s:7180/api/v11/clusters/%s/services/%s/nameservices' % (cm_host,
                                                                                    cluster_name,
@@ -66,7 +54,7 @@ def update_hadoop_env(env):
     tmp_env = dict(env)
     logging.debug('Updating environment descriptor')
     if env['hadoop_distro'] == 'CDH':
-        fill_hadoop_env_cdh(tmp_env)
+        logging.error('CDH is not a supported hadoop distribution')
     elif env['hadoop_distro'] == 'HDP':
         fill_hadoop_env_hdp(tmp_env)
     else:
@@ -184,105 +172,6 @@ def fill_hadoop_env_hdp(env):
                 env['hive_server'] = '%s' % component_host(component_detail)
                 env['hive_port'] = '10001'
 
-def fill_hadoop_env_cdh(env):
-    # pylint: disable=E1103
-    api = connect_cm(
-        env['hadoop_manager_host'],
-        env['hadoop_manager_username'],
-        env['hadoop_manager_password'])
-
-    for cluster_detail in api.get_all_clusters():
-        cluster_name = cluster_detail.name
-        break
-
-    logging.debug('getting %s', cluster_name)
-    env['cm_status_links'] = {}
-    env.pop('yarn_node_managers', None)
-    env.pop('yarn_resource_manager_host', None)
-    env.pop('zookeeper_quorum', None)
-
-    cluster = api.get_cluster(cluster_name)
-    for service in cluster.get_all_services():
-        env['cm_status_links']['%s' % service.name] = service.serviceUrl
-        if service.type == "HDFS":
-            nameservice = get_nameservice(env['hadoop_manager_host'], cluster_name,
-                                          service.name,
-                                          user_name=env['hadoop_manager_username'],
-                                          password=env['hadoop_manager_password'])
-            if nameservice:
-                env['name_node'] = 'hdfs://%s' % nameservice
-            for role in service.get_all_roles():
-                if not nameservice and role.type == "NAMENODE":
-                    env['name_node'] = 'hdfs://%s:8020' % api.get_host(role.hostRef.hostId).hostname
-                if role.type == "HTTPFS":
-                    env['webhdfs_host'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['webhdfs_port'] = '14000'
-        elif service.type == "YARN":
-            for role in service.get_all_roles():
-                if role.type == "RESOURCEMANAGER":
-                    if 'yarn_resource_manager_host' in env:
-                        rm_instance = '_backup'
-                    else:
-                        rm_instance = ''
-                    env['yarn_resource_manager_host%s' % rm_instance] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['yarn_resource_manager_port%s' % rm_instance] = '8088'
-                    env['yarn_resource_manager_mr_port%s' % rm_instance] = '8032'
-                if role.type == "NODEMANAGER":
-                    if 'yarn_node_managers' in env:
-                        env['yarn_node_managers'] = '%s,%s' % (env['yarn_node_managers'], api.get_host(role.hostRef.hostId).hostname)
-                    else:
-                        env['yarn_node_managers'] = '%s' % api.get_host(
-                            role.hostRef.hostId).hostname
-        elif service.type == "MAPREDUCE":
-            for role in service.get_all_roles():
-                if role.type == "JOBTRACKER":
-                    env['job_tracker'] = '%s:8021' % api.get_host(role.hostRef.hostId).hostname
-                    break
-        elif service.type == "ZOOKEEPER":
-            for role in service.get_all_roles():
-                if role.type == "SERVER":
-                    if 'zookeeper_quorum' in env:
-                        env['zookeeper_quorum'] += ',%s' % api.get_host(role.hostRef.hostId).hostname
-                    else:
-                        env['zookeeper_quorum'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                        env['zookeeper_port'] = '2181'
-        elif service.type == "HBASE":
-            for role in service.get_all_roles():
-                if role.type == "HBASERESTSERVER":
-                    env['hbase_rest_server'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['hbase_rest_port'] = '20550'
-                elif role.type == "HBASETHRIFTSERVER":
-                    env['hbase_thrift_server'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-        elif service.type == "OOZIE":
-            for role in service.get_all_roles():
-                if role.type == "OOZIE_SERVER":
-                    env['oozie_uri'] = 'http://%s:11000/oozie' % api.get_host(role.hostRef.hostId).hostname
-                    break
-        elif service.type == "HIVE":
-            for role in service.get_all_roles():
-                if role.type == "HIVESERVER2":
-                    env['hive_server'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['hive_port'] = '10000'
-                    break
-        elif service.type == "IMPALA":
-            for role in service.get_all_roles():
-                if role.type == "IMPALAD":
-                    env['impala_host'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['impala_port'] = '21050'
-                    break
-        elif service.type == "KUDU":
-            for role in service.get_all_roles():
-                if role.type == "KUDU_MASTER":
-                    env['kudu_host'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['kudu_port'] = '7051'
-                    break
-        elif service.type == "HUE":
-            for role in service.get_all_roles():
-                if role.type == "HUE_SERVER":
-                    env['hue_host'] = '%s' % api.get_host(role.hostRef.hostId).hostname
-                    env['hue_port'] = '8888'
-                    break
-
 def tree(archive_filepath):
     file_handle = file(archive_filepath, 'rb')
     tar_file = tarfile.open(None, 'r', file_handle)
@@ -299,7 +188,6 @@ def tree(archive_filepath):
 
     return root
 
-
 def canonicalize(path):
     path = path.replace('\\', '/')
     path = path.replace('//', '/')
diff --git a/api/src/main/resources/deployment_manager.py b/api/src/main/resources/deployment_manager.py
@@ -131,7 +131,7 @@ def _get_saved_package_data(self, package):
         package_metadata = None
         if self._package_registrar.package_exists(package):
             package_metadata = self._package_registrar.get_package_metadata(package)
-            print package_metadata
+            logging.debug(package_metadata)
             package_owner = package_metadata['metadata']['user']
             package_exists = True
         return package_owner, package_exists, package_metadata
diff --git a/api/src/main/resources/requirements.txt b/api/src/main/resources/requirements.txt
@@ -1,9 +1,6 @@
 argparse==1.2.1
-backports-abc==0.5
-backports.ssl-match-hostname==3.5.0.1
 certifi==2017.4.17
 cffi==1.9.1
-cm-api==14.0.0
 cryptography==1.7.1
 enum34==1.1.6
 happybase==1.0.0
@@ -27,4 +24,3 @@ thrift==0.9.3
 thriftpy==0.3.9
 tornado==4.4.2
 tornado-cors==0.6.0
-wsgiref==0.1.2
