pylint fixes

Fix up some pylint issues for pylint 1.8.3

Author: jeclarke

api/src/main/resources/app.py

@@ -22,14 +22,14 @@
 import sys
 import json
 import logging
-from tornado_cors import CorsMixin
 
 import tornado.httpserver
 import tornado.options
 import tornado.web
 from tornado.ioloop import IOLoop
 from tornado.web import asynchronous
 from tornado.options import define, options
+from tornado_cors import CorsMixin
 
 import package_registrar
 import application_registrar

api/src/main/resources/application_creator.py

@@ -87,7 +87,7 @@ def destroy_application(self, application_name, application_create_data):
         for component_type, component_create_data in application_create_data.iteritems():
             creator = self._load_creator(component_type)
             creator.destroy_components(application_name, component_create_data)
-            if len(component_create_data) > 0 and 'application_hdfs_root' in component_create_data[0]:
+            if component_create_data and 'application_hdfs_root' in component_create_data[0]:
                 app_hdfs_root = component_create_data[0]['application_hdfs_root']
 
         local_path = '/opt/%s/%s/' % (self._service, application_name)
@@ -122,10 +122,10 @@ def validate_package(self, package_name, package_metadata):
         for component_type, component_metadata in package_metadata['component_types'].iteritems():
             creator = self._load_creator(component_type)
             validation_errors = creator.validate_components(component_metadata)
-            if len(validation_errors) > 0:
+            if validation_errors:
                 result[component_type] = validation_errors
 
-        if len(result) > 0:
+        if result:
             raise FailedValidation(result)
 
     def _validate_name(self, package_name, package_metadata):

api/src/main/resources/application_registrar.py

@@ -72,7 +72,7 @@ def delete_application(self, application_name):
     def get_application(self, application_name):
         logging.debug("Reading %s", application_name)
         application_data = self._read_from_db(application_name)
-        if len(application_data) == 0:
+        if not application_data:
             return None
         return {'overrides': json.loads(application_data['cf:overrides']),
                 'defaults': json.loads(application_data['cf:defaults']),
@@ -84,7 +84,7 @@ def get_application(self, application_name):
     def application_exists(self, application_name):
         logging.debug("Checking %s", application_name)
         application_data = self._read_from_db(application_name)
-        if len(application_data) == 0:
+        if not application_data:
             return False
         # Note: this last line is problematic, as with the current API:
         # a "NOTCREATED" app can have an error state in the db

api/src/main/resources/application_summary.py

@@ -21,7 +21,6 @@
 _MAX_PROCESS_TIME = 6
 COMPONENT_STATUS = dict([("green", "OK"), ("amber", "WARN"), ("red", "ERROR")])
 
-# pylint: disable=R0204
 def spark_job_handler(app_id):
     """
     Find Job and Stage status of Spark Application
@@ -339,6 +338,7 @@ def oozie_application(job_handle):
         return oozie_coordinator_handler(oozie_info)
     if 'appName' in oozie_info:
         return oozie_workflow_handler(oozie_info)
+    return None
 
 def compare_yarn_start_time(app_info):
     """

api/src/main/resources/deployer_system_test.py

@@ -320,7 +320,7 @@ def test_get_application_detail(self, test_application):
             retries_performed += 1
             if retries_performed > 60:
                 raise Exception("Exceeded retry limit, Is something wrong?")
-            logging.debug("Retry #" + str(retries_performed))
+            logging.debug("Retry #%s", str(retries_performed))
 
     def test_create_application(self, test_package, test_application):
         logging.debug("Creating test application...")
@@ -382,7 +382,7 @@ def await_application_status(self, application_name, expected, legal_values):
             retries_performed += 1
             if retries_performed > 60:
                 raise Exception("Exceeded retry limit, Is something wrong?")
-            logging.debug("Retry #" + str(retries_performed))
+            logging.debug("Retry #%s", str(retries_performed))
 
     def await_package_status(self, package_name, expected, legal_values):
         """
@@ -399,7 +399,7 @@ def await_package_status(self, package_name, expected, legal_values):
             retries_performed += 1
             if retries_performed > 60:
                 raise Exception("Exceeded retry limit, Is something wrong?")
-            logging.debug("Retry #" + str(retries_performed))
+            logging.debug("Retry #%s", str(retries_performed))
 
     def find_a_test_package(self):
         """

api/src/main/resources/deployer_utils.py

@@ -28,10 +28,10 @@
 import time
 from threading import Thread
 
+import requests
 import spur
 from pywebhdfs.webhdfs import PyWebHdfsClient
 
-import requests
 from cm_api.api_client import ApiResource
 
 
@@ -115,7 +115,7 @@ def get_hdfs_hdp(ambari, cluster_name):
 def component_host(component_detail):
     host_list = ''
     for host_detail in component_detail['host_components']:
-        if len(host_list) > 0:
+        if host_list:
             host_list += ','
         host_list += host_detail['HostRoles']['host_name']
     return host_list
@@ -193,7 +193,7 @@ def fill_hadoop_env_cdh(env):
         cluster_name = cluster_detail.name
         break
 
-    logging.debug('getting ' + cluster_name)
+    logging.debug('getting %s', cluster_name)
     env['cm_status_links'] = {}
     env.pop('yarn_node_managers', None)
     env.pop('yarn_resource_manager_host', None)

api/src/main/resources/exceptiondef.py

@@ -65,5 +65,3 @@ class FailedConnection(DmException):
     def __init__(self, arg):
         super(FailedConnection, self).__init__(arg)
         self.msg = arg
-
-

api/src/main/resources/package_registrar.py

@@ -97,7 +97,7 @@ def delete_package(self, package_name):
     def get_package_data(self, package_name):
         logging.debug("Reading %s", package_name)
         record = self._read_from_db(package_name, ['cf:package_data'])
-        if len(record) == 0:
+        if not record:
             return None
         local_package_path = "%s/%s" % (self._package_local_dir_path, package_name)
         self._read_from_hdfs(record['cf:package_data'], local_package_path)
@@ -107,7 +107,7 @@ def get_package_metadata(self, package_name):
         logging.debug("Reading %s", package_name)
         package_data = self._read_from_db(
             package_name, ['cf:metadata', 'cf:name', 'cf:version'])
-        if len(package_data) == 0:
+        if not package_data:
             return None
         return {"metadata": json.loads(package_data["cf:metadata"]), "name": package_data[
             "cf:name"], "version": package_data["cf:version"]}
@@ -124,7 +124,7 @@ def get_package_deploy_status(self, package_name):
         """
         logging.debug("Checking %s", package_name)
         package_data = self._read_from_db(package_name, columns=[self.COLUMN_DEPLOY_STATUS])
-        if len(package_data) == 0:
+        if not package_data:
             return None
         # all status is stored as json, so parse it and return it
         deploy_status_as_string = package_data[self.COLUMN_DEPLOY_STATUS]
@@ -138,8 +138,8 @@ def list_packages(self):
             connection = happybase.Connection(self._hbase_host)
             table = connection.table(self._table_name)
             result = [key for key, _ in table.scan(columns=['cf:name'])]
-        except Exception as e:
-            logging.debug(str(e))
+        except Exception as exc:
+            logging.debug(str(exc))
             raise FailedConnection('Unable to connect to the HBase master')
         finally:
             if connection:

api/src/main/resources/package_repo_rest_client.py

@@ -20,10 +20,10 @@
 """
 import json
 import logging
-import requests
 import re
-from exceptiondef import FailedConnection
+import requests
 from requests.exceptions import RequestException
+from exceptiondef import FailedConnection
 
 
 class PackageRepoRestClient(object):
@@ -42,9 +42,9 @@ def put_package(self, package_name, package_data):
         :param package_data: The actual binary data of the package
         """
         url = self.api_url + "/packages/" + package_name
-        logging.debug("PUT: " + url)
+        logging.debug("PUT: %s", url)
         response = requests.put(url, data=package_data)
-        logging.debug("response code: " + str(response.status_code))
+        logging.debug("response code: %s", str(response.status_code))
         assert response.status_code == 200
 
     def get_package(self, package_name, expected_codes=None):
@@ -72,32 +72,32 @@ def get_package_list(self, recency=None):
         return json.loads(response.content)
 
     @staticmethod
-    def parse_error_msg_from_html_response(html_str):
+    def parse_error_msg_from_response(html_str):
         title_tag = re.search('<title>(.+?)<.*/title>', html_str)
         if title_tag:
-            cause_msg = re.sub('<[A-Za-z\/][^>]*>', '', title_tag.group())
+            cause_msg = re.sub(r'<[A-Za-z\/][^>]*>', '', title_tag.group())
             return cause_msg
         return html_str
 
     def make_rest_get_request(self, path, expected_codes=None):
         if not expected_codes:
             expected_codes = [200]
         url = self.api_url + path
-        logging.debug("GET: " + url)
+        logging.debug("GET: %s", url)
 
         try:
             response = requests.get(url, timeout=120)
-        except RequestException as e:
-            logging.debug("Request error: " + str(e))
+        except RequestException as exc:
+            logging.debug("Request error: %s", str(exc))
             error_msg = 'Unable to connect to the Package Repository Manager'
             raise FailedConnection(error_msg)
 
-        logging.debug("response code: " + str(response.status_code))
+        logging.debug("response code: %s", str(response.status_code))
 
         if response.status_code not in expected_codes:
-            error_msg = PackageRepoRestClient.parse_error_msg_from_html_response(response.text)
+            error_msg = PackageRepoRestClient.parse_error_msg_from_response(response.text)
             error_msg = "Package Repository Manager - {} (request path = {})".format(error_msg, path)
-            logging.debug("Server error: " + str(error_msg))
+            logging.debug("Server error: %s", str(error_msg))
         assert response.status_code in expected_codes, error_msg
 
         return response

api/src/main/resources/plugins/base_creator.py

@@ -273,7 +273,7 @@ def validate_components(self, components):
         result = {}
         for component_name, component in components.iteritems():
             validation_errors = self.validate_component(component)
-            if len(validation_errors) > 0:
+            if validation_errors:
                 result[component_name] = validation_errors
         return result