Merge pull request #90 from Hitesh-Maplelabs/PNDA-4772

PNDA-4772: Fix to run the hive query

Author: jeclarke

api/src/main/resources/deployer_utils.py

@@ -180,7 +180,7 @@ def fill_hadoop_env_hdp(env):
 
             elif role_name == "HIVE_SERVER":
                 env['hive_server'] = '%s' % component_host(component_detail)
-                env['hive_port'] = '10000'
+                env['hive_port'] = '10001'
 
 def fill_hadoop_env_cdh(env):
     # pylint: disable=E1103

api/src/main/resources/hbase_descriptor.py

@@ -24,7 +24,7 @@
 import json
 import logging
 import starbase
-import pyhs2
+import subprocess
 
 
 def create(descriptor_path, environment):
@@ -35,20 +35,21 @@ def create(descriptor_path, environment):
     logging.debug("_deploy_hbase: %s", descriptor)
     hbase = starbase.Connection(host=environment['hbase_rest_server'], port=int(environment['hbase_rest_port']))
     hive_host = environment['hive_server']
-    hive_port = environment['hive_port']
-    hive = pyhs2.connect(
-        host=hive_host,
-        port=hive_port,
-        authMechanism="PLAIN",
-        user='hdfs',
-        password='test',
-        database='default')
-
+    hive_port = int(environment['hive_port'])
+    
     for element in descriptor:
         if 'table' in element and 'col_family' in element:
             table = hbase.table('%s' % element['table'])
             table.create(element['col_family'])
             for qry in element['hive_schema']:
-                hive.cursor().execute(qry)
+                beeline_output = run_hive_query(qry, hive_host, hive_port)
+                logging.info(beeline_output)
+
 
-    hive.close()
+def run_hive_query(query, hive_host, hive_port):
+    beeline_output = subprocess.check_output([
+    "beeline",
+    "-u", "jdbc:hive2://%s:%s/;transportMode=http;httpPath=cliservice" % (hive_host, hive_port),
+    "-e",
+    query])
+    return beeline_output

api/src/main/resources/test_hbase_descriptor.py

@@ -29,9 +29,7 @@
 
 class TestHbaseDescriptor(unittest.TestCase):
     @patch('starbase.Connection')
-    @patch('pyhs2.connect')
-    def test_normal_use(self, pyhs_mock, hbase_mock):
-        pyhs_mock.return_value = Mock()
+    def test_normal_use(self, hbase_mock):
 
         my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":[]}]'
         mocked_open_function = mock_open(read_data=my_text)
@@ -49,10 +47,11 @@ def test_normal_use(self, pyhs_mock, hbase_mock):
         hbase_mock.return_value.table.return_value.create.assert_called_once_with('a_cf')
 
     @patch('starbase.Connection')
-    @patch('pyhs2.connect')
-    def test_with_impala_schema(self, pyhs_mock, hbase_mock):
-        my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":["some ddl","more ddl"]}]'
+    @patch('subprocess.check_output')
+    def test_with_impala_schema(self, subprocess_mock, hbase_mock):
+        my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":["some ddl"]}]'
         mocked_open_function = mock_open(read_data=my_text)
+        subprocess_mock.return_value = 'something'
 
         with patch("__builtin__.open", mocked_open_function):
             environment = {
@@ -65,5 +64,4 @@ def test_with_impala_schema(self, pyhs_mock, hbase_mock):
         hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
         hbase_mock.return_value.table.assert_called_once_with('a_table')
         hbase_mock.return_value.table.return_value.create.assert_called_once_with('a_cf')
-        pyhs_mock.return_value.cursor.return_value.execute.assert_any_call('some ddl')
-        pyhs_mock.return_value.cursor.return_value.execute.assert_any_call('more ddl')
+        subprocess_mock.assert_called_once_with(["beeline", "-u", "jdbc:hive2://5.6.7.8:9876/;transportMode=http;httpPath=cliservice", "-e", "some ddl"])