Fixed tests and volume api

Author: tonyl

Makefile

@@ -58,7 +58,7 @@ test_collection_cleanup:
 	docker cp ${EVENTGEN_TEST_IMAGE}:$(shell pwd)/tests/functional_orca_test.log functional_orca_test.log || echo "no functional_orca_test.log"
 
 	@echo 'Stopping test container'
-	docker stop ${EVENTGEN_TEST_IMAGE}
+	docker stop ${EVENTGEN_TEST_IMAGE} || true
 
 clean:
 	rm *.spl || true
@@ -69,6 +69,7 @@ clean:
 	rm -rf docs/_book || true
 	rm -rf node_modules || true
 	rm -rf docs/node_modules || true
+	rm splunk_eventgen/default/eventgen_wsgi.conf || true
 	find . -name "*.pyc" -type f -delete || true
 	find . -name "*.log" -type f -delete || true
 	find . -name "*.pyc" -type f -delete || true

splunk_eventgen/eventgen_core.py

@@ -235,7 +235,7 @@ def _create_generator_workers(self, workercount=20):
     def _setup_loggers(self, args=None, config=None):
         log_path = getattr(args, "log_path", os.path.join(file_path, 'logs'))
         eventgen_main_logger_path = os.path.join(log_path, 'eventgen-main.log')
-        eventgen_listener_logger_path = os.path.join(log_path, 'eventgen-listener-process.log')
+        eventgen_controller_logger_path = os.path.join(log_path, 'eventgen-controller.log')
         eventgen_metrics_logger_path = os.path.join(log_path, 'eventgen-metrics.log')
         eventgen_error_logger_path = os.path.join(log_path, 'eventgen-errors.log')
         eventgen_server_logger_path = os.path.join(log_path, 'eventgen-server.log')
@@ -256,9 +256,9 @@ def _setup_loggers(self, args=None, config=None):
             file_handler.setFormatter(detailed_formatter)
             file_handler.setLevel(logging.DEBUG)
 
-            eventgen_listener_file_handler = logging.handlers.RotatingFileHandler(eventgen_listener_logger_path, maxBytes=2500000, backupCount=20)
-            eventgen_listener_file_handler.setFormatter(detailed_formatter)
-            eventgen_listener_file_handler.setLevel(logging.DEBUG)
+            eventgen_controller_file_handler = logging.handlers.RotatingFileHandler(eventgen_controller_logger_path, maxBytes=2500000, backupCount=20)
+            eventgen_controller_file_handler.setFormatter(detailed_formatter)
+            eventgen_controller_file_handler.setLevel(logging.DEBUG)
 
             error_file_handler = logging.handlers.RotatingFileHandler(eventgen_error_logger_path, maxBytes=2500000, backupCount=20)
             error_file_handler.setFormatter(detailed_formatter)
@@ -283,12 +283,13 @@ def _setup_loggers(self, args=None, config=None):
             logger.addHandler(error_file_handler)
 
             # Configure eventgen listener
-            logger = logging.getLogger('eventgen_listener')
+            logger = logging.getLogger('eventgen_controller')
             logger.setLevel(self.args.verbosity or logging.ERROR)
             logger.propagate = False
             logger.handlers = []
-            logger.addHandler(eventgen_listener_file_handler)
+            logger.addHandler(eventgen_controller_file_handler)
             logger.addHandler(error_file_handler)
+            logger.addHandler(console_handler)
 
             # Configure eventgen mertics logger
             logger = logging.getLogger('eventgen_metrics')
@@ -303,6 +304,7 @@ def _setup_loggers(self, args=None, config=None):
             logger.propagate = False
             logger.handlers = []
             logger.addHandler(server_file_handler)
+            logger.addHandler(console_handler)
         else:
             self.logger_config = config
             logging.config.dictConfig(self.logger_config)

splunk_eventgen/eventgen_nameko_controller.py

@@ -456,7 +456,7 @@ def receive_conf(self, data):
             self.server_confs[data['server_name']] = data['server_conf']
 
     def receive_volume(self, data):
-        if data['server_name'] and data["total_volume"]:
+        if data['server_name'] and "total_volume" in data:
             self.server_volumes[data['server_name']] = data['total_volume']
 
     def process_server_status(self, current_time):

splunk_eventgen/eventgen_nameko_server.py

@@ -418,7 +418,9 @@ def get_volume(self):
         try:
             config = json.loads(self.get_conf())
             self.log.info(config)
-            self.total_volume = float(self.get_data_volumes(config))
+            fetched_volume = float(self.get_data_volumes(config))
+            if fetched_volume:
+                self.total_volume = fetched_volume
             self.send_volume_to_controller(total_volume=self.total_volume)
             return str(self.total_volume)
         except Exception as e:
@@ -450,6 +452,8 @@ def set_volume(self, volume):
                     if "perDayVolume" in config[stanza].keys():
                         divided_value = float(config[stanza]["perDayVolume"]) * ratio
                     else:
+                        if not num_stanzas:
+                            num_stanzas = 1
                         divided_value = float(volume) / float(num_stanzas)
                     update_json[stanza] = {"perDayVolume": divided_value}
             output = self.edit_conf(json.dumps(update_json))

tests/large/test_eventgen_orchestration.py

@@ -113,19 +113,15 @@ def test_controller_index(self):
 		assert "You are running Eventgen Controller" in r.content
 
 	def test_controller_status(self):
-		time.sleep(10)
 		max_retry = 5
 		current_retry = 1
 		output = {}
 		while not output and current_retry <= max_retry:
-			response = requests.get("http://127.0.0.1:{}/status".format(self.controller_eventgen_webport))
+			response = requests.get("http://127.0.0.1:{}/status".format(self.controller_eventgen_webport), timeout=10)
 			if response.status_code == 200:
-				print response.status_code
-				print response.content
 				output = json.loads(response.content)
 			current_retry += 1
-			time.sleep(2)
-		print output
+			time.sleep(10)
 		assert output
 
 	def test_controller_start(self):
@@ -174,10 +170,16 @@ def test_controller_bundle_with_url_and_target(self):
 		assert "Bundle event dispatched to {} with url http://server.com/bundle.tgz".format(TestEventgenOrchestration.server_id[:12]) in r.content
 
 	def test_controller_get_volume(self):
-		r = requests.get("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport))
-		assert r.status_code == 200
-		output = json.loads(r.content)
-		assert output[TestEventgenOrchestration.server_id[:12]] == {}
+		max_retry = 5
+		current_retry = 1
+		output = {}
+		while not output and current_retry <= max_retry:
+			response = requests.get("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport), timeout=10)
+			if response.status_code == 200:
+				output = json.loads(response.content)
+			current_retry += 1
+			time.sleep(10)
+		assert output[TestEventgenOrchestration.server_id[:12]] == 0.0
 
 	def test_controller_set_volume_invalid_request(self):
 		r = requests.post("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport))