diff --git a/lib/schema_checker.py b/lib/schema_checker.py index aa156cab1..66f7ed8d6 100644 --- a/lib/schema_checker.py +++ b/lib/schema_checker.py @@ -99,6 +99,15 @@ def check_usage_scenario(self, usage_scenario): Optional("environment"): self.single_or_list(Or(dict,str)), Optional("ports"): self.single_or_list(Or(str, int)), Optional("depends_on"): Or([str],dict), + Optional("healthcheck"): { + Optional('test'): Or(list, str), + Optional('interval'): str, + Optional('timeout'): str, + Optional('retries'): int, + Optional('start_period'): str, + # Optional('start_interval'): str, docker CLI does not support this atm + Optional('disable'): bool, + }, Optional("setup-commands"): [str], Optional("volumes"): self.single_or_list(str), Optional("folder-destination"):str, diff --git a/metric_providers/psu/energy/ac/mcp/machine/provider.py b/metric_providers/psu/energy/ac/mcp/machine/provider.py index e4522cf4c..2f4c81dc7 100644 --- a/metric_providers/psu/energy/ac/mcp/machine/provider.py +++ b/metric_providers/psu/energy/ac/mcp/machine/provider.py @@ -4,7 +4,7 @@ from metric_providers.base import BaseMetricProvider class PsuEnergyAcMcpMachineProvider(BaseMetricProvider): - def __init__(self, resolutions, skip_check=False): + def __init__(self, resolution, skip_check=False): super().__init__( metric_name='psu_energy_ac_mcp_machine', metrics={'time': int, 'value': int}, diff --git a/metric_providers/psu/energy/ac/xgboost/machine/model b/metric_providers/psu/energy/ac/xgboost/machine/model index 16f45ee04..e42affa17 160000 --- a/metric_providers/psu/energy/ac/xgboost/machine/model +++ b/metric_providers/psu/energy/ac/xgboost/machine/model @@ -1 +1 @@ -Subproject commit 16f45ee04d57442544422097179e20fb0a420665 +Subproject commit e42affa1765ff350b64c800c4115fa802fd3e9b2 diff --git a/runner.py b/runner.py index 6896300ed..e568c756d 100755 --- a/runner.py +++ b/runner.py @@ -89,7 +89,7 @@ def __init__(self, name, uri, uri_type, filename='usage_scenario.yml', branch=None, debug_mode=False, allow_unsafe=False, no_file_cleanup=False, skip_system_checks=False, skip_unsafe=False, verbose_provider_boot=False, full_docker_prune=False, - dry_run=False, dev_repeat_run=False, docker_prune=False, job_id=None): + dev_no_sleeps=False, dev_no_build=False, dev_no_metrics=False, docker_prune=False, job_id=None): if skip_unsafe is True and allow_unsafe is True: raise RuntimeError('Cannot specify both --skip-unsafe and --allow-unsafe') @@ -104,8 +104,9 @@ def __init__(self, self._verbose_provider_boot = verbose_provider_boot self._full_docker_prune = full_docker_prune self._docker_prune = docker_prune - self._dry_run = dry_run - self._dev_repeat_run = dev_repeat_run + self._dev_no_sleeps = dev_no_sleeps + self._dev_no_build = dev_no_build + self._dev_no_metrics = dev_no_metrics self._uri = uri self._uri_type = uri_type self._original_filename = filename @@ -145,7 +146,7 @@ def __init__(self, # self.__filename = self._original_filename # this can be changed later if working directory changes def custom_sleep(self, sleep_time): - if not self._dry_run: + if not self._dev_no_sleeps: print(TerminalColors.HEADER, '\nSleeping for : ', sleep_time, TerminalColors.ENDC) time.sleep(sleep_time) @@ -387,13 +388,13 @@ def check_running_containers(self): def populate_image_names(self): for service_name, service in self._usage_scenario.get('services', {}).items(): if not service.get('image', None): # image is a non-mandatory field. But we need it, so we tmp it - if self._dev_repeat_run: + if self._dev_no_build: service['image'] = f"{service_name}" else: service['image'] = f"{service_name}_{random.randint(500000,10000000)}" def remove_docker_images(self): - if self._dev_repeat_run: + if self._dev_no_build: return print(TerminalColors.HEADER, '\nRemoving all temporary GMT images', TerminalColors.ENDC) @@ -477,6 +478,10 @@ def update_and_insert_specs(self): ) def import_metric_providers(self): + if self._dev_no_metrics: + print(TerminalColors.HEADER, '\nSkipping import of metric providers', TerminalColors.ENDC) + return + config = GlobalConfig().config print(TerminalColors.HEADER, '\nImporting metric providers', TerminalColors.ENDC) @@ -520,6 +525,10 @@ def import_metric_providers(self): self.__metric_providers.sort(key=lambda item: 'rapl' not in item.__class__.__name__.lower()) def download_dependencies(self): + if self._dev_no_build: + print(TerminalColors.HEADER, '\nSkipping downloading dependencies', TerminalColors.ENDC) + return + print(TerminalColors.HEADER, '\nDownloading dependencies', TerminalColors.ENDC) subprocess.run(['docker', 'pull', 'gcr.io/kaniko-project/executor:latest'], check=True) @@ -567,6 +576,7 @@ def build_docker_images(self): encoding='UTF-8', check=True) # The image exists so exit and don't build + print(f"Image {service['image']} exists in build cache. Skipping build ...") continue except subprocess.CalledProcessError: pass @@ -656,10 +666,11 @@ def order_service_names(service_name, visited=None): raise RuntimeError(f"Cycle found in depends_on definition with service '{service_name}'!") visited.add(service_name) + if service_name not in services: + raise RuntimeError(f"Dependent service '{service_name}' defined in 'depends_on' does not exist in usage_scenario!") + service = services[service_name] if 'depends_on' in service: - if isinstance(service['depends_on'], dict): - raise RuntimeError(f"Service definition of {service_name} uses the long form of 'depends_on', however, GMT only supports the short form!") for dep in service['depends_on']: if dep not in names_ordered: order_service_names(dep, visited) @@ -834,6 +845,35 @@ def setup_services(self): if 'pause-after-phase' in service: self.__services_to_pause_phase[service['pause-after-phase']] = self.__services_to_pause_phase.get(service['pause-after-phase'], []) + [container_name] + if 'healthcheck' in service: # must come last + if 'disable' in service['healthcheck'] and service['healthcheck']['disable'] is True: + docker_run_string.append('--no-healthcheck') + else: + if 'test' in service['healthcheck']: + docker_run_string.append('--health-cmd') + health_string = service['healthcheck']['test'] + if isinstance(service['healthcheck']['test'], list): + health_string_copy = service['healthcheck']['test'].copy() + health_string_command = health_string_copy.pop(0) + if health_string_command not in ['CMD', 'CMD-SHELL']: + raise RuntimeError(f"Healthcheck starts with {health_string_command}. Please use 'CMD' or 'CMD-SHELL' when supplying as list. For disabling do not use 'NONE' but the disable argument.") + health_string = ' '.join(health_string_copy) + docker_run_string.append(health_string) + if 'interval' in service['healthcheck']: + docker_run_string.append('--health-interval') + docker_run_string.append(service['healthcheck']['interval']) + if 'timeout' in service['healthcheck']: + docker_run_string.append('--health-timeout') + docker_run_string.append(service['healthcheck']['timeout']) + if 'retries' in service['healthcheck']: + docker_run_string.append('--health-retries') + docker_run_string.append(service['healthcheck']['retries']) + if 'start_period' in service['healthcheck']: + docker_run_string.append('--health-start-period') + docker_run_string.append(service['healthcheck']['start_period']) + if 'start_interval' in service['healthcheck']: + raise RuntimeError('start_interval is not supported atm in healthcheck') + docker_run_string.append(self.clean_image_name(service['image'])) # Before starting the container, check if the dependent containers are "ready". @@ -842,29 +882,55 @@ def setup_services(self): # In the future we want to implement an health check to know if dependent containers are actually ready. if 'depends_on' in service: for dependent_container in service['depends_on']: + print(f"Waiting for dependent container {dependent_container}") time_waited = 0 - state = "" + state = '' + health = 'healthy' # default because some containers have no health max_waiting_time = config['measurement']['boot']['wait_time_dependencies'] while time_waited < max_waiting_time: - # TODO: Check health status instead if `healthcheck` is enabled (https://github.com/green-coding-berlin/green-metrics-tool/issues/423) - # This waiting loop is actually a pre-work for the upcoming health check. For the check if the container is "running", as implemented here, the waiting loop is not needed. status_output = subprocess.check_output( ["docker", "container", "inspect", "-f", "{{.State.Status}}", dependent_container], stderr=subprocess.STDOUT, - encoding='utf-8' + encoding='UTF-8', ) state = status_output.strip() - - if state == "running": + print(f"State of container '{dependent_container}': {state}") + + if isinstance(service['depends_on'], dict) \ + and 'condition' in service['depends_on'][dependent_container]: + + condition = service['depends_on'][dependent_container]['condition'] + if condition == 'service_healthy': + ps = subprocess.run( + ["docker", "container", "inspect", "-f", "{{.State.Health.Status}}", dependent_container], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # put both in one stream + encoding='UTF-8' + ) + health = ps.stdout.strip() + if ps.returncode != 0 or health == '': + raise RuntimeError(f"Health check for dependent_container '{dependent_container}' was requested, but container has no healthcheck implemented! (Output was: {health})") + if health == 'unhealthy': + raise RuntimeError('ontainer healthcheck failed terminally with status "unhealthy")') + print(f"Health of container '{dependent_container}': {health}") + elif condition == 'service_started': + pass + else: + raise RuntimeError(f"Unsupported condition in healthcheck for service '{service_name}': {condition}") + + if state == 'running' and health == 'healthy': break - print(f"State of container '{dependent_container}': {state}. Waiting for 1 second") - self.custom_sleep(1) + print('Waiting for 1 second') + time.sleep(1) time_waited += 1 - if state != "running": - raise RuntimeError(f"Dependent container '{dependent_container}' of '{container_name}' is not running after waiting for {time_waited} sec! Consider checking your service configuration, the entrypoint of the container or the logs of the container.") + if state != 'running': + raise RuntimeError(f"Dependent container '{dependent_container}' of '{container_name}' is not running but {state} after waiting for {time_waited} sec! Consider checking your service configuration, the entrypoint of the container or the logs of the container.") + if health != 'healthy': + raise RuntimeError(f"Dependent container '{dependent_container}' of '{container_name}' is not healthy but '{health}' after waiting for {time_waited} sec! Consider checking your service configuration, the entrypoint of the container or the logs of the container.") if 'command' in service: # must come last for cmd in service['command'].split(): @@ -946,6 +1012,9 @@ def add_to_log(self, container_name, message, cmd=''): def start_metric_providers(self, allow_container=True, allow_other=True): + if self._dev_no_metrics: + return + print(TerminalColors.HEADER, '\nStarting metric providers', TerminalColors.ENDC) for metric_provider in self.__metric_providers: @@ -1099,6 +1168,9 @@ def run_flows(self): # this function should never be called twice to avoid double logging of metrics def stop_metric_providers(self): + if self._dev_no_metrics: + return + print(TerminalColors.HEADER, 'Stopping metric providers and parsing measurements', TerminalColors.ENDC) errors = [] for metric_provider in self.__metric_providers: @@ -1454,8 +1526,9 @@ def run(self): parser.add_argument('--verbose-provider-boot', action='store_true', help='Boot metric providers gradually') parser.add_argument('--full-docker-prune', action='store_true', help='Stop and remove all containers, build caches, volumes and images on the system') parser.add_argument('--docker-prune', action='store_true', help='Prune all unassociated build caches, networks volumes and stopped containers on the system') - parser.add_argument('--dry-run', action='store_true', help='Removes all sleeps. Resulting measurement data will be skewed.') - parser.add_argument('--dev-repeat-run', action='store_true', help='Checks if a docker image is already in the local cache and will then not build it. Also doesn\'t clear the images after a run') + parser.add_argument('--dev-no-metrics', action='store_true', help='Skips loading the metric providers. Runs will be faster, but you will have no metric') + parser.add_argument('--dev-no-sleeps', action='store_true', help='Removes all sleeps. Resulting measurement data will be skewed.') + parser.add_argument('--dev-no-build', action='store_true', help='Checks if a container images are already in the local cache and will then not build it. Also doesn\'t clear the images after a run. Please note that skipping builds only works the second time you make a run.') parser.add_argument('--print-logs', action='store_true', help='Prints the container and process logs to stdout') args = parser.parse_args() @@ -1470,9 +1543,9 @@ def run(self): error_helpers.log_error('--allow-unsafe and skip--unsafe in conjuction is not possible') sys.exit(1) - if args.dev_repeat_run and (args.docker_prune or args.full_docker_prune): + if args.dev_no_build and (args.docker_prune or args.full_docker_prune): parser.print_help() - error_helpers.log_error('--dev-repeat-run blocks pruning docker images. Combination is not allowed') + error_helpers.log_error('--dev-no-build blocks pruning docker images. Combination is not allowed') sys.exit(1) if args.full_docker_prune and GlobalConfig().config['postgresql']['host'] == 'green-coding-postgres-container': @@ -1515,8 +1588,8 @@ def run(self): branch=args.branch, debug_mode=args.debug, allow_unsafe=args.allow_unsafe, no_file_cleanup=args.no_file_cleanup, skip_system_checks=args.skip_system_checks, skip_unsafe=args.skip_unsafe,verbose_provider_boot=args.verbose_provider_boot, - full_docker_prune=args.full_docker_prune, dry_run=args.dry_run, - dev_repeat_run=args.dev_repeat_run, docker_prune=args.docker_prune) + full_docker_prune=args.full_docker_prune, dev_no_sleeps=args.dev_no_sleeps, + dev_no_build=args.dev_no_build, dev_no_metrics=args.dev_no_metrics, docker_prune=args.docker_prune) # Using a very broad exception makes sense in this case as we have excepted all the specific ones before #pylint: disable=broad-except diff --git a/tests/data/usage_scenarios/depends_on_error_unsupported_condition.yml b/tests/data/usage_scenarios/depends_on_error_unsupported_condition.yml new file mode 100644 index 000000000..7d9f3f927 --- /dev/null +++ b/tests/data/usage_scenarios/depends_on_error_unsupported_condition.yml @@ -0,0 +1,20 @@ +--- +name: Test depends_on +author: Arne Tarara +description: test + +services: + test-container-1: + image: alpine + depends_on: + test-container-2: + condition: service_completed_successfully + test-container-2: + image: alpine + +flow: + - name: dummy + container: test-container-1 + commands: + - type: console + command: pwd diff --git a/tests/data/usage_scenarios/depends_on_error_unsupported_long_form.yml b/tests/data/usage_scenarios/depends_on_long_form.yml similarity index 90% rename from tests/data/usage_scenarios/depends_on_error_unsupported_long_form.yml rename to tests/data/usage_scenarios/depends_on_long_form.yml index c50a2bb37..feefe53b8 100644 --- a/tests/data/usage_scenarios/depends_on_error_unsupported_long_form.yml +++ b/tests/data/usage_scenarios/depends_on_long_form.yml @@ -1,5 +1,5 @@ --- -name: Test depends_on +name: Test depends_on long_form author: David Kopp description: test diff --git a/tests/data/usage_scenarios/healthcheck.yml b/tests/data/usage_scenarios/healthcheck.yml new file mode 100644 index 000000000..9134039fb --- /dev/null +++ b/tests/data/usage_scenarios/healthcheck.yml @@ -0,0 +1,23 @@ +--- +name: Test depends_on +author: David Kopp +description: test + +services: + test-container-1: + image: alpine + depends_on: + test-container-2: + condition: service_healthy + test-container-2: + image: alpine + healthcheck: + test: ls + interval: 1s + +flow: + - name: dummy + container: test-container-1 + commands: + - type: console + command: pwd diff --git a/tests/data/usage_scenarios/healthcheck_error_missing.yml b/tests/data/usage_scenarios/healthcheck_error_missing.yml new file mode 100644 index 000000000..9564706f0 --- /dev/null +++ b/tests/data/usage_scenarios/healthcheck_error_missing.yml @@ -0,0 +1,20 @@ +--- +name: Test depends_on +author: David Kopp +description: test + +services: + test-container-1: + image: alpine + depends_on: + test-container-2: + condition: service_healthy + test-container-2: + image: alpine + +flow: + - name: dummy + container: test-container-1 + commands: + - type: console + command: pwd diff --git a/tests/smoke_test.py b/tests/smoke_test.py index bc701dbdb..c02577685 100644 --- a/tests/smoke_test.py +++ b/tests/smoke_test.py @@ -39,12 +39,11 @@ def setup_module(module): err = io.StringIO() GlobalConfig(config_name='test-config.yml').config with redirect_stdout(out), redirect_stderr(err): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) subprocess.run(['docker', 'compose', '-f', uri+'/compose.yml', 'build'], check=True) # Run the application - runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', dev_repeat_run=True, skip_system_checks=False) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', dev_no_build=True, dev_no_sleeps=True, dev_no_metrics=False, skip_system_checks=False) runner.run() #pylint: disable=global-statement diff --git a/tests/test_functions.py b/tests/test_functions.py index e49f65e9f..8ce135d4b 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -35,8 +35,8 @@ def replace_include_in_usage_scenario(usage_scenario_path, docker_compose_filena def setup_runner(usage_scenario, docker_compose=None, uri='default', uri_type='folder', branch=None, debug_mode=False, allow_unsafe=False, no_file_cleanup=False, - skip_unsafe=False, verbose_provider_boot=False, dir_name=None, dev_repeat_run=True, skip_system_checks=True, - dry_run=False): + skip_unsafe=False, verbose_provider_boot=False, dir_name=None, dev_no_build=False, skip_system_checks=True, + dev_no_sleeps=True, dev_no_metrics=True): usage_scenario_path = os.path.join(CURRENT_DIR, 'data/usage_scenarios/', usage_scenario) if docker_compose is not None: docker_compose_path = os.path.join(CURRENT_DIR, 'data/docker-compose-files/', docker_compose) @@ -53,8 +53,8 @@ def setup_runner(usage_scenario, docker_compose=None, uri='default', uri_type='f return Runner(name=RUN_NAME, uri=uri, uri_type=uri_type, filename=usage_scenario, branch=branch, debug_mode=debug_mode, allow_unsafe=allow_unsafe, no_file_cleanup=no_file_cleanup, - skip_unsafe=skip_unsafe, verbose_provider_boot=verbose_provider_boot, dev_repeat_run=dev_repeat_run, - skip_system_checks=skip_system_checks, dry_run=dry_run) + skip_unsafe=skip_unsafe, verbose_provider_boot=verbose_provider_boot, dev_no_build=dev_no_build, + skip_system_checks=skip_system_checks, dev_no_sleeps=dev_no_sleeps, dev_no_metrics=dev_no_metrics) # This function runs the runner up to and *including* the specified step # remember to catch in try:finally and do cleanup when calling this! diff --git a/tests/test_runner.py b/tests/test_runner.py index 26fd5c0f8..2515d5844 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -33,9 +33,9 @@ # os.remove(os.path.join(REPO_ROOT, config_file)) def test_reporters_still_running(): - runner = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, skip_system_checks=False, dry_run=True) + runner = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, skip_system_checks=False, dev_no_sleeps=True, dev_no_build=True, dev_no_metrics=False) - runner2 = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, skip_system_checks=False, dry_run=True) + runner2 = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, skip_system_checks=False, dev_no_sleeps=True, dev_no_build=True, dev_no_metrics=False) runner.check_system('start') # should not fail diff --git a/tests/test_usage_scenario.py b/tests/test_usage_scenario.py index 6bdf5b684..ab9a77bea 100644 --- a/tests/test_usage_scenario.py +++ b/tests/test_usage_scenario.py @@ -31,8 +31,7 @@ # This should be done once per module @pytest.fixture(autouse=True, scope="module", name="build_image") def build_image_fixture(): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) subprocess.run(['docker', 'compose', '-f', uri+'/compose.yml', 'build'], check=True) GlobalConfig().override_config(config_name='test-config.yml') @@ -72,7 +71,7 @@ def get_env_vars(runner): # Test allowed characters def test_env_variable_allowed_characters(): - runner = Tests.setup_runner(usage_scenario='env_vars_stress_allowed.yml', skip_unsafe=False, dry_run=True) + runner = Tests.setup_runner(usage_scenario='env_vars_stress_allowed.yml', skip_unsafe=False, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) env_var_output = get_env_vars(runner) assert 'TESTALLOWED=alpha-num123_' in env_var_output, Tests.assertion_info('TESTALLOWED=alpha-num123_', env_var_output) @@ -82,7 +81,7 @@ def test_env_variable_allowed_characters(): # Test too long values def test_env_variable_too_long(): - runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', dry_run=True) + runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with pytest.raises(RuntimeError) as e: get_env_vars(runner) @@ -90,7 +89,7 @@ def test_env_variable_too_long(): # Test skip_unsafe=true def test_env_variable_skip_unsafe_true(): - runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', skip_unsafe=True, dry_run=True) + runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', skip_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) env_var_output = get_env_vars(runner) # Only allowed values should be in env vars, forbidden ones should be skipped @@ -99,7 +98,7 @@ def test_env_variable_skip_unsafe_true(): # Test allow_unsafe=true def test_env_variable_allow_unsafe_true(): - runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', allow_unsafe=True, dry_run=True) + runner = Tests.setup_runner(usage_scenario='env_vars_stress_forbidden.yml', allow_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) env_var_output = get_env_vars(runner) # Both allowed and forbidden values should be in env vars @@ -126,14 +125,14 @@ def get_port_bindings(runner): return port, err def test_port_bindings_allow_unsafe_true(): - runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml', allow_unsafe=True) + runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml', allow_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) port, _ = get_port_bindings(runner) assert port.startswith('0.0.0.0:9017'), Tests.assertion_info('0.0.0.0:9017', port) def test_port_bindings_skip_unsafe_true(): out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml', skip_unsafe=True) + runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml', skip_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) # need to catch exception here as otherwise the subprocess returning an error will # fail the test @@ -147,7 +146,7 @@ def test_port_bindings_skip_unsafe_true(): Tests.assertion_info(f"Warning: {expected_warning}", 'no/different warning') def test_port_bindings_no_skip_or_allow(): - runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml') + runner = Tests.setup_runner(usage_scenario='port_bindings_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with pytest.raises(Exception) as e: _, docker_port_err = get_port_bindings(runner) expected_container_error = 'Error: No public port \'9018/tcp\' published for test-container\n' @@ -163,7 +162,7 @@ def test_port_bindings_no_skip_or_allow(): def test_setup_commands_one_command(): out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='setup_commands_stress.yml') + runner = Tests.setup_runner(usage_scenario='setup_commands_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with redirect_stdout(out), redirect_stderr(err): try: @@ -178,7 +177,7 @@ def test_setup_commands_one_command(): def test_setup_commands_multiple_commands(): out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='setup_commands_multiple_stress.yml') + runner = Tests.setup_runner(usage_scenario='setup_commands_multiple_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with redirect_stdout(out), redirect_stderr(err): try: @@ -238,7 +237,7 @@ def assert_order(text, first, second): def test_depends_on_order(): out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='depends_on.yml', dry_run=True) + runner = Tests.setup_runner(usage_scenario='depends_on.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with redirect_stdout(out), redirect_stderr(err): try: @@ -255,7 +254,7 @@ def test_depends_on_order(): def test_depends_on_huge(): out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='depends_on_huge.yml', dry_run=True) + runner = Tests.setup_runner(usage_scenario='depends_on_huge.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with redirect_stdout(out), redirect_stderr(err): try: @@ -328,7 +327,7 @@ def test_depends_on_huge(): def test_depends_on_error_not_running(): - runner = Tests.setup_runner(usage_scenario='depends_on_error_not_running.yml', dry_run=True) + runner = Tests.setup_runner(usage_scenario='depends_on_error_not_running.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: with pytest.raises(RuntimeError) as e: Tests.run_until(runner, 'setup_services') @@ -339,7 +338,7 @@ def test_depends_on_error_not_running(): Tests.assertion_info('test-container-2 is not running', str(e.value)) def test_depends_on_error_cyclic_dependency(): - runner = Tests.setup_runner(usage_scenario='depends_on_error_cycle.yml', dry_run=True) + runner = Tests.setup_runner(usage_scenario='depends_on_error_cycle.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: with pytest.raises(RuntimeError) as e: Tests.run_until(runner, 'setup_services') @@ -349,22 +348,66 @@ def test_depends_on_error_cyclic_dependency(): assert "Cycle found in depends_on definition with service 'test-container-1'" in str(e.value) , \ Tests.assertion_info('cycle in depends_on with test-container-1', str(e.value)) -def test_depends_on_error_unsupported_long_form(): - runner = Tests.setup_runner(usage_scenario='depends_on_error_unsupported_long_form.yml', dry_run=True) +def test_depends_on_error_unsupported_condition(): + runner = Tests.setup_runner(usage_scenario='depends_on_error_unsupported_condition.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: with pytest.raises(RuntimeError) as e: Tests.run_until(runner, 'setup_services') finally: runner.cleanup() - assert "long form" in str(e.value) , \ - Tests.assertion_info('long form is not supported', str(e.value)) + message = 'Unsupported condition in healthcheck for service \'test-container-1\': service_completed_successfully' + assert message in str(e.value) , \ + Tests.assertion_info(message, str(e.value)) + +def test_depends_on_long_form(): + runner = Tests.setup_runner(usage_scenario='depends_on_long_form.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) + out = io.StringIO() + err = io.StringIO() + + try: + with redirect_stdout(out), redirect_stderr(err): + runner.run() + message = 'State of container' + assert message in out.getvalue(), \ + Tests.assertion_info(message, out.getvalue()) + finally: + runner.cleanup() + +def test_depends_on_healthcheck(): + runner = Tests.setup_runner(usage_scenario='healthcheck.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) + out = io.StringIO() + err = io.StringIO() + + try: + with redirect_stdout(out), redirect_stderr(err): + runner.run() + message = 'Health of container \'test-container-2\': starting' + assert message in out.getvalue(), Tests.assertion_info(message, out.getvalue()) + message2 = 'Health of container \'test-container-2\': healthy' + assert message2 in out.getvalue(), Tests.assertion_info(message, out.getvalue()) + + finally: + runner.cleanup() + +def test_depends_on_healthcheck_error_missing(): + runner = Tests.setup_runner(usage_scenario='healthcheck_error_missing.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) + + try: + with pytest.raises(RuntimeError) as e: + runner.run() + finally: + runner.cleanup() + + expected_exception = "Health check for dependent_container 'test-container-2' was requested, but container has no healthcheck implemented!" + assert str(e.value).startswith(expected_exception),\ + Tests.assertion_info(f"Exception: {expected_exception}", str(e.value)) #volumes: [array] (optional) #Array of volumes to be mapped. Only read of runner.py is executed with --allow-unsafe flag def test_volume_bindings_allow_unsafe_true(): create_test_file('/tmp/gmt-test-data') - runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml', allow_unsafe=True) + runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml', allow_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) ls = get_contents_of_bound_volume(runner) assert 'test-file' in ls, Tests.assertion_info('test-file', ls) @@ -372,7 +415,7 @@ def test_volumes_bindings_skip_unsafe_true(): create_test_file('/tmp/gmt-test-data') out = io.StringIO() err = io.StringIO() - runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml', skip_unsafe=True) + runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml', skip_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with redirect_stdout(out), redirect_stderr(err), pytest.raises(Exception): ls = get_contents_of_bound_volume(runner) @@ -383,7 +426,7 @@ def test_volumes_bindings_skip_unsafe_true(): def test_volumes_bindings_no_skip_or_allow(): create_test_file('/tmp/gmt-test-data') - runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml') + runner = Tests.setup_runner(usage_scenario='volume_bindings_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) with pytest.raises(RuntimeError) as e: ls = get_contents_of_bound_volume(runner) assert ls == '', Tests.assertion_info('empty list', ls) @@ -392,7 +435,7 @@ def test_volumes_bindings_no_skip_or_allow(): Tests.assertion_info(f"Exception: {expected_exception}", str(e.value)) def test_network_created(): - runner = Tests.setup_runner(usage_scenario='network_stress.yml') + runner = Tests.setup_runner(usage_scenario='network_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: Tests.run_until(runner, 'setup_networks') ps = subprocess.run( @@ -408,7 +451,7 @@ def test_network_created(): assert 'gmt-test-network' in ls, Tests.assertion_info('gmt-test-network', ls) def test_container_is_in_network(): - runner = Tests.setup_runner(usage_scenario='network_stress.yml') + runner = Tests.setup_runner(usage_scenario='network_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: Tests.run_until(runner, 'setup_services') ps = subprocess.run( @@ -428,7 +471,7 @@ def test_container_is_in_network(): # When container does not have a daemon running typically a shell # is started here to have the container running like bash or sh def test_cmd_ran(): - runner = Tests.setup_runner(usage_scenario='cmd_stress.yml') + runner = Tests.setup_runner(usage_scenario='cmd_stress.yml', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: Tests.run_until(runner, 'setup_services') ps = subprocess.run( @@ -448,12 +491,11 @@ def test_cmd_ran(): # The URI to get the usage_scenario.yml from. Can be either a local directory starting with # / or a remote git repository starting with http(s):// def test_uri_local_dir(): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-system-checks', '--dev-repeat-run'], + '--skip-system-checks', '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -465,7 +507,7 @@ def test_uri_local_dir(): assert ps.stderr == '', Tests.assertion_info('no errors', ps.stderr) def test_uri_local_dir_missing(): - runner = Tests.setup_runner(usage_scenario='basic_stress.yml', uri='/tmp/missing') + runner = Tests.setup_runner(usage_scenario='basic_stress.yml', uri='/tmp/missing', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) try: with pytest.raises(FileNotFoundError) as e: runner.run() @@ -481,7 +523,7 @@ def test_uri_github_repo(): RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-system-checks', '--dev-repeat-run'], + '--skip-system-checks', '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -495,7 +537,7 @@ def test_uri_github_repo(): ## --branch BRANCH # Optionally specify the git branch when targeting a git repository def test_uri_local_branch(): - runner = Tests.setup_runner(usage_scenario='basic_stress.yml', branch='test-branch') + runner = Tests.setup_runner(usage_scenario='basic_stress.yml', branch='test-branch', dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) out = io.StringIO() err = io.StringIO() with redirect_stdout(out), redirect_stderr(err), pytest.raises(RuntimeError) as e: @@ -513,7 +555,7 @@ def test_uri_github_repo_branch(): ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--branch', 'test-branch' , '--filename', 'basic_stress.yml', - '--config-override', 'test-config.yml', '--skip-system-checks', '--dev-repeat-run'], + '--config-override', 'test-config.yml', '--skip-system-checks', '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -532,7 +574,11 @@ def test_uri_github_repo_branch_missing(): runner = Tests.setup_runner(usage_scenario='basic_stress.yml', uri='https://github.com/green-coding-berlin/pytest-dummy-repo', uri_type='URL', - branch='missing-branch') + branch='missing-branch', + dev_no_sleeps=True, + dev_no_build=True, + dev_no_metrics=True, + ) with pytest.raises(subprocess.CalledProcessError) as e: runner.run() expected_exception = 'returned non-zero exit status 128' @@ -542,12 +588,11 @@ def test_uri_github_repo_branch_missing(): # # --name NAME # # A name which will be stored to the database to discern this run from others def test_name_is_in_db(): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-system-checks'], + '--skip-system-checks', '--dev-no-metrics', '--dev-no-sleeps', '--dev-no-build'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -570,7 +615,7 @@ def test_different_filename(): ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--filename', 'basic_stress.yml', '--config-override', 'test-config.yml', - '--skip-system-checks', '--dev-repeat-run'], + '--skip-system-checks', '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -589,7 +634,7 @@ def test_different_filename_missing(): uri = os.path.abspath(os.path.join(CURRENT_DIR, '..', 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) - runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename='basic_stress.yml', skip_system_checks=True) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename='basic_stress.yml', skip_system_checks=True, dev_no_build=True, dev_no_sleeps=True, dev_no_metrics=True) with pytest.raises(FileNotFoundError) as e: runner.run() @@ -600,8 +645,7 @@ def test_different_filename_missing(): # --no-file-cleanup # Do not delete files in /tmp/green-metrics-tool def test_no_file_cleanup(): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , @@ -617,19 +661,18 @@ def test_no_file_cleanup(): #pylint: disable=unused-variable def test_skip_and_allow_unsafe_both_true(): with pytest.raises(RuntimeError) as e: - runner = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, allow_unsafe=True) + runner = Tests.setup_runner(usage_scenario='basic_stress.yml', skip_unsafe=True, allow_unsafe=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=True) expected_exception = 'Cannot specify both --skip-unsafe and --allow-unsafe' assert str(e.value) == expected_exception, Tests.assertion_info('', str(e.value)) def test_debug(monkeypatch): monkeypatch.setattr('sys.stdin', io.StringIO('Enter')) - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--debug', '--config-override', 'test-config.yml', '--skip-system-checks', - '--dev-repeat-run'], + '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -645,13 +688,12 @@ def test_debug(monkeypatch): ## rethink this one def wip_test_verbose_provider_boot(): - uri = os.path.abspath(os.path.join( - CURRENT_DIR, 'stress-application/')) + uri = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/')) RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--verbose-provider-boot', '--config-override', 'test-config.yml', - '--dev-repeat-run'], + '--dev-no-sleeps', '--dev-no-build', '--dev-no-metrics'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/tests/test_volume_loading.py b/tests/test_volume_loading.py index 18e165fd3..b90b1dca4 100644 --- a/tests/test_volume_loading.py +++ b/tests/test_volume_loading.py @@ -39,8 +39,7 @@ def check_if_container_running(container_name): def test_volume_load_no_escape(): tmp_dir_name = utils.randomword(12) tmp_dir = os.path.join(CURRENT_DIR, 'tmp', tmp_dir_name, 'basic_stress_w_import.yml') - runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', - docker_compose='volume_load_etc_passwords.yml', dir_name=tmp_dir_name) + runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', docker_compose='volume_load_etc_passwords.yml', dir_name=tmp_dir_name, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) Tests.replace_include_in_usage_scenario(tmp_dir, 'volume_load_etc_passwords.yml') try: @@ -83,7 +82,7 @@ def test_load_files_from_within_gmt(): copy_compose_and_edit_directory('volume_load_within_proj.yml', tmp_dir) # setup runner and run test - runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name) + runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) Tests.replace_include_in_usage_scenario(os.path.join(tmp_dir, 'basic_stress_w_import.yml'), 'docker-compose.yml') try: @@ -111,7 +110,7 @@ def test_symlinks_should_fail(): copy_compose_and_edit_directory('volume_load_symlinks_negative.yml', tmp_dir) - runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name) + runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) Tests.replace_include_in_usage_scenario(os.path.join(tmp_dir, 'basic_stress_w_import.yml'), 'docker-compose.yml') try: @@ -128,8 +127,7 @@ def test_symlinks_should_fail(): def test_non_bind_mounts_should_fail(): tmp_dir_name = create_tmp_dir()[1] tmp_dir_usage = os.path.join(CURRENT_DIR, 'tmp', tmp_dir_name, 'basic_stress_w_import.yml') - runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', - docker_compose='volume_load_non_bind_mounts.yml', dir_name=tmp_dir_name) + runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', docker_compose='volume_load_non_bind_mounts.yml', dir_name=tmp_dir_name, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) Tests.replace_include_in_usage_scenario(tmp_dir_usage, 'volume_load_non_bind_mounts.yml') try: @@ -149,7 +147,7 @@ def test_load_volume_references(): copy_compose_and_edit_directory('volume_load_references.yml', tmp_dir) - runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name) + runner = Tests.setup_runner(usage_scenario='basic_stress_w_import.yml', dir_name=tmp_dir_name, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) Tests.replace_include_in_usage_scenario(os.path.join(tmp_dir, 'basic_stress_w_import.yml'), 'docker-compose.yml') try: @@ -172,7 +170,7 @@ def test_load_volume_references(): def test_volume_loading_subdirectories_root(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') RUN_NAME = 'test_' + utils.randomword(12) - runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', skip_system_checks=True) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', skip_system_checks=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) out = io.StringIO() err = io.StringIO() @@ -200,7 +198,7 @@ def test_volume_loading_subdirectories_root(): def test_volume_loading_subdirectories_subdir(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') RUN_NAME = 'test_' + utils.randomword(12) - runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/usage_scenario_subdir.yml", skip_system_checks=True) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/usage_scenario_subdir.yml", skip_system_checks=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) out = io.StringIO() err = io.StringIO() @@ -219,7 +217,7 @@ def test_volume_loading_subdirectories_subdir(): def test_volume_loading_subdirectories_subdir2(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') RUN_NAME = 'test_' + utils.randomword(12) - runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/subdir2/usage_scenario_subdir2.yml", skip_system_checks=True) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/subdir2/usage_scenario_subdir2.yml", skip_system_checks=True, dev_no_metrics=True, dev_no_sleeps=True, dev_no_build=False) out = io.StringIO() err = io.StringIO()