diff --git a/.travis.yml b/.travis.yml index d084acf..f05070a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,14 +25,14 @@ matrix: node_js: "4.2.6" env: - PYTHONPATH=/home/travis/build/seecloud/availability - - AVAILABILITY_CONF=/home/travis/build/seecloud/availability/tests/ci/api/config.json + - AVAILABILITY_CONF=/home/travis/build/seecloud/availability/tests/ci/api/config.yaml services: - docker install: - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.tar.gz -O - | tar xz - ./elasticsearch-5.1.1/bin/elasticsearch -d - docker build -t availability . - - docker run -d -p 127.0.0.1:5000:5000 --net=host -e "AVAILABILITY_CONF=/app/tests/ci/api/config.json" availability /app/entrypoint-api.sh + - docker run -d -p 127.0.0.1:5000:5000 --net=host -e "AVAILABILITY_CONF=/app/tests/ci/api/config.yaml" availability /app/entrypoint-api.sh - git clone http://github.com/cybertk/abao && cd abao && git checkout 0.5.0 - npm install - cp ../tests/ci/api/abao_hooks.js hooks.js diff --git a/README.rst b/README.rst index 123f9a7..2b18d2c 100644 --- a/README.rst +++ b/README.rst @@ -31,11 +31,6 @@ Here is a simple example: .. code-block:: { - "flask": { - "PORT": 5020, - "HOST": "0.0.0.0", - "DEBUG": false - }, "backend": { "type": "elastic", "connection": [{"host": "127.0.0.1", "port": 9200}] diff --git a/availability/api/v1/api.py b/availability/api/v1/api.py index 64c050b..c6e4426 100644 --- a/availability/api/v1/api.py +++ b/availability/api/v1/api.py @@ -17,12 +17,9 @@ import flask -from availability import config from availability import storage - -LOG = logging.getLogger("api") -LOG.setLevel(config.get_config().get("logging", {}).get("level", "INFO")) +LOG = logging.getLogger(__name__) def get_period_interval(period): diff --git a/availability/app.py b/availability/app.py new file mode 100644 index 0000000..28acfc6 --- /dev/null +++ b/availability/app.py @@ -0,0 +1,34 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import flask +from oss_lib import routing + +from availability.api.v1 import api +from availability.api.v1 import regions + +app = flask.Flask(__name__, static_folder=None) + + +@app.errorhandler(404) +def not_found(error): + return flask.jsonify({"error": "Not Found"}), 404 + + +for bp in [api, regions]: + for url_prefix, blueprint in bp.get_blueprints(): + app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix) + +app = routing.add_routing_map(app, html_uri=None, json_uri="/") diff --git a/availability/config.py b/availability/config.py index 57028c5..d1698eb 100644 --- a/availability/config.py +++ b/availability/config.py @@ -13,113 +13,68 @@ # License for the specific language governing permissions and limitations # under the License. -import json -import logging -import os +DEFAULT_CONF_PATH = "/etc/availability/config.yaml" -import jsonschema - - -CONF = None - -DEFAULT_CONF = { - "flask": { - "HOST": "0.0.0.0", - "PORT": 5000, - "DEBUG": False - }, +DEFAULT = { "backend": { "type": "elastic", - "connection": [{"host": "127.0.0.1", "port": 9200}] + "connection": [ + {"host": "127.0.0.1", "port": 9200}, + ], }, - "regions": [] + "regions": [], + "period": 60, + "connection_timeout": 1, + "read_timeout": 10, } -CONF_SCHEMA = { - "type": "object", - "$schema": "http://json-schema.org/draft-04/schema", - "properties": { - "flask": { - "type": "object", - "properties": { - "PORT": {"type": "integer"}, - "HOST": {"type": "string"}, - "DEBUG": {"type": "boolean"} - } +SCHEMA = { + "backend": { + "type": "object", + "properties": { + "type": {"type": "string"}, + "connection": { + "type": "array", + "items": { + # TODO(akscram): Here should be enum. + "type": "object", + "properties": { + "host": {"type": "string"}, + "port": {"type": "integer"}, + }, + "required": ["host"], + "additionalProperties": False, + }, + "minItems": 1, + }, }, - "backend": { + "required": ["type", "connection"], + "additionalProperties": False, + }, + "regions": { + "type": "array", + "items": { "type": "object", "properties": { - "type": {"type": "string"}, - "connection": { + "name": {"type": "string"}, + "services": { "type": "array", "items": { "type": "object", "properties": { - "host": {"type": "string"}, - "port": {"type": "integer"} + "name": {"type": "string"}, + "url": {"type": "string"} }, - "required": ["host"] + "required": ["name", "url"], + "additionalProperties": False, }, - "minItems": 1 - } + "minItems": 1, + }, }, - "required": ["type", "connection"] - }, - "regions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "services": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "url": {"type": "string"} - }, - "required": ["name", "url"] - }, - "minItems": 1 - } - } - } + "additionalProperties": False, }, - "period": {"type": "number", "minimum": 5}, - "connection_timeout": {"type": "number"}, - "read_timeout": {"type": "number"}, - "logging": { - "type": "object", - "properties": { - "level": {"type": "string"} - } - } }, - "required": ["flask", "backend", "regions"] + "period": {"type": "number", "minimum": 5}, + "connection_timeout": {"type": "number"}, + "read_timeout": {"type": "number"}, } - - -def get_config(): - """Get cached configuration. - - :returns: application config - :rtype: dict - """ - global CONF - if not CONF: - path = os.environ.get("AVAILABILITY_CONF", - "/etc/availability/config.json") - try: - config = json.load(open(path)) - logging.info("Config is '%s'" % path) - jsonschema.validate(config, CONF_SCHEMA) - CONF = config - except IOError as exc: - logging.warning("Failed to load config from '%s': %s", path, exc) - CONF = DEFAULT_CONF - except jsonschema.exceptions.ValidationError as exc: - logging.error("Configuration file %s is not valid: %s", path, exc) - raise - return CONF diff --git a/availability/main.py b/availability/main.py index 5aff009..8ae3b84 100644 --- a/availability/main.py +++ b/availability/main.py @@ -13,35 +13,29 @@ # License for the specific language governing permissions and limitations # under the License. -import flask -from flask_helpers import routing +import argparse -from availability.api.v1 import api -from availability.api.v1 import regions -from availability import config +from oss_lib import config - -app = flask.Flask(__name__, static_folder=None) -app.config.update(config.get_config()["flask"]) - - -@app.errorhandler(404) -def not_found(error): - return flask.jsonify({"error": "Not Found"}), 404 - - -for bp in [api, regions]: - for url_prefix, blueprint in bp.get_blueprints(): - app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix) - - -app = routing.add_routing_map(app, html_uri=None, json_uri="/") +from availability import app +from availability import config as cfg def main(): - app.run(host=app.config.get("HOST", "0.0.0.0"), - port=app.config.get("PORT", 5000)) - - -if __name__ == "__main__": - main() + parser = argparse.ArgumentParser() + parser.add_argument("--host", + default="0.0.0.0", + help="A host to bind development server. " + "(default 0.0.0.0)") + parser.add_argument("--port", + type=int, + default=5000, + help="A port to bind development server. " + "(default 5000)") + args = config.process_args("AVAILABILITY", + parser=parser, + default_config_path=cfg.DEFAULT_CONF_PATH, + defaults=cfg.DEFAULT, + validation_schema=cfg.SCHEMA) + app.app.config.update(config.CONF, **{"DEBUG": args.debug}) + app.app.run(host=args.host, port=args.port) diff --git a/availability/storage.py b/availability/storage.py index 7e24e55..fbb1dbd 100644 --- a/availability/storage.py +++ b/availability/storage.py @@ -17,11 +17,11 @@ import logging import elasticsearch +from oss_lib import config -from availability import config +CONF = config.CONF +LOG = logging.getLogger(__name__) -LOG = logging.getLogger("storage") -LOG.setLevel(config.get_config().get("logging", {}).get("level", "INFO")) NUMBER_OF_SHARDS = 2 @@ -32,7 +32,7 @@ def get_elasticsearch(check_availability=False): :returns: Elasticsearch or None on failure :rtype: elasticsearch.Elasticsearch """ - nodes = config.get_config()["backend"]["connection"] + nodes = CONF["backend"]["connection"] try: es = elasticsearch.Elasticsearch(nodes) if check_availability: diff --git a/availability/watcher.py b/availability/watcher.py index 49b02d7..a5072fe 100644 --- a/availability/watcher.py +++ b/availability/watcher.py @@ -16,25 +16,21 @@ import datetime as dt import json import logging -import sys import threading import time import uuid from elasticsearch import exceptions as es_exceptions +from oss_lib import config import queue import requests import schedule -from availability import config +from availability import config as cfg from availability import storage - -SERVICE_CONN_TIMEOUT = config.get_config().get("connection_timeout", 1) -SERVICE_READ_TIMEOUT = config.get_config().get("read_timeout", 10) - -LOG = logging.getLogger("watcher") -LOG.setLevel(config.get_config().get("logging", {}).get("level", "INFO")) +CONF = config.CONF +LOG = logging.getLogger(__name__) def check_availability(data, results_queue): @@ -45,8 +41,9 @@ def check_availability(data, results_queue): :rtype: None """ try: - requests.get(data["url"], verify=False, timeout=(SERVICE_CONN_TIMEOUT, - SERVICE_READ_TIMEOUT)) + requests.get(data["url"], verify=False, + timeout=(CONF["connection_timeout"], + CONF["read_timeout"])) data["status"] = 1 except Exception as e: @@ -105,7 +102,7 @@ def save_availability(results_queue): def watch_services(): """Query services of all regions and save results.""" - for region in config.get_config().get("regions"): + for region in CONF["regions"]: for service in region.get("services"): LOG.info("Checking service '%(name)s' availability on %(url)s" % service) @@ -131,18 +128,23 @@ def main(period=None): This runs infinite availability check with given period. :param period: period in seconds """ - if not config.get_config().get("regions"): + config.process_args("AVAILABILITY", + default_config_path=cfg.DEFAULT_CONF_PATH, + defaults=cfg.DEFAULT, + validation_schema=cfg.SCHEMA) + + if not CONF["regions"]: LOG.error("No regions configured. Quitting.") return 1 - period = period or config.get_config().get("period", 60) + period = period or CONF["period"] - if SERVICE_CONN_TIMEOUT + SERVICE_READ_TIMEOUT > period: + if CONF["connection_timeout"] + CONF["read_timeout"] > period: LOG.error("Period can not be lesser than timeout, " "otherwise threads could crowd round.") return 1 - backend = config.get_config().get("backend") + backend = CONF["backend"] if backend["type"] != "elastic": LOG.error("Unexpected backend: %(type)s" % backend) return 1 @@ -157,10 +159,3 @@ def main(period=None): while True: time.sleep(1) schedule.run_pending() - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - LOG.error("Got SIGINT. Quitting...") diff --git a/availability/wsgi.py b/availability/wsgi.py new file mode 100644 index 0000000..0cd407e --- /dev/null +++ b/availability/wsgi.py @@ -0,0 +1,27 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oss_lib import config + +from availability import app +from availability import config as cfg + + +config.process_env("AVAILABILITY", + default_config_path=cfg.DEFAULT_CONF_PATH, + defaults=cfg.DEFAULT, + validation_schema=cfg.SCHEMA) + +application = app.app diff --git a/entrypoint-all.sh b/entrypoint-all.sh index 850e98a..625475b 100755 --- a/entrypoint-all.sh +++ b/entrypoint-all.sh @@ -1,6 +1,6 @@ #!/bin/bash availability-watcher & -gunicorn -w 4 -b 0.0.0.0:5000 availability.main:app & +gunicorn -w 4 -b 0.0.0.0:5000 availability.wsgi & wait -n diff --git a/entrypoint-api.sh b/entrypoint-api.sh index 53b135c..8a1cac8 100755 --- a/entrypoint-api.sh +++ b/entrypoint-api.sh @@ -1,3 +1,3 @@ #!/bin/bash -gunicorn -w 4 -b 0.0.0.0:5000 availability.main:app +gunicorn -w 4 -b 0.0.0.0:5000 availability.wsgi diff --git a/etc/sample_config.json b/etc/sample_config.json deleted file mode 100644 index fe52ea4..0000000 --- a/etc/sample_config.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "flask": { - "PORT": 5000, - "HOST": "0.0.0.0", - "DEBUG": false - }, - "backend": { - "type": "elastic", - "connection": [{"host": "127.0.0.1", "port": 9200}] - }, - "regions": [ - { - "name": "west-1", - "services": [ - {"name": "nova", "url": "http://foo.example.org:1234/"}, - {"name": "cinder", "url": "http://foo.example.org:4567/"} - ] - }, - { - "name": "west-2", - "services": [ - {"name": "keystone", "url": "http://example.org/ab/"}, - {"name": "cinder", "url": "http://example.org/cd/"} - ] - } - ], - "period": 60, - "connection_timeout": 1, - "read_timeout": 10, - "logging": { - "level": "INFO" - } -} diff --git a/etc/sample_config.yaml b/etc/sample_config.yaml new file mode 100644 index 0000000..fc904fb --- /dev/null +++ b/etc/sample_config.yaml @@ -0,0 +1,21 @@ +backend: + type: elastic + connection: + - host: 127.0.0.1 + port: 9200 +regions: + - name: west-1 + services: + - name: nova + url: http://foo.example.org:1234/ + - name: cinder + url: http://foo.example.org:4567/ + - name: west-2 + services: + - name: keystone + url: http://example.org/ab/ + - name: cinder + url: http://example.org/cd/ +period: 60 +connection_timeout: 1 +read_timeout: 10 diff --git a/requirements.txt b/requirements.txt index 45d4e0a..2451b28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ gunicorn==19.6.0 Flask==0.11.1 -flask-helpers==0.1 +oss-lib==0.1.1 requests==2.11.1 jsonschema==2.5.1 future==0.16.0 diff --git a/service/availability-api.yaml b/service/availability-api.yaml index c0ab82c..1b21e41 100644 --- a/service/availability-api.yaml +++ b/service/availability-api.yaml @@ -19,11 +19,11 @@ service: --name availability-api --workers {{ availability.wsgi.workers }} --bind 0.0.0.0:{{ availability.port.cont }} - availability.main:app + availability.wsgi files: - availability.conf files: availability.conf: - path: /etc/availability/config.json + path: /etc/availability/config.yaml content: availability.conf.j2 diff --git a/service/availability-watcher.yaml b/service/availability-watcher.yaml index 9ffeb7f..5ba2796 100644 --- a/service/availability-watcher.yaml +++ b/service/availability-watcher.yaml @@ -16,5 +16,5 @@ service: files: availability.conf: - path: /etc/availability/config.json + path: /etc/availability/config.yaml content: availability.conf.j2 diff --git a/service/files/availability.conf.j2 b/service/files/availability.conf.j2 index c26fe78..2a5c481 100644 --- a/service/files/availability.conf.j2 +++ b/service/files/availability.conf.j2 @@ -1,43 +1,20 @@ -{ - "flask": { - "HOST": "0.0.0.0", - "PORT": {{ availability.port.cont }}, - "DEBUG": {% if availability.debug %}true{% else %}false{% endif %} - }, - "backend": { - "type": "elastic", - "connection": [ - {%- for node in elasticsearch.hosts %} - { - "host": "{{ node.host }}", - "port": {{ node.port }} - } - {%- if loop.length > 1 and not loop.last %},{% endif -%} - {%- endfor -%} - ] - }, - "regions": [ - {%- for region in availability.regions %} - { - "name": "{{ region.name }}", - "services": [ - {%- for service in region.services -%} - { - "name": "{{ service.name }}", - "url": "{{ service.url }}" - } - {%- if loop.length > 1 and not loop.last %},{% endif -%} - {%- endfor -%} - ] - } - {%- if loop.length > 1 and not loop.last %},{% endif -%} +backend: + type: elastic + connection: + {%- for node in elasticsearch.hosts %} + - host: {{ node.host }} + port: {{ node.port }} {%- endfor %} - ], +regions: + {%- for region in availability.regions %} + - name: {{ region.name }} + services: + {%- for service in region.services %} + - name: {{ service.name }} + url: {{ service.url }} + {%- endfor -%} + {%- endfor %} - "period": {{ availability.watcher.period }}, - "connection_timeout": {{ availability.watcher.connection_timeout }}, - "read_timeout": {{ availability.watcher.read_timeout }}, - "logging": { - "level": "{% if availability.debug %}DEBUG{% else %}INFO{% endif %}" - } -} +period: {{ availability.watcher.period }} +connection_timeout: {{ availability.watcher.connection_timeout }} +read_timeout: {{ availability.watcher.read_timeout }} \ No newline at end of file diff --git a/tests/ci/api/config.json b/tests/ci/api/config.json deleted file mode 100644 index 0160a19..0000000 --- a/tests/ci/api/config.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "flask": { - "PORT": 5000, - "HOST": "0.0.0.0", - "DEBUG": true - }, - "backend": { - "type": "elastic", - "connection": [{"host": "127.0.0.1", "port": 9200}] - }, - "regions": [ - { - "name": "foo_region", - "services": [ - {"name": "alpha", "url": "http://alpha.example.org/"}, - {"name": "beta", "url": "http://beta.example.org/"}, - {"name": "gamma", "url": "http://gamma.example.org:5555/"} - ] - }, - { - "name": "bar_region", - "services": [ - {"name": "delta", "url": "http://example.org/delta/"}, - {"name": "epsilon", "url": "http://example.org/epsilon/"} - ] - } - ], - "period": 60, - "logging": { - "level": "DEBUG" - } -} diff --git a/tests/ci/api/config.yaml b/tests/ci/api/config.yaml new file mode 100644 index 0000000..e5e037a --- /dev/null +++ b/tests/ci/api/config.yaml @@ -0,0 +1,23 @@ +backend: + type: elastic + connection: + - host: 127.0.0.1 + port: 9200 +regions: + - name: foo_region + services: + - name: alpha + url: http://alpha.example.org/ + - name: beta + url: http://beta.example.org/ + - name: gamma + url: http://gamma.example.org:5555/ + - name: bar_region + services: + - name: delta + url: http://example.org/delta/ + - name: epsilon + url: http://example.org/epsilon/ +period: 60 +connection_timeout: 1 +read_timeout: 10 diff --git a/tests/ci/api/populate_elastic.py b/tests/ci/api/populate_elastic.py index fb244b0..27be985 100644 --- a/tests/ci/api/populate_elastic.py +++ b/tests/ci/api/populate_elastic.py @@ -15,11 +15,13 @@ import datetime as dt import json -import os import random import sys import uuid +from oss_lib import config + +from availability import config as cfg from availability import storage @@ -57,8 +59,10 @@ def random_status(success_probability=0.7): def populate_elastic(**kwargs): started_at = dt.datetime.now() - conf = json.load(open(os.environ.get("AVAILABILITY_CONF"))) - regions = {r["name"]: r["services"] for r in conf["regions"]} + config.process_env("AVAILABILITY", + defaults=cfg.DEFAULT, + validation_schema=cfg.SCHEMA) + regions = {r["name"]: r["services"] for r in config.CONF["regions"]} elastic = storage.get_elasticsearch(check_availability=True) # Create indices diff --git a/tests/unit/api/v1/test_api.py b/tests/unit/api/v1/test_api.py index 33f2c24..c7ef6f5 100644 --- a/tests/unit/api/v1/test_api.py +++ b/tests/unit/api/v1/test_api.py @@ -15,6 +15,7 @@ import mock +from availability.api.v1 import api from tests.unit import test @@ -74,3 +75,37 @@ def test_get_region_availability(self, mock_process_results, mock_storage.es_search.reset_mock() mock_process_results.assert_called_once_with("foo_buckets") mock_process_results.reset_mock() + + +# TODO(akscram): Just to pass coverage, re-write this test completely. +class ResultsTestCase(test.TestCase): + def test_process_results(self): + buckets = [{ + "key": "key_av", + "availability": {"value": 1}, + "data": { + "buckets": [ + { + "availability": {"value": 1}, + "key_as_string": "key_1", + }, + { + "availability": {"value": 1}, + "key_as_string": "key_2", + }, + ], + }, + }] + expected = { + "availability": { + "key_av": { + "availability": 1, + "availability_data": [ + ["key_1", 1], + ["key_2", 1], + ], + }, + }, + } + result = api.process_results(buckets) + self.assertEqual(result, expected) diff --git a/tests/unit/test.py b/tests/unit/test.py index 855f538..2ec2e37 100644 --- a/tests/unit/test.py +++ b/tests/unit/test.py @@ -13,19 +13,36 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import json +import mock +from oss_lib import config import testtools -from availability import main +from availability import app +from availability import config as cfg class TestCase(testtools.TestCase): def setUp(self): super(TestCase, self).setUp() - self.app = main.app.test_client() + self.app = app.app.test_client() def get(self, *args, **kwargs): rv = self.app.get(*args, **kwargs) return rv.status_code, json.loads(rv.data.decode()) + + def mock_config(self, update=None): + patch = mock.patch("oss_lib.config._CONF") + patch.start() + self.addCleanup(patch.stop) + + defaults = copy.deepcopy(cfg.DEFAULT) + if update: + config.merge_dicts(defaults, update) + config.setup_config( + defaults=defaults, + validation_schema=cfg.SCHEMA, + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 68432f0..1e8e318 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -14,60 +14,11 @@ # under the License. import jsonschema -import mock from availability import config from tests.unit import test class ConfigTestCase(test.TestCase): - - @mock.patch("availability.config.CONF", new=None) - @mock.patch("availability.config.os.environ.get") - @mock.patch("availability.config.open", create=True) - @mock.patch("availability.config.json.load") - @mock.patch("availability.config.jsonschema.validate") - def test_get_config(self, mock_validate, mock_load, mock_open, mock_get): - mock_get.return_value = "foo_path" - mock_load.return_value = {"foo": 42, "bar": "spam"} - mock_open.return_value = "foo_stream" - - cfg = config.get_config() - - self.assertEqual({"foo": 42, "bar": "spam"}, cfg) - mock_get.assert_called_once_with( - "AVAILABILITY_CONF", "/etc/availability/config.json") - mock_open.assert_called_once_with("foo_path") - mock_load.assert_called_once_with("foo_stream") - mock_validate.assert_called_once_with({"foo": 42, "bar": "spam"}, - config.CONF_SCHEMA) - - @mock.patch("availability.config.CONF", new=None) - @mock.patch("availability.config.os.environ.get") - @mock.patch("availability.config.open", create=True) - @mock.patch("availability.config.json.load") - @mock.patch("availability.config.jsonschema.validate") - def test_get_config_open_error( - self, mock_validate, mock_load, mock_open, mock_get): - mock_open.side_effect = IOError - cfg = config.get_config() - self.assertEqual(config.DEFAULT_CONF, cfg) - - @mock.patch("availability.config.CONF", new=None) - @mock.patch("availability.config.os.environ.get") - @mock.patch("availability.config.open", create=True) - @mock.patch("availability.config.json.load") - @mock.patch("availability.config.jsonschema.validate") - def test_get_config_validation_error( - self, mock_validate, mock_load, mock_open, mock_get): - validation_exc = jsonschema.exceptions.ValidationError - mock_validate.side_effect = validation_exc(1) - - self.assertRaises(validation_exc, config.get_config) - - @mock.patch("availability.config.CONF", new=42) - def test_get_config_cached(self): - self.assertEqual(42, config.get_config()) - def test_validate_default_config(self): - jsonschema.validate(config.DEFAULT_CONF, config.CONF_SCHEMA) + jsonschema.validate(config.DEFAULT, config.SCHEMA) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index a2f0904..b2f8dd9 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -26,14 +26,25 @@ def test_not_found(self): self.assertEqual(404, code) self.assertEqual({"error": "Not Found"}, resp) - @mock.patch("availability.main.app") - def test_main(self, mock_app): - mock_app.config = {} + @mock.patch("availability.main.argparse.ArgumentParser") + @mock.patch("availability.main.config.process_args") + @mock.patch("availability.main.app.app") + def test_main_default(self, mock_app, mock_process, mock_parser): + mock_process.return_value.configure_mock(**{ + "host": "0.0.0.0", + "port": 5000, + }) main.main() mock_app.run.assert_called_once_with(host="0.0.0.0", port=5000) - mock_app.run.reset_mock() - mock_app.config = {"HOST": "foo_host", "PORT": 42} + @mock.patch("availability.main.argparse.ArgumentParser") + @mock.patch("availability.main.config.process_args") + @mock.patch("availability.main.app.app") + def test_main_custom(self, mock_app, mock_process, mock_parser): + mock_process.return_value.configure_mock(**{ + "host": "foo_host", + "port": 42, + }) main.main() mock_app.run.assert_called_once_with(host="foo_host", port=42) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index ce16765..96aa025 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -20,8 +20,7 @@ from tests.unit import test -class StorageTestCase(test.TestCase): - +class EnsureIndexTestCase(test.TestCase): @mock.patch("availability.storage.json.dumps") @mock.patch("availability.storage.LOG") @mock.patch("availability.storage.get_elasticsearch") @@ -57,21 +56,30 @@ def test_ensure_es_index_exists( mock_es.indices.create.assert_called_once_with( body="foo_dumped_str", index="foo_index") + +class GetElasticTestCase(test.TestCase): + def setUp(self): + super(GetElasticTestCase, self).setUp() + self.mock_config({ + "backend": { + "type": "elastic", + "connection": [{"host": "node0"}], + }, + }) + @mock.patch("availability.storage.elasticsearch.Elasticsearch") - @mock.patch("availability.storage.config") @mock.patch("availability.storage.LOG") - def test_get_elasticsearch(self, mock_log, mock_config, mock_elastic): - mock_es = mock.Mock() - mock_es.indices.exists.side_effect = ValueError - mock_elastic.return_value = mock_es - mock_config.get_config.return_value = ( - {"backend": {"connection": "nodes"}}) - self.assertFalse(mock_es.info.called) - self.assertEqual(mock_es, storage.get_elasticsearch()) + def test_get_elasticsearch(self, mock_log, mock_elastic): + result = storage.get_elasticsearch() - mock_es.indices.exists.side_effect = None - mock_elastic.reset_mock() - self.assertEqual(mock_es, storage.get_elasticsearch( - check_availability=True)) - mock_elastic.assert_called_once_with("nodes") - mock_es.info.assert_called_once_with() + self.assertFalse(mock_elastic.return_value.info.called) + self.assertEqual(result, mock_elastic.return_value) + mock_elastic.assert_called_once_with([{"host": "node0"}]) + + @mock.patch("availability.storage.elasticsearch.Elasticsearch") + @mock.patch("availability.storage.LOG") + def test_get_elasticsearch_check(self, mock_log, mock_elastic): + result = storage.get_elasticsearch(check_availability=True) + + self.assertTrue(mock_elastic.return_value.info.called) + self.assertEqual(result, mock_elastic.return_value) diff --git a/tests/unit/test_watcher.py b/tests/unit/test_watcher.py index 3ed8d0e..f6f25ac 100644 --- a/tests/unit/test_watcher.py +++ b/tests/unit/test_watcher.py @@ -14,17 +14,21 @@ # under the License. import mock +from oss_lib import config import requests from availability import watcher from tests.unit import test +CONF = config.CONF + class WatcherTestCase(test.TestCase): @mock.patch("availability.watcher.requests.get") @mock.patch("availability.watcher.LOG") def test_check_availability(self, mock_log, mock_get): + self.mock_config() mock_queue = mock.Mock() mock_get.side_effect = ValueError watcher.check_availability({"url": "http://foo/"}, mock_queue) @@ -39,8 +43,8 @@ def test_check_availability(self, mock_log, mock_get): self.assertIsNone(result) mock_get.assert_called_once_with( "http://foo/", verify=False, - timeout=(watcher.SERVICE_CONN_TIMEOUT, - watcher.SERVICE_READ_TIMEOUT)) + timeout=(CONF["connection_timeout"], + CONF["read_timeout"])) mock_queue.put.assert_called_once_with({"url": "http://foo/", "status": 1}) @@ -55,19 +59,21 @@ def test_check_availability(self, mock_log, mock_get): @mock.patch("availability.watcher.storage") @mock.patch("availability.watcher.uuid") @mock.patch("availability.watcher.json.dumps") - @mock.patch("availability.watcher.config.get_config") @mock.patch("availability.watcher.LOG") - def test_save_availability(self, mock_log, mock_get_config, mock_dumps, - mock_uuid, mock_storage): - backend = {"type": "elastic", "connection": "foo_connection"} - mock_get_config.return_value = {"backend": backend} + def test_save_availability(self, mock_log, mock_dumps, mock_uuid, + mock_storage): + self.mock_config({ + "backend": { + "type": "elastic", + "connection": [{"host": "foo_host"}], + }, + }) + mock_queue = mock.Mock() queue_side_effect = [{"region": "foo"}, {"region": "bar"}, watcher.queue.Empty] mock_queue.get.side_effect = queue_side_effect - backend = {"type": "elastic", "connection": "foo_connection"} - mock_get_config.return_value = {"backend": backend} mock_dumps.side_effect = ["value-1", "value-2", "value-3", "value-4"] mock_es = mock.Mock() mock_storage.get_elasticsearch.return_value = mock_es @@ -102,12 +108,11 @@ def test_save_availability(self, mock_log, mock_get_config, mock_dumps, self.assertIsNone(watcher.save_availability(mock_queue)) @mock.patch("availability.watcher.dt") - @mock.patch("availability.watcher.config.get_config") @mock.patch("availability.watcher.threading.Thread") @mock.patch("availability.watcher.results_queue") @mock.patch("availability.watcher.LOG") def test_watch_services(self, mock_log, mock_queue, mock_thread, - mock_get_config, mock_dt): + mock_dt): mock_isoformat = mock.Mock() mock_isoformat.side_effect = ["time-1", "time-2"] mock_dt.datetime.now.return_value.isoformat = mock_isoformat @@ -116,9 +121,9 @@ def test_watch_services(self, mock_log, mock_queue, mock_thread, "services": [{"name": "a_svc", "url": "a_url"}]}, {"name": "b_reg", "services": [{"name": "b_svc", "url": "b_url"}]}]} - mock_get_config.return_value = regions + self.mock_config(regions) + watcher.watch_services() - mock_get_config.assert_called_once_with() calls = [ mock.call(args=({"url": "a_url", "region": "a_reg", "name": "a_svc", "timestamp": "time-1"}, @@ -134,39 +139,62 @@ def test_watch_services(self, mock_log, mock_queue, mock_thread, mock.call().start()] self.assertEqual(calls, mock_thread.mock_calls) + +class MainTestCase(test.TestCase): + BASE_CONF = { + "regions": [ + { + "services": [ + { + "name": "foo_name", + "url": "http://foo_url", + }, + ], + } + ], + } + UNEXPECTED_BACKEND_CONF = dict(BASE_CONF, **{ + "backend": { + "type": "unexpected", + "connection": [{"host": "foo_host"}], + }, + }) + ELASTIC_BACKEND_CONF = dict(BASE_CONF, **{ + "backend": { + "type": "elastic", + "connection": [{"host": "foo_host"}], + }, + }) + PERIOD_CONF = dict(ELASTIC_BACKEND_CONF, **{ + "period": 42, + }) + + @mock.patch("oss_lib.config.process_args") @mock.patch("availability.watcher.schedule") @mock.patch("availability.watcher.storage") @mock.patch("availability.watcher.watch_services") - @mock.patch("availability.watcher.config.get_config") @mock.patch("availability.watcher.time") @mock.patch("availability.watcher.LOG") - def test_main(self, mock_log, mock_time, mock_get_config, - mock_watch_services, mock_storage, mock_schedule): + def test_main(self, mock_log, mock_time, mock_watch_services, mock_storage, + mock_schedule, mock_process): class BreakInfinityCicle(Exception): pass run_effect = [None, None, None, BreakInfinityCicle] mock_schedule.run_pending.side_effect = run_effect - mock_get_config.return_value = {} - self.assertEqual(1, watcher.main()) - - mock_get_config.return_value = {"regions": []} + self.mock_config() self.assertEqual(1, watcher.main()) - mock_get_config.return_value = {"regions": ["foo_region"]} + self.mock_config(self.BASE_CONF) # NOTE(amaretskiy): # SERVICE_CONN_TIMEOUT + SERVICE_READ_TIMEOUT > 10 self.assertEqual(1, watcher.main(10)) - backend = {"type": "unexpected", "connection": "foo_conn"} - mock_get_config.return_value = {"regions": ["foo_region"], - "backend": backend} + self.mock_config(self.UNEXPECTED_BACKEND_CONF) self.assertEqual(1, watcher.main()) - backend = {"type": "elastic", "connection": "foo_conn"} - mock_get_config.return_value = {"regions": ["foo_region"], - "backend": backend} + self.mock_config(self.ELASTIC_BACKEND_CONF) mock_storage.get_elasticsearch.return_value = None self.assertEqual(1, watcher.main()) mock_storage.get_elasticsearch.assert_called_once_with( @@ -181,8 +209,7 @@ class BreakInfinityCicle(Exception): mock_storage.get_elasticsearch.assert_called_once_with( check_availability=True) - mock_get_config.return_value = {"regions": ["foo_region"], - "backend": backend, "period": 42} + self.mock_config(self.PERIOD_CONF) mock_schedule.reset_mock() mock_schedule.run_pending.side_effect = run_effect mock_time.reset_mock()