Use oss_lib.onfig for configuration

akscram · akscram · commit e4e1ee3b9577 · 2016-12-27T16:30:55.000+04:00
diff --git a/availability/api/v1/api.py b/availability/api/v1/api.py
@@ -17,11 +17,9 @@
 
 import flask
 
-from availability import config
 from availability import storage
 
-
-LOG = logging.getLogger("api")
+LOG = logging.getLogger(__name__)
 
 
 def get_period_interval(period):
diff --git a/availability/app.py b/availability/app.py
@@ -14,7 +14,7 @@
 #    under the License.
 
 import flask
-from flask_helpers import routing
+from oss_lib import routing
 
 from availability.api.v1 import api
 from availability.api.v1 import regions
diff --git a/availability/config.py b/availability/config.py
@@ -13,94 +13,68 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
-import logging
-import os
+DEFAULT_CONF_PATH = "/etc/availability/config.json"
 
-import jsonschema
-
-
-CONF = None
-
-DEFAULT_CONF = {
+DEFAULT = {
     "backend": {
         "type": "elastic",
-        "connection": [{"host": "127.0.0.1", "port": 9200}]
+        "connection": [
+            {"host": "127.0.0.1", "port": 9200},
+        ],
     },
-    "regions": []
+    "regions": [],
+    "period": 60,
+    "connection_timeout": 1,
+    "read_timeout": 10,
 }
 
-CONF_SCHEMA = {
-    "type": "object",
-    "$schema": "http://json-schema.org/draft-04/schema",
-    "properties": {
-        "backend": {
+SCHEMA = {
+    "backend": {
+        "type": "object",
+        "properties": {
+            "type": {"type": "string"},
+            "connection": {
+                "type": "array",
+                "items": {
+                    # TODO(akscram): Here should be enum.
+                    "type": "object",
+                    "properties": {
+                        "host": {"type": "string"},
+                        "port": {"type": "integer"},
+                    },
+                    "required": ["host"],
+                    "additionalProperties": False,
+                },
+                "minItems": 1,
+            },
+        },
+        "required": ["type", "connection"],
+        "additionalProperties": False,
+    },
+    "regions": {
+        "type": "array",
+        "items": {
             "type": "object",
             "properties": {
-                "type": {"type": "string"},
-                "connection": {
+                "name": {"type": "string"},
+                "services": {
                     "type": "array",
                     "items": {
                         "type": "object",
                         "properties": {
-                            "host": {"type": "string"},
-                            "port": {"type": "integer"}
+                            "name": {"type": "string"},
+                            "url": {"type": "string"}
                         },
-                        "required": ["host"]
+                        "required": ["name", "url"],
+                        "additionalProperties": False,
                     },
-                    "minItems": 1
-                }
+                    "minItems": 1,
+                },
             },
-            "required": ["type", "connection"]
-        },
-        "regions": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "name": {"type": "string"},
-                    "services": {
-                        "type": "array",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "name": {"type": "string"},
-                                "url": {"type": "string"}
-                            },
-                            "required": ["name", "url"]
-                        },
-                        "minItems": 1
-                    }
-                }
-            }
+            "additionalProperties": False,
         },
-        "period": {"type": "number", "minimum": 5},
-        "connection_timeout": {"type": "number"},
-        "read_timeout": {"type": "number"},
     },
-    "required": ["backend", "regions"]
+    "period": {"type": "number", "minimum": 5},
+    "connection_timeout": {"type": "number"},
+    "read_timeout": {"type": "number"},
 }
-
-
-def get_config():
-    """Get cached configuration.
-
-    :returns: application config
-    :rtype: dict
-    """
-    global CONF
-    if not CONF:
-        path = os.environ.get("AVAILABILITY_CONF",
-                              "/etc/availability/config.json")
-        try:
-            config = json.load(open(path))
-            logging.info("Config is '%s'" % path)
-            jsonschema.validate(config, CONF_SCHEMA)
-            CONF = config
-        except IOError as exc:
-            logging.warning("Failed to load config from '%s': %s", path, exc)
-            CONF = DEFAULT_CONF
-        except jsonschema.exceptions.ValidationError as exc:
-            logging.error("Configuration file %s is not valid: %s", path, exc)
-            raise
-    return CONF
diff --git a/availability/main.py b/availability/main.py
@@ -15,12 +15,13 @@
 
 import argparse
 
+from oss_lib import config
+
 from availability import app
-from availability import config
+from availability import config as cfg
 
 
 def main():
-    config.get_config()
     parser = argparse.ArgumentParser()
     parser.add_argument("--host",
                         default="0.0.0.0",
@@ -31,5 +32,10 @@ def main():
                         default=5000,
                         help="A port to bind development server. "
                              "(default 5000)")
-    args = parser.parse_args()
+    args = config.process_args("AVAILABILITY",
+                               parser=parser,
+                               default_config_path=cfg.DEFAULT_CONF_PATH,
+                               defaults=cfg.DEFAULT,
+                               validation_schema=cfg.SCHEMA)
+    app.app.config.update(config.CONF, **{"DEBUG": args.debug})
     app.app.run(host=args.host, port=args.port)
diff --git a/availability/storage.py b/availability/storage.py
@@ -17,10 +17,11 @@
 import logging
 
 import elasticsearch
+from oss_lib import config
 
-from availability import config
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
 
-LOG = logging.getLogger("storage")
 NUMBER_OF_SHARDS = 2
 
 
@@ -31,7 +32,7 @@ def get_elasticsearch(check_availability=False):
     :returns: Elasticsearch or None on failure
     :rtype: elasticsearch.Elasticsearch
     """
-    nodes = config.get_config()["backend"]["connection"]
+    nodes = CONF["backend"]["connection"]
     try:
         es = elasticsearch.Elasticsearch(nodes)
         if check_availability:
diff --git a/availability/watcher.py b/availability/watcher.py
@@ -22,18 +22,15 @@
 import uuid
 
 from elasticsearch import exceptions as es_exceptions
+from oss_lib import config
 import queue
 import requests
 import schedule
 
-from availability import config
 from availability import storage
 
-
-SERVICE_CONN_TIMEOUT = config.get_config().get("connection_timeout", 1)
-SERVICE_READ_TIMEOUT = config.get_config().get("read_timeout", 10)
-
-LOG = logging.getLogger("watcher")
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
 
 
 def check_availability(data, results_queue):
@@ -44,8 +41,9 @@ def check_availability(data, results_queue):
     :rtype: None
     """
     try:
-        requests.get(data["url"], verify=False, timeout=(SERVICE_CONN_TIMEOUT,
-                                                         SERVICE_READ_TIMEOUT))
+        requests.get(data["url"], verify=False,
+                     timeout=(CONF["connection_timeout"],
+                              CONF["read_timeout"]))
 
         data["status"] = 1
     except Exception as e:
@@ -104,7 +102,7 @@ def save_availability(results_queue):
 
 def watch_services():
     """Query services of all regions and save results."""
-    for region in config.get_config().get("regions"):
+    for region in CONF["regions"]:
         for service in region.get("services"):
             LOG.info("Checking service '%(name)s' availability on %(url)s"
                      % service)
@@ -130,18 +128,18 @@ def main(period=None):
     This runs infinite availability check with given period.
     :param period: period in seconds
     """
-    if not config.get_config().get("regions"):
+    if not CONF["regions"]:
         LOG.error("No regions configured. Quitting.")
         return 1
 
-    period = period or config.get_config().get("period", 60)
+    period = period or CONF["period"]
 
-    if SERVICE_CONN_TIMEOUT + SERVICE_READ_TIMEOUT > period:
+    if CONF["connection_timeout"] + CONF["read_timeout"] > period:
         LOG.error("Period can not be lesser than timeout, "
                   "otherwise threads could crowd round.")
         return 1
 
-    backend = config.get_config().get("backend")
+    backend = CONF["backend"]
     if backend["type"] != "elastic":
         LOG.error("Unexpected backend: %(type)s" % backend)
         return 1
@@ -156,10 +154,3 @@ def main(period=None):
     while True:
         time.sleep(1)
         schedule.run_pending()
-
-
-if __name__ == "__main__":
-    try:
-        sys.exit(main())
-    except KeyboardInterrupt:
-        LOG.error("Got SIGINT. Quitting...")
diff --git a/availability/wsgi.py b/availability/wsgi.py
@@ -13,10 +13,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oss_lib import config
 
 from availability import app
-from availability import config
+from availability import config as cfg
 
-config.get_config()
+
+config.process_env("AVAILABILITY",
+                   default_config_path=cfg.DEFAULT_CONF_PATH,
+                   defaults=cfg.DEFAULT,
+                   validation_schema=cfg.SCHEMA)
 
 application = app.app
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 gunicorn==19.6.0
 Flask==0.11.1
-flask-helpers==0.1
+oss-lib==0.1.1
 requests==2.11.1
 jsonschema==2.5.1
 future==0.16.0
diff --git a/tests/ci/api/populate_elastic.py b/tests/ci/api/populate_elastic.py
@@ -15,11 +15,13 @@
 
 import datetime as dt
 import json
-import os
 import random
 import sys
 import uuid
 
+from oss_lib import config
+
+from availability import config as cfg
 from availability import storage
 
 
@@ -57,8 +59,10 @@ def random_status(success_probability=0.7):
 
 def populate_elastic(**kwargs):
     started_at = dt.datetime.now()
-    conf = json.load(open(os.environ.get("AVAILABILITY_CONF")))
-    regions = {r["name"]: r["services"] for r in conf["regions"]}
+    config.process_env("AVAILABILITY",
+                       defaults=cfg.DEFAULT,
+                       validation_schema=cfg.SCHEMA)
+    regions = {r["name"]: r["services"] for r in config.CONF["regions"]}
     elastic = storage.get_elasticsearch(check_availability=True)
 
     # Create indices
diff --git a/tests/unit/api/v1/test_api.py b/tests/unit/api/v1/test_api.py
@@ -15,6 +15,7 @@
 
 import mock
 
+from availability.api.v1 import api
 from tests.unit import test
 
 
@@ -74,3 +75,37 @@ def test_get_region_availability(self, mock_process_results,
             mock_storage.es_search.reset_mock()
             mock_process_results.assert_called_once_with("foo_buckets")
             mock_process_results.reset_mock()
+
+
+# TODO(akscram): Just to pass coverage, re-write this test completely.
+class ResultsTestCase(test.TestCase):
+    def test_process_results(self):
+        buckets = [{
+            "key": "key_av",
+            "availability": {"value": 1},
+            "data": {
+                "buckets": [
+                    {
+                        "availability": {"value": 1},
+                        "key_as_string": "key_1",
+                    },
+                    {
+                        "availability": {"value": 1},
+                        "key_as_string": "key_2",
+                    },
+                ],
+            },
+        }]
+        expected = {
+            "availability": {
+                "key_av": {
+                    "availability": 1,
+                    "availability_data": [
+                        ["key_1", 1],
+                        ["key_2", 1],
+                    ],
+                },
+            },
+        }
+        result = api.process_results(buckets)
+        self.assertEqual(result, expected)
diff --git a/tests/unit/test.py b/tests/unit/test.py
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py
diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py
diff --git a/tests/unit/test_watcher.py b/tests/unit/test_watcher.py
