Support for dev definitions library (#358)

* Support for dev definitions library
Co-authored-by: Anton Benkevich <[email protected]>

Author: anton-b

.travis.yml

@@ -19,6 +19,11 @@ jobs:
     install:
       - pip install ./
     script: python scripts/validate_my_definition.py -d alsdkdefs/apis/assets_query/
+# TODO test private defs
+#  - stage: test_private_defs
+#    name: "Test private definitions"
+#    install: pip install -U tox-travis
+#    script: git clone https://[email protected]/alertlogic/alertlogic-sdk-definitions-dev && pip install ./alertlogic-sdk-definitions-dev && tox
 #  - stage: test_other_os
 #    language: sh
 #    python: 3

README.md

@@ -24,7 +24,7 @@ List available service definitions:
 ```
 >>> import alsdkdefs
 >>> alsdkdefs.list_services()
-['aefr', 'aerta', 'aetag', 'aetuner', 'aims', 'assets_query', 'kelt', 'credentials', 'deployments', 'ingest', 'iris', 'policies', 'search', 'themis']
+OrderedDict([('aecontent', ServiceDefinition(aecontent)), ('aefr', ServiceDefinition(aefr)), ('aepublish', ServiceDefinition(aepublish)), ('aerta', ServiceDefinition(aerta)), ('aetag', ServiceDefinition(aetag)), ('aetuner', ServiceDefinition(aetuner)), ('aims', ServiceDefinition(aims)), ('assets_query', ServiceDefinition(assets_query)), ('assets_write', ServiceDefinition(assets_write)), ('connectors', ServiceDefinition(connectors)), ('credentials', ServiceDefinition(credentials)), ('deployments', ServiceDefinition(deployments)), ('herald', ServiceDefinition(herald)), ('ingest', ServiceDefinition(ingest)), ('iris', ServiceDefinition(iris)), ('kalm', ServiceDefinition(kalm)), ('notify', ServiceDefinition(notify)), ('otis', ServiceDefinition(otis)), ('policies', ServiceDefinition(policies)), ('remediations', ServiceDefinition(remediations)), ('responder', ServiceDefinition(responder)), ('search', ServiceDefinition(search)), ('subscriptions', ServiceDefinition(subscriptions)), ('themis', ServiceDefinition(themis))])
 ```
 
 Get path to a service definitions paths:

alsdkdefs/__init__.py

@@ -2,6 +2,7 @@
 from os.path import join as pjoin
 import glob
 import collections
+from collections import OrderedDict
 import jsonschema
 from urllib.parse import urlparse, urlsplit, urlunsplit
 from urllib.request import urlopen, url2pathname
@@ -12,10 +13,43 @@
 import json
 import re
 
+try:
+    import alsdkdefs_dev
+    DEV_SDK_DEFS = True
+except ImportError:
+    DEV_SDK_DEFS = False
+
 OPENAPI_SCHEMA_URL = 'https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/schemas/v3.0/schema.json'
 URI_SCHEMES = ['file', 'http', 'https']
 
 
+class ServiceDefinition:
+    def __init__(self, name, filespath):
+        self.service_name = name
+        self.filespath = filespath
+
+    def __str__(self):
+        return self.service_name
+
+    def __repr__(self):
+        return f"ServiceDefinition({self.service_name})"
+
+    def __gt__(self, other):
+        return self.get_service_name() > other.get_service_name()
+
+    def __eq__(self, other):
+        return other.get_service_name() == self.get_service_name()
+
+    def __lt__(self, other):
+        return self.get_service_name() < other.get_service_name()
+
+    def get_service_name(self):
+        return self.service_name
+
+    def get_files_path(self):
+        return self.filespath
+
+
 class AlertLogicOpenApiValidationException(Exception):
     pass
 
@@ -116,7 +150,11 @@ def get_apis_dir():
 
 def load_service_spec(service_name, apis_dir=None, version=None):
     """Loads a version of service from library apis directory, if version is not specified, latest is loaded"""
-    service_api_dir = pjoin(apis_dir or get_apis_dir(), service_name)
+    services = list_services()
+    servicedef = services.get(service_name)
+    if not servicedef:
+        raise FileNotFoundError(f'Service {service_name} definition files has not been found')
+    service_api_dir = servicedef.get_files_path() or apis_dir
     if not version:
         # Find the latest version of the service api spes
         version = 0
@@ -152,16 +190,23 @@ def get_spec(uri):
             return json.loads(stream.read())
 
 
+@lru_cache()
 def list_services():
     """Lists services definitions available"""
     base_dir = get_apis_dir()
-    return sorted(next(os.walk(base_dir))[1])
+    dev_services = []
+    if DEV_SDK_DEFS:
+        dev_dirs = alsdkdefs_dev.get_apis_dir()
+        dev_services = [ServiceDefinition(s, pjoin(dev_dirs, s)) for s in next(os.walk(dev_dirs))[1]]
+    pub_services = [ServiceDefinition(s, pjoin(base_dir, s)) for s in next(os.walk(base_dir))[1]]
+    services_search = OrderedDict(sorted([(str(servicedef), servicedef) for servicedef in pub_services + dev_services]))
+    return services_search
 
 
 def get_service_defs(service_name):
     """Lists a service's definitions available"""
-    service_dir = pjoin(get_apis_dir(), service_name)
-    return glob.glob(f"{service_dir}/{service_name}.v*.yaml")
+    service = list_services()[service_name]
+    return glob.glob(pjoin(service.get_files_path(), f"{service_name}.v*.yaml"))
 
 
 @lru_cache()