Merge pull request #26 from CodersOfTheNight/0.3

0.3

Author: zaibacu

.dockerignore

@@ -0,0 +1,2 @@
+*.pyc
+__pycache__/

.travis.yml

@@ -1,10 +1,14 @@
 language: python
 python:
 - 3.5
-- 3.6
+#- 3.6
+#- 3.7
+
+services:
+  - docker
 
 install:
-  - pip install -r requirements/release.txt
+  - python setup.py install
   - pip install -r requirements/test.txt
 
 script:

Makefile

@@ -0,0 +1,13 @@
+__PHONY__: all
+
+pull-containers:
+	docker-compose pull
+
+rebuild-compose:
+	docker-compose down && docker-compose build
+
+run-integration-tests: pull-containers rebuild-compose
+	docker-compose up --remove-orphans --abort-on-container-exit --exit-code-from oshino-query
+
+run-dev-environment: pull-containers rebuild-compose
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans

compose/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.7-alpine
+
+WORKDIR '/usr/src/oshino'
+
+COPY oshino ./oshino
+COPY requirements ./requirements
+COPY tests ./tests
+COPY README.md ./README.md
+COPY setup.py .
+
+COPY compose/config.yaml .
+
+
+RUN python setup.py install 
+RUN pip install -r ./requirements/test.txt
+
+CMD ["/usr/local/bin/oshino", "--config=config.yaml"]

compose/config.yaml

@@ -0,0 +1,11 @@
+---
+interval: 5
+loglevel: DEBUG
+riemann:
+  host: riemann
+  port: 5555
+agents:
+  - name: health-check
+    module: oshino.agents.http_agent.HttpAgent
+    url: https://www.python.org
+    tag: healthcheck

docker-compose.dev.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+  oshino-query:
+    command: ["/bin/sh"]
+  riemann:
+    ports:
+      - "5555:5555"
+      - "5556:5556"
+  riemann-ui:
+    image: "rlister/riemann-dash"
+    depends_on:
+      - riemann
+    ports:
+      - "4567:4567"

docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  oshino:
+    build:
+      context: .
+      dockerfile: compose/Dockerfile
+    depends_on:
+      - riemann
+  oshino-query:
+    build:
+      context: .
+      dockerfile: compose/Dockerfile
+    command: ["/usr/local/bin/pytest", "tests/it_query.py"]
+    depends_on:
+      - oshino
+      - riemann
+  riemann:
+    image: "rlister/riemann"

oshino/agents/__init__.py

@@ -1,7 +1,13 @@
-class Agent(object):
+from abc import ABC, abstractmethod
+
+from oshino.util import current_ts, timer
+
+
+class Agent(ABC):
 
     def __init__(self, cfg):
         self._data = cfg
+        self._last_run = current_ts()
 
     def on_start(self):
         pass
@@ -21,8 +27,60 @@ def get_logger(self):
         from logbook import Logger
         return Logger(self.__class__.__name__)
 
+    @abstractmethod
     async def process(self, event_fn):
+        """
+        Each agent must implement this one to provide actual logic
+        """
         pass
 
     def is_valid(self):
         return "name" in self._data
+
+    async def pull_metrics(self, event_fn, loop=None):
+        """
+        Method called by core.
+        Should not be overwritten.
+        """
+        if self.lazy and not self.ready:
+            return None
+        logger = self.get_logger()
+
+        ts = timer()
+        logger.trace("Waiting for process event")
+        result = await self.process(event_fn)
+        td = int(timer() - ts)
+        logger.trace("It took: {}ms".format(td))
+        self._last_run = current_ts()
+        return result
+
+    @property
+    def lazy(self):
+        """
+        Agents with flag `lazy` gives data when they want to,
+        not when they are requested for.
+        """
+        return self._data.get("lazy", False)
+
+    @property
+    def ready(self):
+        """
+        Function used when agent is `lazy`.
+        It is being processed only when `ready` condition is satisfied
+        """
+        logger = self.get_logger()
+        now = current_ts()
+        logger.trace("Current time: {0}".format(now))
+        logger.trace("Last Run: {0}".format(self._last_run))
+        delta = (now - self._last_run)
+        logger.trace("Delta: {0}, Interval: {1}"
+                     .format(delta, self.interval * 1000))
+        return delta > self.interval * 1000
+
+    @property
+    def interval(self):
+        """
+        By default, lazy agents expects to have some kind of time interval.
+        `ready` output is calculated according to this interval.
+        """
+        return self._data["interval"]

oshino/agents/http_agent.py

@@ -1,6 +1,6 @@
 import aiohttp
 
-from time import time
+from ..util import timer
 from . import Agent
 
 
@@ -46,13 +46,13 @@ def translate_status(self, code):
 
     async def process(self, event_fn):
         logger = self.get_logger()
-        ts = time()
+        ts = timer()
         state = None
         async with aiohttp.ClientSession() as session:
             async with session.get(self.url) as resp:
                 state = self.translate_status(resp.status)
-        te = time()
-        span = int((te - ts) * 1000)
+        te = timer()
+        span = int(te - ts)
         logger.debug("Request to {url} returned status code {code}(as {state})"
                      "in {span} milliseconds.".format(url=self.url,
                                                       code=resp.status,

oshino/agents/subprocess_agent.py

@@ -1,4 +1,5 @@
 import re
+import json
 import asyncio
 
 from functools import partial
@@ -49,6 +50,11 @@ def regex_transform(m, logger):
         return None
 
 
+def json_transform(m, logger, selector):
+    raw = json.loads(m)
+    return selector, raw[selector]
+
+
 def is_parsed(m):
     return m is not None