Merge pull request #25 from CodersOfTheNight/dockerized-integration-t…

…esting

Dockerized integration testing

.dockerignore

@@ -0,0 +1,2 @@
+*.pyc
+__pycache__/

.travis.yml

@@ -1,7 +1,11 @@
 language: python
 python:
 - 3.5
-- 3.6
+#- 3.6
+#- 3.7
+
+services:
+ - docker
 
 install:
  - python setup.py install

Makefile

@@ -0,0 +1,13 @@
+__PHONY__: all
+
+pull-containers:
+ docker-compose pull
+
+rebuild-compose:
+ docker-compose down && docker-compose build
+
+run-integration-tests: pull-containers rebuild-compose
+ docker-compose up --remove-orphans --abort-on-container-exit --exit-code-from oshino-query
+
+run-dev-environment: pull-containers rebuild-compose
+ docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans

README.md

@@ -28,7 +28,7 @@ How to install
 
 Quickstart
 ==========
-It is highly recommended for new users to use [Quickstart Guide](quickstart.md)
+It is highly recommended for new users to use [Quickstart Guide](docs/quickstart.md)
 
 
 Riemann. What? Why? How?

compose/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.7-alpine
+
+WORKDIR '/usr/src/oshino'
+
+COPY oshino ./oshino
+COPY requirements ./requirements
+COPY tests ./tests
+COPY README.md ./README.md
+COPY setup.py .
+
+COPY compose/config.yaml .
+
+
+RUN python setup.py install 
+RUN pip install -r ./requirements/test.txt
+
+CMD ["/usr/local/bin/oshino", "--config=config.yaml"]

compose/config.yaml

@@ -0,0 +1,11 @@
+---
+interval: 5
+loglevel: DEBUG
+riemann:
+ host: riemann
+ port: 5555
+agents:
+ - name: health-check
+ module: oshino.agents.http_agent.HttpAgent
+ url: https://www.python.org
+ tag: healthcheck

docker-compose.dev.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+ oshino-query:
+ command: ["/bin/sh"]
+ riemann:
+ ports:
+ - "5555:5555"
+ - "5556:5556"
+ riemann-ui:
+ image: "rlister/riemann-dash"
+ depends_on:
+ - riemann
+ ports:
+ - "4567:4567"

docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+ oshino:
+ build:
+ context: .
+ dockerfile: compose/Dockerfile
+ depends_on:
+ - riemann
+ oshino-query:
+ build:
+ context: .
+ dockerfile: compose/Dockerfile
+ command: ["/usr/local/bin/pytest", "tests/it_query.py"]
+ depends_on:
+ - oshino
+ - riemann
+ riemann:
+ image: "rlister/riemann"

oshino/agents/__init__.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 
-from oshino.util import current_ts
+from oshino.util import current_ts, timer
 
 
 class Agent(ABC):
@@ -37,15 +37,20 @@ async def process(self, event_fn):
  def is_valid(self):
  return "name" in self._data
 
- async def pull_metrics(self, event_fn):
+ async def pull_metrics(self, event_fn, loop=None):
  """
  Method called by core.
  Should not be overwritten.
  """
  if self.lazy and not self.ready:
  return None
+ logger = self.get_logger()
 
+ ts = timer()
+ logger.trace("Waiting for process event")
  result = await self.process(event_fn)
+ td = int(timer() - ts)
+ logger.trace("It took: {}ms".format(td))
  self._last_run = current_ts()
  return result
 

oshino/agents/http_agent.py

@@ -1,6 +1,6 @@
 import aiohttp
 
-from time import time
+from ..util import timer
 from . import Agent
 
 
@@ -46,13 +46,13 @@ def translate_status(self, code):
 
  async def process(self, event_fn):
  logger = self.get_logger()
- ts = time()
+ ts = timer()
  state = None
  async with aiohttp.ClientSession() as session:
  async with session.get(self.url) as resp:
  state = self.translate_status(resp.status)
- te = time()
- span = int((te - ts) * 1000)
+ te = timer()
+ span = int(te - ts)
  logger.debug("Request to {url} returned status code {code}(as {state})"
  "in {span} milliseconds.".format(url=self.url,
  code=resp.status,

oshino/config.py

@@ -78,10 +78,14 @@ def default():
  def ca_certs(self):
  return self._data.get("ca-certs", None)
 
- def transport(self, noop=False):
+ def get_transport(self, noop=False):
  if noop:
  return BlankTransport
+ else:
+ return self.transport
 
+ @property
+ def transport(self):
  raw = self._data.get("transport", None)
  if raw: # Transport is defined
  return getattr(riemann_client.transport, raw)

oshino/core/heart.py

@@ -2,7 +2,6 @@
 import sys
 import logbook
 
-from time import time
 from typing import TypeVar, Generic
 from asyncio import BaseEventLoop
 
@@ -14,6 +13,7 @@
 
 from ..config import Config
 from ..version import get_version
+from ..util import timer
 from . import (send_heartbeat,
  send_timedelta,
  send_pending_events_count,
@@ -76,12 +76,17 @@ def event_fn(**kwargs):
  kwargs["tags"] = tags
 
  if "time" not in kwargs:
- kwargs["time"] = int(time())
+ kwargs["time"] = int(timer())
 
  client.event(**kwargs)
 
- tasks.append(agent.pull_metrics(event_fn))
- return await asyncio.wait(tasks, timeout=timeout)
+ tasks.append(
+ loop.create_task(
+ agent.pull_metrics(event_fn, loop=loop)
+ )
+ )
+ (done, pending) = await asyncio.wait(tasks, timeout=timeout, loop=loop)
+ return (done, pending)
 
 
 def instrumentation(client: processor.QClient,
@@ -90,10 +95,27 @@ def instrumentation(client: processor.QClient,
  delta: int,
  events_count: int,
  pending_events: int):
- send_heartbeat(client.event, logger, int(interval * 1.5))
- send_timedelta(client.event, logger, delta, interval)
- send_metrics_count(client.event, logger, events_count)
- send_pending_events_count(client.event, logger, events_count)
+ send_heartbeat(
+ event_fn=client.event,
+ logger=logger,
+ ttl=int(interval * 1.5)
+ )
+ send_timedelta(
+ event_fn=client.event,
+ logger=logger,
+ td=delta,
+ interval=interval
+ )
+ send_metrics_count(
+ event_fn=client.event,
+ logger=logger,
+ count=events_count
+ )
+ send_pending_events_count(
+ event_fn=client.event,
+ logger=logger,
+ count=events_count
+ )
 
 
 async def main_loop(cfg: Config,
@@ -109,23 +131,28 @@ async def main_loop(cfg: Config,
  executor = cfg.executor_class(max_workers=cfg.executors_count)
  loop.set_default_executor(executor)
 
+ def handle_async_errors(loop, ctx):
+ logger.error("Received error on async loop: {}"
+ .format(ctx["exception"]))
+
  init(agents)
 
  while True:
- ts = time()
+ ts = timer()
  (done, pending) = await step(client,
  agents,
- timeout=cfg.interval * 1.5,
+ timeout=cfg.interval,
  loop=loop)
+ logger.debug("Pending tasks: {}".format(pending))
 
- te = time()
- td = te - ts
- instrumentation(client,
- logger,
- cfg.interval,
- td,
- len(client.queue.events),
- len(pending))
+ te = timer()
+ td = int(te - ts)
+ instrumentation(client=client,
+ logger=logger,
+ interval=cfg.interval,
+ delta=td,
+ events_count=len(client.queue.events),
+ pending_events=len(pending))
 
  await processor.flush(client, transport, logger)
  if continue_fn():
@@ -161,7 +188,7 @@ def start_loop(cfg: Config, noop=False):
  try:
  loop.run_until_complete(main_loop(cfg,
  logger,
- cfg.riemann.transport(noop),
+ cfg.riemann.get_transport(noop),
  forever,
  loop=loop))
  finally:

oshino/run.py

@@ -1,9 +1,16 @@
 import click
 from dotenv import load_dotenv, find_dotenv
+from logbook import Logger
 from .config import load
 from .core.heart import start_loop
 
-load_dotenv(find_dotenv())
+
+logger = Logger("Runner")
+
+try:
+ load_dotenv(find_dotenv())
+except Exception as ex:
+ logger.error("Error while loading .env: '{}'. Ignoring.".format(ex))
 
 
 @click.command()

oshino/util.py

@@ -1,3 +1,6 @@
+import sys
+import time
+
 from datetime import datetime
 
 
@@ -12,3 +15,15 @@ def current_ts():
  """
  utcnow = datetime.utcnow()
  return int(utcnow.timestamp() * 1000)
+
+
+def timer():
+ """
+ Timer used for calculate time elapsed
+ """
+ if sys.platform == "win32":
+ default_timer = time.clock
+ else:
+ default_timer = time.time
+
+ return default_timer()

oshino/version.py

@@ -3,7 +3,7 @@
 That's all.
 """
 
-VERSION = (0, 3, "0a5")
+VERSION = (0, 3, "0")
 
 
 def get_version():

requirements/release.txt

@@ -1,13 +1,11 @@
-aiohttp==1.0.5
-async-timeout==1.1.0
+aiohttp~=3.4.0
 click==6.7
 coverage==4.2
-Jinja2==2.8
+Jinja2
 logbook==1.0.0
 MarkupSafe==0.23
-multidict==2.1.2
 protobuf==3.1.0.post1
-python-dotenv==0.6.0
+python-dotenv==0.9.1
 PyYAML==3.12
 riemann-client==6.3.0
 six==1.10.0

requirements/test.txt

@@ -10,3 +10,4 @@ flask
 coveralls
 logbook
 py
+wait

setup.py

@@ -1,7 +1,10 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
 from setuptools import setup, find_packages
-from pip.req import parse_requirements
+try:
+ from pip._internal.req import parse_requirements
+except:
+ from pip.req import parse_requirements
 
 from oshino.version import get_version
 

tests/data/it_config.yml

@@ -0,0 +1,6 @@
+---
+interval: 10
+loglevel: DEBUG
+riemann:
+ host: localhost
+ port: 5555