Resurrect built-in histograms (#11)

* resurrect histodraw.py from the depths

* refactor histodraw to get its data from Prometheus

* fix text rendering

* inline base

* simplify the graph rendering code by generating a single-channel image and scaling it

* split the rendering part into its own function for reusability

* move into src

* add HTTP view to render histograms

* add more sophisticated config, target search and response codes

* check for unknown env vars

* add link to the graphs to the main UI

* only display graph buttons if Prometheus is configured

* add description and example of the built-in heatmap feature to the README

Dockerfile

@@ -16,14 +16,19 @@ RUN cd /opt/meshping/oping-py && python3 setup.py build && python3 setup.py inst
 
 FROM alpine:latest
 
-RUN apk add --no-cache python3 liboping bash py3-netifaces~=0.10.9 dumb-init
+RUN apk add --no-cache python3 liboping bash py3-netifaces~=0.10.9 py3-pillow dumb-init ttf-dejavu
 COPY requirements.txt /opt/meshping/requirements.txt
-RUN pip3 install -r /opt/meshping/requirements.txt
+RUN pip3 install --no-cache-dir -r /opt/meshping/requirements.txt
+
+RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
+RUN apk add --no-cache py3-pandas
+
 
 WORKDIR /opt/meshping
 COPY --from=0 /opt/meshping/ui/node_modules/jquery/dist/jquery.slim.min.js            /opt/meshping/ui/node_modules/jquery/dist/
 COPY --from=0 /opt/meshping/ui/node_modules/bootstrap/dist/css/bootstrap.min.css      /opt/meshping/ui/node_modules/bootstrap/dist/css/
 COPY --from=0 /opt/meshping/ui/node_modules/bootstrap/dist/js/bootstrap.bundle.min.js /opt/meshping/ui/node_modules/bootstrap/dist/js/
+COPY --from=0 /opt/meshping/ui/node_modules/bootstrap-icons/icons/graph-up.svg        /opt/meshping/ui/node_modules/bootstrap-icons/icons/
 COPY --from=0 /opt/meshping/ui/node_modules/bootstrap-icons/icons/trash.svg           /opt/meshping/ui/node_modules/bootstrap-icons/icons/
 COPY --from=0 /opt/meshping/ui/node_modules/vue/dist/vue.min.js                       /opt/meshping/ui/node_modules/vue/dist/
 COPY --from=0 /opt/meshping/ui/node_modules/vue-resource/dist/vue-resource.min.js     /opt/meshping/ui/node_modules/vue-resource/dist/

README.md

@@ -124,6 +124,29 @@ Otherwise you'll lose the heatmap effect because every data point will be its ow
 In the examples directory, there's also a [json dashboard definition](examples/grafana.json) that you can import.
 
 
+## Built-in heatmaps
+
+Meshping also has a built-in heatmaps feature. These do not look as pretty as the ones from Grafana, but I find they are better readable and more useful. They look like this:
+
+![built-in heatmap](examples/heatmap4.png)
+
+To enable these, configure the `MESHPING_PROMETHEUS_URL` environment variable to point to your prometheus instance, like so:
+
+```
+MESHPING_PROMETHEUS_URL="http://192.168.0.1:9090"
+```
+
+Then Meshping will enable buttons in the UI to view these graphs.
+
+The default query that Meshping sends to Prometheus to fetch the data is this one:
+
+```
+increase(meshping_pings_bucket{instance="%(pingnode)s",name="%(name)s",target="%(addr)s"}[1h])
+```
+
+Names get substituted with the respective values before sending the query. If you need to modify it, you can override it via the `MESHPING_PROMETHEUS_QUERY` environment variable.
+
+
 # Deploying
 
 Deploying meshping is easiest using `docker-compose`:

src/api.py

@@ -3,15 +3,24 @@
 # pylint: disable=unused-variable
 
 import socket
+import os
+import httpx
 
-from quart  import Response, render_template, request, jsonify, send_from_directory
+from time   import time
+from io     import BytesIO
+from quart  import Response, render_template, request, jsonify, send_from_directory, send_file, abort
 
 from ifaces import Ifaces4
+from histodraw import render
 
 def add_api_views(app, mp):
     @app.route("/")
     async def index():
-        return await render_template("index.html", Hostname=socket.gethostname())
+        return await render_template(
+            "index.html",
+            Hostname=socket.gethostname(),
+            HaveProm=("true" if "MESHPING_PROMETHEUS_URL" in os.environ else "false"),
+        )
 
     @app.route("/metrics")
     async def metrics():
@@ -57,7 +66,7 @@ async def metrics():
             buckets = sorted(histogram.keys(), key=float)
             count = 0
             for bucket in buckets:
-                nextping = 2 ** ((bucket + 1) / 10.) - 0.01
+                nextping = 2 ** ((bucket + 1) / 10.)
                 count += histogram[bucket]
                 respdata.append(
                     'meshping_pings_bucket{name="%(name)s",target="%(addr)s",le="%(le).2f"} %(count)d' % dict(
@@ -193,3 +202,42 @@ async def edit_target(target):
     async def clear_stats():
         mp.clear_stats()
         return jsonify(success=True)
+
+    @app.route("/histogram/<node>/<target>.png")
+    async def histogram(node, target):
+        prom_url = os.environ.get("MESHPING_PROMETHEUS_URL")
+        if prom_url is None:
+            abort(503)
+
+        prom_query = os.environ.get(
+            "MESHPING_PROMETHEUS_QUERY",
+            'increase(meshping_pings_bucket{instance="%(pingnode)s",name="%(name)s",target="%(addr)s"}[1h])'
+        )
+
+        if "@" in target:
+            name, addr = target.split("@")
+        else:
+            for _, name, addr in mp.iter_targets():
+                if name == target or addr == target:
+                    break
+            else:
+                abort(400)
+
+        async with httpx.AsyncClient() as client:
+            response = (await client.get(prom_url + "/api/v1/query_range", timeout=2, params={
+                "query": prom_query % dict(pingnode=node, name=name, addr=addr),
+                "start": time() - 3 * 24 * 60 * 60,
+                "end":   time(),
+                "step":  3600,
+            })).json()
+
+        if response["status"] != "success":
+            abort(500)
+        if not response["data"]["result"]:
+            abort(404)
+
+        img = render(response)
+        img_io = BytesIO()
+        img.save(img_io, 'png')
+        img_io.seek(0)
+        return await send_file(img_io, mimetype='image/png')

src/histodraw.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# kate: space-indent on; indent-width 4; replace-tabs on;
+
+import sys
+import math
+
+from time import time
+from datetime import datetime
+
+import pandas
+
+from PIL import Image, ImageDraw, ImageFont
+
+def render(prometheus_json):
+    histograms_df = None
+
+    # Parse Prometheus timeseries into a two-dimensional DataFrame.
+    # Columns: t (time), plus one for every Histogram bucket.
+    for result in prometheus_json["data"]["result"]:
+        bucket = int(math.log(float(result["metric"]["le"]), 2) * 10) - 1
+        metric_df = (
+            pandas.DataFrame(result["values"], dtype=float, columns=["t", bucket])
+                .set_index("t")
+        )
+        if histograms_df is None:
+            histograms_df = metric_df
+        else:
+            histograms_df = histograms_df.join(metric_df)
+
+    # Transpose (so that `le` is the first column, rather than `t`), sort and diff
+    # (Prometheus uses cumulative histograms rather than absolutes)
+    # then transpose back so we can continue our work
+    transposed_df = histograms_df.T.sort_index().diff()
+    histograms_df = transposed_df.T
+
+    # Normalize Buckets by transforming the number of actual pings sent
+    # into a float [0..1] indicating the grayness of that bucket.
+    biggestbkt = transposed_df.max()
+    normalized_df = histograms_df.div(biggestbkt, axis="index")
+    # prune outliers -> keep only values > 0.05%
+    pruned_df = normalized_df[normalized_df > 0.05]
+    # drop columns that contain only NaNs now
+    dropped_df = pruned_df.dropna(axis="columns", how="all")
+    # replace all the _remaining_ NaNs with 0
+    histograms_df = dropped_df.fillna(0)
+
+    # detect dynamic range
+    hmin = int(histograms_df.columns.min())
+    hmax = int(histograms_df.columns.max())
+
+    rows = hmax - hmin + 1
+    cols = len(histograms_df)
+
+    # How big do you want the squares to be?
+    sqsz = 8
+
+    # Draw the graph in a pixels array which we then copy to an image
+    width  = cols
+    height = rows
+    pixels = [0xFF] * (width * height)
+
+    for col, (tstamp, histogram) in enumerate(histograms_df.iterrows()):
+        for bktval, bktgrayness in histogram.items():
+            pixelval = int((1.0 - bktgrayness) * 0xFF)
+            #       (     y       )            (x)
+            pixels[((hmax - bktval) * width) + col] = pixelval
+
+    # copy pixels to an Image and paste that into the output image
+    graph = Image.new("L", (width, height), "white")
+    graph.putdata(pixels)
+
+    # Scale graph so each Pixel becomes a square
+    width  *= sqsz
+    height *= sqsz
+
+    graph = graph.resize((width, height))
+
+    # X position of the graph
+    graph_x = 70
+
+    # im will hold the output image
+    im = Image.new("RGB", (width + graph_x + 20, height + 100), "white")
+    im.paste(graph, (graph_x, 0))
+
+    # draw a rect around the graph
+    draw = ImageDraw.Draw(im)
+    draw.rectangle((graph_x, 0, graph_x + width - 1, height - 1), outline=0x333333)
+
+    try:
+        font = ImageFont.truetype("DejaVuSansMono.ttf", 10)
+    except IOError:
+        font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", 10)
+
+    # Y axis ticks and annotations
+    for hidx in range(hmin, hmax, 5):
+        bottomrow = (hidx - hmin)
+        offset_y = height - bottomrow * sqsz - 1
+        draw.line((graph_x - 2, offset_y, graph_x + 2, offset_y), fill=0xAAAAAA)
+
+        ping = 2 ** (hidx / 10.)
+        label = "%.2f" % ping
+        draw.text((graph_x - len(label) * 6 - 10, offset_y - 5), label, 0x333333, font=font)
+
+    # X axis ticks
+    for col, (tstamp, _) in list(enumerate(histograms_df.iterrows()))[::3]:
+        offset_x = graph_x + col * 8
+        draw.line((offset_x, height - 2, offset_x, height + 2), fill=0xAAAAAA)
+
+    # X axis annotations
+    # Create a temp image for the bottom label that we then rotate by 90° and attach to the other one
+    # since this stuff is rotated by 90° while we create it, all the coordinates are inversed...
+    tmpim = Image.new("RGB", (80, width + 20), "white")
+    tmpdraw = ImageDraw.Draw(tmpim)
+
+    for col, (tstamp, _) in list(enumerate(histograms_df.iterrows()))[::6]:
+        dt = datetime.fromtimestamp(tstamp)
+        offset_x = col * 8
+        tmpdraw.text(( 6, offset_x + 0), dt.strftime("%Y-%m-%d"), 0x333333, font=font)
+        tmpdraw.text((18, offset_x + 8), dt.strftime("%H:%M:%S"), 0x333333, font=font)
+
+    im.paste( tmpim.rotate(90, expand=1), (graph_x - 10, height + 1) )
+
+    # This worked pretty well for Tobi Oetiker...
+    tmpim = Image.new("RGB", (170, 11), "white")
+    tmpdraw = ImageDraw.Draw(tmpim)
+    tmpdraw.text((0, 0), "Meshping by Michael Ziegler", 0x999999, font=font)
+    im.paste( tmpim.rotate(270, expand=1), (width + graph_x + 9, 0) )
+
+    return im
+
+
+def main():
+    if len(sys.argv) != 5:
+        print("Usage: %s <prometheus URL> <pingnode> <target> <output.png>" % sys.argv[0], file=sys.stderr)
+        return 2
+
+    _, prometheus, pingnode, target, outfile = sys.argv
+
+    response = requests.get(prometheus + "/api/v1/query_range", timeout=2, params={
+        "query": 'increase(meshping_pings_bucket{instance="%s",name="%s"}[1h])' % (pingnode, target),
+        "start": time() - 3 * 24 * 60 * 60,
+        "end":   time(),
+        "step":  3600,
+    }).json()
+
+    assert response["status"] == "success", "Prometheus query failed"
+    assert response["data"]["result"], "Result is empty"
+
+    im = render(response)
+
+    if sys.argv[2] != "-":
+        im.save(outfile)
+    else:
+        im.save(sys.stdout, format="png")
+
+    return 0
+
+
+if __name__ == '__main__':
+    import requests
+    sys.exit(main())

src/meshping.py

@@ -7,6 +7,7 @@
 import os.path
 import math
 import socket
+import sys
 
 from uuid       import uuid4
 from time       import time
@@ -176,6 +177,18 @@ def main():
     if os.getuid() != 0:
         raise RuntimeError("need to be root, sorry about that")
 
+    known_env_vars = (
+        "MESHPING_REDIS_HOST",
+        "MESHPING_PING_TIMEOUT",
+        "MESHPING_PROMETHEUS_URL",
+        "MESHPING_PROMETHEUS_QUERY",
+    )
+
+    for key in os.environ.keys():
+        if key.startswith("MESHPING_") and key not in known_env_vars:
+            print("env var %s is unknown" % key, file=sys.stderr)
+            sys.exit(1)
+
     app = QuartTrio(__name__, static_url_path="")
     app.config["TEMPLATES_AUTO_RELOAD"] = True
     app.secret_key = str(uuid4())

src/templates/index.html

@@ -7,6 +7,10 @@
     <script type="text/javascript" src="/ui/node_modules/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
     <script type="text/javascript" src="/ui/node_modules/vue/dist/vue.min.js"></script>
     <script type="text/javascript" src="/ui/node_modules/vue-resource/dist/vue-resource.min.js"></script>
+    <script type="text/javascript">
+      window.meshping_hostname = "{{ Hostname }}";
+      window.meshping_haveprom = {{ HaveProm }};
+    </script>
   </head>
   <body>
     <div class="container mt-sm-3">
@@ -67,7 +71,10 @@ <h1>Meshping: {{ Hostname }}</h1>
             <td class="text-right">{{ target.avg24h.toFixed(2) }}</td>
             <td class="text-right">{{ target.max.toFixed(2)    }}</td>
             <td class="text-right">{{ target.last.toFixed(2)   }}</td>
-            <td>
+            <td style="white-space: nowrap">
+              <a v-if="haveprom" v-bind:href="'/histogram/' + hostname + '/' + target.name + '%40' + target.addr + '.png'" target="_blank">
+                <img class="border" src="/ui/node_modules/bootstrap-icons/icons/graph-up.svg" alt="graph" title="Graph" />
+              </a>
               <img class="border" src="/ui/node_modules/bootstrap-icons/icons/trash.svg" alt="del" title="Delete Target" v-on:click="delete_target(target)" />
             </td>
           </tr>

ui/src/main.js

@@ -1,6 +1,8 @@
 var app = new Vue({
     el: '#app',
     data: {
+        hostname: window.meshping_hostname,
+        haveprom: window.meshping_haveprom,
         success_msg: "",
         last_update: 0,
         search: localStorage.getItem("meshping_search") || "",