working on zwatch

kvmagent/kvmagent/plugins/deip.py

@@ -8,6 +8,8 @@
 from zstacklib.utils import shell
 from zstacklib.utils import ebtables
 from zstacklib.utils.bash import *
+from kvmagent.plugins import prometheus
+from prometheus_client.core import GaugeMetricFamily
 
 logger = log.get_logger(__name__)
 EBTABLES_CMD = ebtables.get_ebtables_cmd()
@@ -17,6 +19,87 @@ def __init__(self):
         self.success = True
         self.error = None
 
+@in_bash
+def collect_vip_statistics():
+    def parse_eip_string(estr):
+        ip = vip_uuid = None
+        ws = estr.split(',')
+        for w in ws:
+            if w.startswith('eip_addr'):
+                ip = w.split(':')[1]
+            elif w.startswith('vip'):
+                vip_uuid = w.split(':')[1]
+
+        return ip, vip_uuid
+
+    def find_namespace_name_by_ip(ip):
+        ns_name_suffix = ip.replace('.', '_')
+        o = bash_o('ip netns')
+        for l in o.split('\n'):
+            if l.endswith(ns_name_suffix):
+                return l
+
+        return None
+
+    def create_metric(line, ip, vip_uuid):
+        pairs = line.split()
+        pkts = pairs[0]
+        bs = pairs[1]
+        src = pairs[7]
+        dst = pairs[8]
+
+        VIP_LABEL_NAME = 'VipUUID'
+        ret = []
+        # out traffic
+        if src.startswith(ip):
+            g = GaugeMetricFamily('zstack_vip_out_bytes', 'VIP outbound traffic in bytes', labels=[VIP_LABEL_NAME])
+            g.add_metric([vip_uuid], float(bs))
+
+            g = GaugeMetricFamily('zstack_vip_out_packages', 'VIP outbound traffic packages', labels=[VIP_LABEL_NAME])
+            g.add_metric([vip_uuid], float(pkts))
+        # in traffic
+        if dst.startswith(ip):
+            g = GaugeMetricFamily('zstack_vip_in_bytes', 'VIP inbound traffic in bytes',  labels=[VIP_LABEL_NAME])
+            g.add_metric([vip_uuid], float(bs))
+
+            g = GaugeMetricFamily('zstack_vip_in_packages', 'VIP inbound traffic packages', labels=[VIP_LABEL_NAME])
+            g.add_metric([vip_uuid], float(pkts))
+
+        return ret
+
+    def collect(ip, vip_uuid):
+        ret = []
+        ns_name = find_namespace_name_by_ip(ip)
+        if not ns_name:
+            return ret
+
+        CHAIN_NAME = "vip-perf"
+        o = bash_o("ip netns exec {{ns_name}} iptables -nvxL {{CHAIN_NAME}} | sed '1,2d'")
+        for l in o.split('\n'):
+            ret.append(create_metric(l, ip, vip_uuid))
+
+        return ret
+
+    o = bash_o('ip -o -d link')
+    words = o.split()
+    eip_strings = [w.startswith('eip:') for w in words]
+
+    ret = []
+    for estr in eip_strings:
+        ip, vip_uuid = parse_eip_string(estr)
+        if ip is None:
+            logger.warn("no ip field found in %s" % estr)
+            continue
+        if vip_uuid is None:
+            logger.warn("no vip field found in %s" % estr)
+            continue
+
+        ret.extend(collect(ip, vip_uuid))
+
+    return ret
+
+prometheus.register_prometheus_collector(collect_vip_statistics)
+
 class DEip(kvmagent.KvmAgent):
 
     APPLY_EIP_PATH = "/flatnetworkprovider/eip/apply"
@@ -139,7 +222,7 @@ def _apply_eip(self, eip):
         EBTABLE_CHAIN_NAME= eip.vmBridgeName
         PRI_BR_PHY_DEV= eip.vmBridgeName.replace('br_', '', 1)
 
-        EIP_DESC = "eip:%s,eip_addr:%s,vnic:%s,vnic_ip:%s,vm:%s" % (eip.eipUuid, VIP, eip.nicName, NIC_IP, eip.vmUuid)
+        EIP_DESC = "eip:%s,eip_addr:%s,vnic:%s,vnic_ip:%s,vm:%s,vip:%s" % (eip.eipUuid, VIP, eip.nicName, NIC_IP, eip.vmUuid, eip.vipUuid)
 
         NS = "ip netns exec {{NS_NAME}}"
 
@@ -229,10 +312,18 @@ def set_gateway_arp_if_needed():
             create_ebtable_rule_if_needed('nat', 'POSTROUTING', "-p ARP -o {{PRI_ODEV}} -j {{BLOCK_CHAIN_NAME}}")
             create_ebtable_rule_if_needed('nat', BLOCK_CHAIN_NAME, "-p ARP -o {{PRI_ODEV}} --arp-op Request --arp-ip-dst {{NIC_GATEWAY}} --arp-mac-src ! {{NIC_MAC}} -j DROP")
 
+        def create_perf_monitor():
+            CHAIN_NAME = "vip-perf"
+            bash_r("eval {{NS}} iptables -N {{CHAIN_NAME}} > /dev/null")
+            create_iptable_rule_if_needed("-t filter", "-A FORWARD -s {{VIP}} -j {{CHAIN_NAME}}")
+            create_iptable_rule_if_needed("-t filter", "-A FORWARD -d {{VIP}} -j {{CHAIN_NAME}}")
+            create_iptable_rule_if_needed("-t filter", "-A {{CHAIN_NAME}} -s {{VIP}} -j RETURN")
+            create_iptable_rule_if_needed("-t filter", "-A {{CHAIN_NAME}} -d {{VIP}} -j RETURN")
+
         if bash_r('eval {{NS}} ip link show > /dev/null') != 0:
             bash_errorout('ip netns add {{NS_NAME}}')
 
-        # To be compatibled with old Oversion
+        # To be compatibled with old version
         for i in range(len(OLD_PUB_IDEVS)):
             delete_orphan_outer_dev(OLD_PUB_IDEVS[i], OLD_PUB_ODEVS[i])
             delete_orphan_outer_dev(OLD_PRI_IDEVS[i], OLD_PRI_ODEVS[i])
@@ -258,6 +349,7 @@ def set_gateway_arp_if_needed():
         set_gateway_arp_if_needed()
         set_eip_rules()
         set_default_route_if_needed()
+        create_perf_monitor()
 
     @lock.lock('eip')
     def _apply_eips(self, eips):

kvmagent/kvmagent/plugins/prometheus.py

@@ -10,11 +10,19 @@
 import re
 import time
 import traceback
+from prometheus_client.core import GaugeMetricFamily,REGISTRY
+from prometheus_client import start_http_server
+
+
+def register_prometheus_collector(collector):
+    PrometheusPlugin.COLLECTORS.append(collector)
 
 class PrometheusPlugin(kvmagent.KvmAgent):
 
     COLLECTD_PATH = "/prometheus/collectdexporter/start"
 
+    COLLECTORS = []
+
     @kvmagent.replyerror
     @in_bash
     def start_collectd_exporter(self, req):
@@ -145,9 +153,24 @@ def start_collectd_exporter(self, req):
 
         return jsonobject.dumps(rsp)
 
+    def install_colletor(self):
+
+        class Collector(object):
+            def collect(self):
+                ret = []
+                for c in PrometheusPlugin.COLLECTORS:
+                    ret.extend(c())
+
+                return ret
+
+        REGISTRY.register(Collector())
+
     def start(self):
         http_server = kvmagent.get_http_server()
         http_server.register_async_uri(self.COLLECTD_PATH, self.start_collectd_exporter)
 
+        self.install_colletor()
+        start_http_server(7069)
+
     def stop(self):
         pass

kvmagent/setup.py

@@ -19,6 +19,7 @@
       zip_safe=True,
       install_requires=[
           # -*- Extra requirements: -*-
+          "prometheus_client"
       ],
       entry_points="""
       # -*- Entry points: -*-

zstackbuild/pypi_source/requirements.txt

@@ -32,3 +32,4 @@ websockify==0.6.1
 Werkzeug==0.10.4
 flask-socketio==2.2
 versiontools==1.7
+prometheus_client==0.1.1