ovn: OVN Support QoS meter

This feature is used to limit the bandwidth of flows, such as floating IP.

ovn-northd changes:
1. add bandwidth column in NB's QOS table.
2. add QOS_METER stages in Logical switch ingress/egress.
3. add set_meter() action in SB's LFlow table.

ovn-controller changes:
add meter_table for meter action process openflow meter table.

Now, This feature is only supported in DPDK.

Signed-off-by: Guoshuai Li <[email protected]>
Signed-off-by: Ben Pfaff <[email protected]>

NEWS

@@ -118,6 +118,7 @@ v2.8.0 - 31 Aug 2017
        gateway.
      * Add support for ACL logging.
      * ovn-northd now has native support for active-standby high availability.
+     * Add support for QoS bandwidth limt with DPDK.
    - Tracing with ofproto/trace now traces through recirculation.
    - OVSDB:
      * New support for role-based access control (see ovsdb-server(1)).

include/ovn/actions.h

@@ -75,7 +75,8 @@ struct ovn_extend_table;
     OVNACT(DNS_LOOKUP,        ovnact_dns_lookup)      \
     OVNACT(LOG,               ovnact_log)             \
     OVNACT(PUT_ND_RA_OPTS,    ovnact_put_opts)        \
-    OVNACT(ND_NS,             ovnact_nest)
+    OVNACT(ND_NS,             ovnact_nest)            \
+    OVNACT(SET_METER,         ovnact_set_meter)
 
 /* enum ovnact_type, with a member OVNACT_<ENUM> for each action. */
 enum OVS_PACKED_ENUM ovnact_type {
@@ -281,6 +282,13 @@ struct ovnact_log {
     char *name;
 };
 
+/* OVNACT_SET_METER. */
+struct ovnact_set_meter {
+    struct ovnact ovnact;
+    uint64_t rate;                   /* rate field, in kbps. */
+    uint64_t burst;                  /* burst rate field, in kbps. */
+};
+
 /* Internal use by the helpers below. */
 void ovnact_init(struct ovnact *, enum ovnact_type, size_t len);
 void *ovnact_put(struct ofpbuf *, enum ovnact_type, size_t len);
@@ -490,6 +498,9 @@ struct ovnact_encode_params {
     /* A struct to figure out the group_id for group actions. */
     struct ovn_extend_table *group_table;
 
+    /* A struct to figure out the meter_id for meter actions. */
+    struct ovn_extend_table *meter_table;
+
     /* OVN maps each logical flow table (ltable), one-to-one, onto a physical
      * OpenFlow flow table (ptable).  A number of parameters describe this
      * mapping and data related to flow tables:

ovn/controller/lflow.c

@@ -27,6 +27,7 @@
 #include "ovn/expr.h"
 #include "ovn/lib/ovn-l7.h"
 #include "ovn/lib/ovn-sb-idl.h"
+#include "ovn/lib/extend-table.h"
 #include "packets.h"
 #include "physical.h"
 #include "simap.h"
@@ -62,6 +63,7 @@ static void consider_logical_flow(struct controller_ctx *ctx,
                                   const struct sbrec_logical_flow *lflow,
                                   const struct hmap *local_datapaths,
                                   struct ovn_extend_table *group_table,
+                                  struct ovn_extend_table *meter_table,
                                   const struct sbrec_chassis *chassis,
                                   struct hmap *dhcp_opts,
                                   struct hmap *dhcpv6_opts,
@@ -144,6 +146,7 @@ add_logical_flows(struct controller_ctx *ctx,
                   const struct chassis_index *chassis_index,
                   const struct hmap *local_datapaths,
                   struct ovn_extend_table *group_table,
+                  struct ovn_extend_table *meter_table,
                   const struct sbrec_chassis *chassis,
                   const struct shash *addr_sets,
                   struct hmap *flow_table,
@@ -174,7 +177,7 @@ add_logical_flows(struct controller_ctx *ctx,
     SBREC_LOGICAL_FLOW_FOR_EACH (lflow, ctx->ovnsb_idl) {
         consider_logical_flow(ctx, chassis_index,
                               lflow, local_datapaths,
-                              group_table, chassis,
+                              group_table, meter_table, chassis,
                               &dhcp_opts, &dhcpv6_opts, &nd_ra_opts,
                               &conj_id_ofs, addr_sets, flow_table,
                               active_tunnels, local_lport_ids);
@@ -191,6 +194,7 @@ consider_logical_flow(struct controller_ctx *ctx,
                       const struct sbrec_logical_flow *lflow,
                       const struct hmap *local_datapaths,
                       struct ovn_extend_table *group_table,
+                      struct ovn_extend_table *meter_table,
                       const struct sbrec_chassis *chassis,
                       struct hmap *dhcp_opts,
                       struct hmap *dhcpv6_opts,
@@ -263,6 +267,7 @@ consider_logical_flow(struct controller_ctx *ctx,
         .is_switch = is_switch(ldp),
         .is_gateway_router = is_gateway_router(ldp, local_datapaths),
         .group_table = group_table,
+        .meter_table = meter_table,
 
         .pipeline = ingress ? OVNACT_P_INGRESS : OVNACT_P_EGRESS,
         .ingress_ptable = OFTABLE_LOG_INGRESS_PIPELINE,
@@ -435,13 +440,14 @@ lflow_run(struct controller_ctx *ctx,
           const struct chassis_index *chassis_index,
           const struct hmap *local_datapaths,
           struct ovn_extend_table *group_table,
+          struct ovn_extend_table *meter_table,
           const struct shash *addr_sets,
           struct hmap *flow_table,
           struct sset *active_tunnels,
           struct sset *local_lport_ids)
 {
     add_logical_flows(ctx, chassis_index, local_datapaths,
-                      group_table, chassis, addr_sets, flow_table,
+                      group_table, meter_table, chassis, addr_sets, flow_table,
                       active_tunnels, local_lport_ids);
     add_neighbor_flows(ctx, flow_table);
 }

ovn/controller/lflow.h

@@ -67,6 +67,7 @@ void lflow_run(struct controller_ctx *,
                const struct chassis_index *,
                const struct hmap *local_datapaths,
                struct ovn_extend_table *group_table,
+               struct ovn_extend_table *meter_table,
                const struct shash *addr_sets,
                struct hmap *flow_table,
                struct sset *active_tunnels,

ovn/controller/ofctrl.c

@@ -134,6 +134,9 @@ static struct hmap installed_flows;
 /* A reference to the group_table. */
 static struct ovn_extend_table *groups;
 
+/* A reference to the meter_table. */
+static struct ovn_extend_table *meters;
+
 /* MFF_* field ID for our Geneve option.  In S_TLV_TABLE_MOD_SENT, this is
  * the option we requested (we don't know whether we obtained it yet).  In
  * S_CLEAR_FLOWS or S_UPDATE_FLOWS, this is really the option we have. */
@@ -145,20 +148,24 @@ static struct ofpbuf *encode_flow_mod(struct ofputil_flow_mod *);
 
 static struct ofpbuf *encode_group_mod(const struct ofputil_group_mod *);
 
+static struct ofpbuf *encode_meter_mod(const struct ofputil_meter_mod *);
+
 static void ovn_flow_table_clear(struct hmap *flow_table);
 static void ovn_flow_table_destroy(struct hmap *flow_table);
 
 static void ofctrl_recv(const struct ofp_header *, enum ofptype);
 
 void
-ofctrl_init(struct ovn_extend_table *group_table)
+ofctrl_init(struct ovn_extend_table *group_table,
+            struct ovn_extend_table *meter_table)
 {
     swconn = rconn_create(5, 0, DSCP_DEFAULT, 1 << OFP13_VERSION);
     tx_counter = rconn_packet_counter_create();
     hmap_init(&installed_flows);
     ovs_list_init(&flow_updates);
     ovn_init_symtab(&symtab);
     groups = group_table;
+    meters = meter_table;
 }
 
 /* S_NEW, for a new connection.
@@ -389,6 +396,18 @@ run_S_CLEAR_FLOWS(void)
         ovn_extend_table_clear(groups, true);
     }
 
+    /* Send a meter_mod to delete all meters. */
+    struct ofputil_meter_mod mm;
+    memset(&mm, 0, sizeof mm);
+    mm.command = OFPMC13_DELETE;
+    mm.meter.meter_id = OFPM13_ALL;
+    queue_msg(encode_meter_mod(&mm));
+
+    /* Clear existing meters, to match the state of the switch. */
+    if (meters) {
+        ovn_extend_table_clear(meters, true);
+    }
+
     /* All flow updates are irrelevant now. */
     struct ofctrl_flow_update *fup, *next;
     LIST_FOR_EACH_SAFE (fup, next, list_node, &flow_updates) {
@@ -760,6 +779,20 @@ add_group_mod(const struct ofputil_group_mod *gm, struct ovs_list *msgs)
     struct ofpbuf *msg = encode_group_mod(gm);
     ovs_list_push_back(msgs, &msg->list_node);
 }
+
+
+static struct ofpbuf *
+encode_meter_mod(const struct ofputil_meter_mod *mm)
+{
+    return ofputil_encode_meter_mod(OFP13_VERSION, mm);
+}
+
+static void
+add_meter_mod(const struct ofputil_meter_mod *mm, struct ovs_list *msgs)
+{
+    struct ofpbuf *msg = encode_meter_mod(mm);
+    ovs_list_push_back(msgs, &msg->list_node);
+}
 
 static void
 add_ct_flush_zone(uint16_t zone_id, struct ovs_list *msgs)
@@ -790,11 +823,10 @@ ofctrl_can_put(void)
 /* Replaces the flow table on the switch, if possible, by the flows added
  * with ofctrl_add_flow().
  *
- * Replaces the group table on the switch, if possible, by the contents of
- * 'groups->desired_groups'.  Regardless of whether the group table
- * is updated, this deletes all the groups from the
- * 'groups->desired_groups' and frees them. (The hmap itself isn't
- * destroyed.)
+ * Replaces the group table and meter table on the switch, if possible, by the
+ *  contents of 'groups->desired'.  Regardless of whether the group table
+ * is updated, this deletes all the groups from the 'groups->desired' and frees
+ * them. (The hmap itself isn't destroyed.)
  *
  * Sends conntrack flush messages to each zone in 'pending_ct_zones' that
  * is in the CT_ZONE_OF_QUEUED state and then moves the zone into the
@@ -808,6 +840,7 @@ ofctrl_put(struct hmap *flow_table, struct shash *pending_ct_zones,
     if (!ofctrl_can_put()) {
         ovn_flow_table_clear(flow_table);
         ovn_extend_table_clear(groups, false);
+        ovn_extend_table_clear(meters, false);
         return;
     }
 
@@ -848,6 +881,28 @@ ofctrl_put(struct hmap *flow_table, struct shash *pending_ct_zones,
         ofputil_uninit_group_mod(&gm);
     }
 
+    /* Iterate through all the desired meters. If there are new ones,
+     * add them to the switch. */
+    struct ovn_extend_table_info *m_desired;
+    EXTEND_TABLE_FOR_EACH_UNINSTALLED (m_desired, meters) {
+        /* Create and install new meter. */
+        struct ofputil_meter_mod mm;
+        enum ofputil_protocol usable_protocols;
+        char *meter_string = xasprintf("meter=%"PRIu32",%s",
+                                       m_desired->table_id,
+                                       ds_cstr(&m_desired->info));
+        char *error = parse_ofp_meter_mod_str(&mm, meter_string, OFPMC13_ADD,
+                                              &usable_protocols);
+        if (!error) {
+            add_meter_mod(&mm, &msgs);
+        } else {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+            VLOG_ERR_RL(&rl, "new meter %s %s", error, meter_string);
+            free(error);
+        }
+        free(meter_string);
+    }
+
     /* Iterate through all of the installed flows.  If any of them are no
      * longer desired, delete them; if any of them should have different
      * actions, update them. */
@@ -936,7 +991,7 @@ ofctrl_put(struct hmap *flow_table, struct shash *pending_ct_zones,
         } else {
             static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
             VLOG_ERR_RL(&rl, "Error deleting group %d: %s",
-                     installed->table_id, error);
+                        installed->table_id, error);
             free(error);
         }
         free(group_string);
@@ -947,6 +1002,33 @@ ofctrl_put(struct hmap *flow_table, struct shash *pending_ct_zones,
     /* Move the contents of groups->desired to groups->existing. */
     ovn_extend_table_move(groups);
 
+    /* Iterate through the installed meters from previous runs. If they
+     * are not needed delete them. */
+    struct ovn_extend_table_info *m_installed, *next_meter;
+    EXTEND_TABLE_FOR_EACH_INSTALLED (m_installed, next_meter, meters) {
+        /* Delete the meter. */
+        struct ofputil_meter_mod mm;
+        enum ofputil_protocol usable_protocols;
+        char *meter_string = xasprintf("meter=%"PRIu32"",
+                                       m_installed->table_id);
+        char *error = parse_ofp_meter_mod_str(&mm, meter_string,
+                                              OFPMC13_DELETE,
+                                              &usable_protocols);
+        if (!error) {
+            add_meter_mod(&mm, &msgs);
+        } else {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+            VLOG_ERR_RL(&rl,  "Error deleting meter %"PRIu32": %s",
+                        m_installed->table_id, error);
+            free(error);
+        }
+        free(meter_string);
+        ovn_extend_table_remove(meters, m_installed);
+    }
+
+    /* Move the contents of meters->desired to meters->existing. */
+    ovn_extend_table_move(meters);
+
     if (!ovs_list_is_empty(&msgs)) {
         /* Add a barrier to the list of messages. */
         struct ofpbuf *barrier = ofputil_encode_barrier_request(OFP13_VERSION);

ovn/controller/ofctrl.h

@@ -31,7 +31,8 @@ struct ovsrec_bridge;
 struct shash;
 
 /* Interface for OVN main loop. */
-void ofctrl_init(struct ovn_extend_table *group_table);
+void ofctrl_init(struct ovn_extend_table *group_table,
+                 struct ovn_extend_table *meter_table);
 enum mf_field_id ofctrl_run(const struct ovsrec_bridge *br_int,
                             struct shash *pending_ct_zones);
 bool ofctrl_can_put(void);

ovn/controller/ovn-controller.c

@@ -597,9 +597,13 @@ main(int argc, char *argv[])
     struct ovn_extend_table group_table;
     ovn_extend_table_init(&group_table);
 
+    /* Initialize meter ids for QoS. */
+    struct ovn_extend_table meter_table;
+    ovn_extend_table_init(&meter_table);
+
     daemonize_complete();
 
-    ofctrl_init(&group_table);
+    ofctrl_init(&group_table, &meter_table);
     pinctrl_init();
     lflow_init();
 
@@ -709,8 +713,8 @@ main(int argc, char *argv[])
                     struct hmap flow_table = HMAP_INITIALIZER(&flow_table);
                     lflow_run(&ctx, chassis,
                               &chassis_index, &local_datapaths, &group_table,
-                              &addr_sets, &flow_table, &active_tunnels,
-                              &local_lport_ids);
+                              &meter_table, &addr_sets, &flow_table,
+                              &active_tunnels, &local_lport_ids);
 
                     if (chassis_id) {
                         bfd_run(&ctx, br_int, chassis, &local_datapaths,
@@ -844,6 +848,7 @@ main(int argc, char *argv[])
     shash_destroy(&pending_ct_zones);
 
     ovn_extend_table_destroy(&group_table);
+    ovn_extend_table_destroy(&meter_table);
 
     ovsdb_idl_loop_destroy(&ovs_idl_loop);
     ovsdb_idl_loop_destroy(&ovnsb_idl_loop);