net: make interface get packets from l3

Instead of l3 (arp/ipv4) pushing packets into interface's queue, make
them register functions that interface can use to ask l3 for packets.

net/arp.cc

@@ -15,7 +15,7 @@ arp_for_protocol::~arp_for_protocol() {
     _arp.del(_proto_num);
 }
 
-arp::arp(interface* netif) : _netif(netif), _proto(netif, eth_protocol_num::arp)
+arp::arp(interface* netif) : _netif(netif), _proto(netif, eth_protocol_num::arp, [this] { return get_packet(); })
     , _rx_packets(_proto.receive([this] (packet p, ethernet_address ea) {
         return process_packet(std::move(p), ea);
     },
@@ -24,6 +24,15 @@ arp::arp(interface* netif) : _netif(netif), _proto(netif, eth_protocol_num::arp)
     })) {
 }
 
+std::experimental::optional<l3_protocol::l3packet> arp::get_packet() {
+    std::experimental::optional<l3_protocol::l3packet> p;
+    if (!_packetq.empty()) {
+        p = std::move(_packetq.front());
+        _packetq.pop_front();
+    }
+    return p;
+}
+
 bool arp::forward(forward_hash& out_hash_data, packet& p, size_t off) {
     auto ah = p.get_header<arp_hdr>(off);
     auto i = _arp_for_protocol.find(ntoh(ah->ptype));

net/arp.hh

@@ -36,6 +36,7 @@ class arp {
     l3_protocol _proto;
     subscription<packet, ethernet_address> _rx_packets;
     std::unordered_map<uint16_t, arp_for_protocol*> _arp_for_protocol;
+    circular_buffer<l3_protocol::l3packet> _packetq;
 private:
     struct arp_hdr {
         packed<uint16_t> htype;
@@ -52,6 +53,7 @@ private:
     ethernet_address l2self() { return _netif->hw_address(); }
     future<> process_packet(packet p, ethernet_address from);
     bool forward(forward_hash& out_hash_data, packet& p, size_t off);
+    std::experimental::optional<l3_protocol::l3packet> get_packet();
     template <class l3_proto>
     friend class arp_for;
 };
@@ -96,6 +98,7 @@ private:
     virtual future<> received(packet p) override;
     future<> handle_request(arp_hdr* ah);
     l2addr l2self() { return _arp.l2self(); }
+    void send(l2addr to, packet p);
 public:
     future<> send_query(const l3addr& paddr);
     explicit arp_for(arp& a) : arp_for_protocol(a, L3::arp_protocol_type()) {}
@@ -123,10 +126,16 @@ arp_for<L3>::make_query_packet(l3addr paddr) {
     return packet(reinterpret_cast<char*>(&hdr), sizeof(hdr));
 }
 
+template <typename L3>
+void arp_for<L3>::send(l2addr to, packet p) {
+    _arp._packetq.push_back(l3_protocol::l3packet{eth_protocol_num::arp, to, std::move(p)});
+}
+
 template <typename L3>
 future<>
 arp_for<L3>::send_query(const l3addr& paddr) {
-    return _arp._proto.send(ethernet::broadcast_address(), make_query_packet(paddr));
+    send(ethernet::broadcast_address(), make_query_packet(paddr));
+    return make_ready_future<>();
 }
 
 class arp_error : public std::runtime_error {
@@ -223,11 +232,9 @@ arp_for<L3>::handle_request(arp_hdr* ah) {
         ah->sender_hwaddr = l2self();
         ah->sender_paddr = _l3self;
         *ah = hton(*ah);
-        packet p(reinterpret_cast<char*>(ah), sizeof(*ah));
-        return _arp._proto.send(ah->target_hwaddr, std::move(p));
-    } else {
-        return make_ready_future<>();
+        send(ah->target_hwaddr, packet(reinterpret_cast<char*>(ah), sizeof(*ah)));
     }
+    return make_ready_future<>();
 }
 
 }

net/ip.cc

@@ -28,7 +28,7 @@ ipv4::ipv4(interface* netif)
     : _netif(netif)
     , _global_arp(netif)
     , _arp(_global_arp)
-    , _l3(netif, eth_protocol_num::ipv4)
+    , _l3(netif, eth_protocol_num::ipv4, [this] { return get_packet(); })
     , _rx_packets(_l3.receive([this] (packet p, ethernet_address ea) {
         return handle_received_packet(std::move(p), ea); },
       [this] (forward_hash& out_hash_data, packet& p, size_t off) {
@@ -269,7 +269,17 @@ future<> ipv4::send(ipv4_address to, ip_protocol_num proto_num, packet p) {
 }
 
 future<> ipv4::send_raw(ethernet_address dst, packet p) {
-    return _l3.send(dst, std::move(p));
+    _packetq.push_back(l3_protocol::l3packet{eth_protocol_num::ipv4, dst, std::move(p)});
+    return make_ready_future<>();
+}
+
+std::experimental::optional<l3_protocol::l3packet> ipv4::get_packet() {
+    std::experimental::optional<l3_protocol::l3packet> p;
+    if (!_packetq.empty()) {
+        p = std::move(_packetq.front());
+        _packetq.pop_front();
+    }
+    return p;
 }
 
 void ipv4::set_host_address(ipv4_address ip) {

net/ip.hh

@@ -220,9 +220,11 @@ private:
     static constexpr uint32_t _frag_high_thresh{4 * 1024 * 1024};
     uint32_t _frag_mem{0};
     timer<lowres_clock> _frag_timer;
+    circular_buffer<l3_protocol::l3packet> _packetq;
 private:
     future<> handle_received_packet(packet p, ethernet_address from);
     bool forward(forward_hash& out_hash_data, packet& p, size_t off);
+    std::experimental::optional<l3_protocol::l3packet> get_packet();
     bool in_my_netmask(ipv4_address a) const;
     void frag_limit_mem();
     void frag_timeout();

net/net.cc

@@ -11,8 +11,9 @@ using std::move;
 
 namespace net {
 
-l3_protocol::l3_protocol(interface* netif, eth_protocol_num proto_num)
-    : _netif(netif), _proto_num(proto_num) {
+l3_protocol::l3_protocol(interface* netif, eth_protocol_num proto_num, packet_provider_type func)
+    : _netif(netif), _proto_num(proto_num)  {
+        _netif->register_packet_provider(std::move(func));
 }
 
 subscription<packet, ethernet_address> l3_protocol::receive(
@@ -21,20 +22,27 @@ subscription<packet, ethernet_address> l3_protocol::receive(
     return _netif->register_l3(_proto_num, std::move(rx_fn), std::move(forward));
 };
 
-future<> l3_protocol::send(ethernet_address to, packet p) {
-    return _netif->send(_proto_num, to, std::move(p));
-}
-
 interface::interface(std::shared_ptr<device> dev)
     : _dev(dev)
     , _rx(_dev->receive([this] (packet p) { return dispatch_packet(std::move(p)); }))
     , _hw_address(_dev->hw_address())
     , _hw_features(_dev->hw_features()) {
-    dev->local_queue().register_packet_provider([this] {
+    dev->local_queue().register_packet_provider([this, idx = 0u] () mutable {
             std::experimental::optional<packet> p;
-            if(!_packetq.empty()) {
-                p = std::move(_packetq.front());
-                _packetq.pop_front();
+            for (size_t i = 0; i < _pkt_providers.size(); i++) {
+                auto l3p = _pkt_providers[idx++]();
+                if (idx == _pkt_providers.size())
+                    idx = 0;
+                if (l3p) {
+                    auto l3pv = std::move(l3p.value());
+                    auto eh = l3pv.p.prepend_header<eth_hdr>();
+                    eh->dst_mac = l3pv.to;
+                    eh->src_mac = _hw_address;
+                    eh->eth_proto = uint16_t(l3pv.proto_num);
+                    *eh = hton(*eh);
+                    p = std::move(l3pv.p);
+                    return p;
+                }
             }
             return p;
         });
@@ -103,14 +111,4 @@ future<> interface::dispatch_packet(packet p) {
     return make_ready_future<>();
 }
 
-future<> interface::send(eth_protocol_num proto_num, ethernet_address to, packet p) {
-    auto eh = p.prepend_header<eth_hdr>();
-    eh->dst_mac = to;
-    eh->src_mac = _hw_address;
-    eh->eth_proto = uint16_t(proto_num);
-    *eh = hton(*eh);
-    _packetq.push_back(std::move(p));
-    return make_ready_future<>();
-}
-
 }

net/net.hh

@@ -64,14 +64,21 @@ struct hw_features {
 };
 
 class l3_protocol {
+public:
+    struct l3packet {
+        eth_protocol_num proto_num;
+        ethernet_address to;
+        packet p;
+    };
+    using packet_provider_type = std::function<std::experimental::optional<l3packet> ()>;
+private:
     interface* _netif;
     eth_protocol_num _proto_num;
 public:
-    explicit l3_protocol(interface* netif, eth_protocol_num proto_num);
+    explicit l3_protocol(interface* netif, eth_protocol_num proto_num, packet_provider_type func);
     subscription<packet, ethernet_address> receive(
             std::function<future<> (packet, ethernet_address)> rx_fn,
             std::function<bool (forward_hash&, packet&, size_t)> forward);
-    future<> send(ethernet_address to, packet p);
 private:
     friend class interface;
 };
@@ -88,10 +95,9 @@ class interface {
     subscription<packet> _rx;
     ethernet_address _hw_address;
     net::hw_features _hw_features;
-    circular_buffer<packet> _packetq;
+    std::vector<l3_protocol::packet_provider_type> _pkt_providers;
 private:
     future<> dispatch_packet(packet p);
-    future<> send(eth_protocol_num proto_num, ethernet_address to, packet p);
 public:
     explicit interface(std::shared_ptr<device> dev);
     ethernet_address hw_address() { return _hw_address; }
@@ -101,6 +107,9 @@ public:
             std::function<bool (forward_hash&, packet&, size_t)> forward);
     void forward(unsigned cpuid, packet p);
     unsigned hash2cpu(uint32_t hash);
+    void register_packet_provider(l3_protocol::packet_provider_type func) {
+        _pkt_providers.push_back(std::move(func));
+    }
     friend class l3_protocol;
 };
 

tests/l3_test.cc

@@ -21,7 +21,7 @@ int main(int ac, char** av) {
 
     auto vnet = create_virtio_net_device(opts);
     interface netif(std::move(vnet));
-    l3_protocol arp(&netif, eth_protocol_num::arp);
+    l3_protocol arp(&netif, eth_protocol_num::arp, []{ return std::experimental::optional<l3_protocol::l3packet>(); });
     dump_arp_packets(arp);
     engine.run();
     return 0;