Relay (slackhq#678)

Co-authored-by: Wade Simmons <[email protected]>

.github/workflows/smoke.yml

@@ -45,4 +45,12 @@ jobs:
  working-directory: ./.github/workflows/smoke
  run: ./smoke.sh
 
+ - name: setup relay docker image
+ working-directory: ./.github/workflows/smoke
+ run: ./build-relay.sh
+
+ - name: run smoke relay
+ working-directory: ./.github/workflows/smoke
+ run: ./smoke-relay.sh
+
  timeout-minutes: 10

.github/workflows/smoke/build-relay.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -e -x
+
+rm -rf ./build
+mkdir ./build
+
+(
+ cd build
+
+ cp ../../../../build/linux-amd64/nebula .
+ cp ../../../../build/linux-amd64/nebula-cert .
+
+ HOST="lighthouse1" AM_LIGHTHOUSE=true ../genconfig.sh >lighthouse1.yml <<EOF
+relay:
+ am_relay: true
+EOF
+
+ export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
+ export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
+
+ HOST="host2" ../genconfig.sh >host2.yml <<EOF
+relay:
+ relays:
+ - 192.168.100.1
+EOF
+
+ export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
+
+ HOST="host3" ../genconfig.sh >host3.yml
+
+ HOST="host4" ../genconfig.sh >host4.yml <<EOF
+relay:
+ use_relays: false
+EOF
+
+ ../../../../nebula-cert ca -name "Smoke Test"
+ ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24"
+ ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24"
+ ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24"
+ ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24"
+)
+
+sudo docker build -t nebula:smoke-relay .

.github/workflows/smoke/genconfig.sh

@@ -40,6 +40,7 @@ pki:
 lighthouse:
  am_lighthouse: ${AM_LIGHTHOUSE:-false}
  hosts: $(lighthouse_hosts)
+ remote_allow_list: ${REMOTE_ALLOW_LIST}
 
 listen:
  host: 0.0.0.0
@@ -51,4 +52,6 @@ tun:
 firewall:
  outbound: ${OUTBOUND:-$FIREWALL_ALL}
  inbound: ${INBOUND:-$FIREWALL_ALL}
+
+$(test -t 0 || cat)
 EOF

.github/workflows/smoke/smoke-relay.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -e -x
+
+set -o pipefail
+
+mkdir -p logs
+
+cleanup() {
+ echo
+ echo " *** cleanup"
+ echo
+
+ set +e
+ if [ "$(jobs -r)" ]
+ then
+ sudo docker kill lighthouse1 host2 host3 host4
+ fi
+}
+
+trap cleanup EXIT
+
+sudo docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+sudo docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
+sudo docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
+sudo docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
+
+sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
+sleep 1
+sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
+sleep 1
+sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
+sleep 1
+sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
+sleep 1
+
+set +x
+echo
+echo " *** Testing ping from lighthouse1"
+echo
+set -x
+sudo docker exec lighthouse1 ping -c1 192.168.100.2
+sudo docker exec lighthouse1 ping -c1 192.168.100.3
+sudo docker exec lighthouse1 ping -c1 192.168.100.4
+
+set +x
+echo
+echo " *** Testing ping from host2"
+echo
+set -x
+sudo docker exec host2 ping -c1 192.168.100.1
+# Should fail because no relay configured in this direction
+! sudo docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
+! sudo docker exec host2 ping -c1 192.168.100.4 -w5 || exit 1
+
+set +x
+echo
+echo " *** Testing ping from host3"
+echo
+set -x
+sudo docker exec host3 ping -c1 192.168.100.1
+sudo docker exec host3 ping -c1 192.168.100.2
+sudo docker exec host3 ping -c1 192.168.100.4
+
+set +x
+echo
+echo " *** Testing ping from host4"
+echo
+set -x
+sudo docker exec host4 ping -c1 192.168.100.1
+# Should fail because relays not allowed
+! sudo docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1
+sudo docker exec host4 ping -c1 192.168.100.3
+
+sudo docker exec host4 sh -c 'kill 1'
+sudo docker exec host3 sh -c 'kill 1'
+sudo docker exec host2 sh -c 'kill 1'
+sudo docker exec lighthouse1 sh -c 'kill 1'
+sleep 1
+
+if [ "$(jobs -r)" ]
+then
+ echo "nebula still running after SIGTERM sent" >&2
+ exit 1
+fi

.github/workflows/smoke/smoke.sh

@@ -7,6 +7,10 @@ set -o pipefail
 mkdir -p logs
 
 cleanup() {
+ echo
+ echo " *** cleanup"
+ echo
+
  set +e
  if [ "$(jobs -r)" ]
  then
@@ -21,13 +25,13 @@ sudo docker run --name host2 --rm nebula:smoke -config host2.yml -test
 sudo docker run --name host3 --rm nebula:smoke -config host3.yml -test
 sudo docker run --name host4 --rm nebula:smoke -config host4.yml -test
 
-sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 &
+sudo docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
 sleep 1
-sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml 2>&1 | tee logs/host2 &
+sudo docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
 sleep 1
-sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml 2>&1 | tee logs/host3 &
+sudo docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
 sleep 1
-sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml 2>&1 | tee logs/host4 &
+sudo docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
 sleep 1
 
 set +x
@@ -81,3 +85,9 @@ sudo docker exec host3 sh -c 'kill 1'
 sudo docker exec host2 sh -c 'kill 1'
 sudo docker exec lighthouse1 sh -c 'kill 1'
 sleep 1
+
+if [ "$(jobs -r)" ]
+then
+ echo "nebula still running after SIGTERM sent" >&2
+ exit 1
+fi

Makefile

@@ -168,6 +168,10 @@ smoke-docker: bin-docker
  cd .github/workflows/smoke/ && ./build.sh
  cd .github/workflows/smoke/ && ./smoke.sh
 
+smoke-relay-docker: bin-docker
+ cd .github/workflows/smoke/ && ./build-relay.sh
+ cd .github/workflows/smoke/ && ./smoke-relay.sh
+
 smoke-docker-race: BUILD_ARGS = -race
 smoke-docker-race: smoke-docker
 

cert.go

@@ -157,14 +157,14 @@ func loadCAFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error)
  }
 
  for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) {
- l.WithField("fingerprint", fp).Infof("Blocklisting cert")
+ l.WithField("fingerprint", fp).Info("Blocklisting cert")
  CAs.BlocklistFingerprint(fp)
  }
 
  // Support deprecated config for at least one minor release to allow for migrations
  //TODO: remove in 2022 or later
  for _, fp := range c.GetStringSlice("pki.blacklist", []string{}) {
- l.WithField("fingerprint", fp).Infof("Blocklisting cert")
+ l.WithField("fingerprint", fp).Info("Blocklisting cert")
  l.Warn("pki.blacklist is deprecated and will not be supported in a future release. Please migrate your config to use pki.blocklist")
  CAs.BlocklistFingerprint(fp)
  }

connection_manager.go

@@ -301,7 +301,7 @@ func (n *connectionManager) handleInvalidCertificate(now time.Time, vpnIp iputil
 
  // Inform the remote and close the tunnel locally
  n.intf.sendCloseTunnel(hostinfo)
- n.intf.closeTunnel(hostinfo, false)
+ n.intf.closeTunnel(hostinfo)
 
  n.ClearIP(vpnIp)
  n.ClearPendingDeletion(vpnIp)

control.go

@@ -28,14 +28,16 @@ type Control struct {
 }
 
 type ControlHostInfo struct {
- VpnIp net.IP `json:"vpnIp"`
- LocalIndex uint32 `json:"localIndex"`
- RemoteIndex uint32 `json:"remoteIndex"`
- RemoteAddrs []*udp.Addr `json:"remoteAddrs"`
- CachedPackets int `json:"cachedPackets"`
- Cert *cert.NebulaCertificate `json:"cert"`
- MessageCounter uint64 `json:"messageCounter"`
- CurrentRemote *udp.Addr `json:"currentRemote"`
+ VpnIp net.IP `json:"vpnIp"`
+ LocalIndex uint32 `json:"localIndex"`
+ RemoteIndex uint32 `json:"remoteIndex"`
+ RemoteAddrs []*udp.Addr `json:"remoteAddrs"`
+ CachedPackets int `json:"cachedPackets"`
+ Cert *cert.NebulaCertificate `json:"cert"`
+ MessageCounter uint64 `json:"messageCounter"`
+ CurrentRemote *udp.Addr `json:"currentRemote"`
+ CurrentRelaysToMe []iputil.VpnIp `json:"currentRelaysToMe"`
+ CurrentRelaysThroughMe []iputil.VpnIp `json:"currentRelaysThroughMe"`
 }
 
 // Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
@@ -60,12 +62,14 @@ func (c *Control) Start() {
 
 // Stop signals nebula to shutdown, returns after the shutdown is complete
 func (c *Control) Stop() {
- //TODO: stop tun and udp routines, the lock on hostMap effectively does that though
+ // Stop the handshakeManager (and other serivces), to prevent new tunnels from
+ // being created while we're shutting them all down.
+ c.cancel()
+
  c.CloseAllTunnels(false)
  if err := c.f.Close(); err != nil {
  c.l.WithError(err).Error("Close interface failed")
  }
- c.cancel()
  c.l.Info("Goodbye")
 }
 
@@ -144,50 +148,74 @@ func (c *Control) CloseTunnel(vpnIp iputil.VpnIp, localOnly bool) bool {
  0,
  hostInfo.ConnectionState,
  hostInfo,
- hostInfo.remote,
  []byte{},
  make([]byte, 12, 12),
  make([]byte, mtu),
  )
  }
 
- c.f.closeTunnel(hostInfo, false)
+ c.f.closeTunnel(hostInfo)
  return true
 }
 
 // CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels
 // the int returned is a count of tunnels closed
 func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
  //TODO: this is probably better as a function in ConnectionManager or HostMap directly
- c.f.hostMap.Lock()
  lighthouses := c.f.lightHouse.GetLighthouses()
- for _, h := range c.f.hostMap.Hosts {
+
+ shutdown := func(h *HostInfo) {
  if excludeLighthouses {
  if _, ok := lighthouses[h.vpnIp]; ok {
- continue
+ return
  }
  }
+ c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
+ c.f.closeTunnel(h)
 
- if h.ConnectionState.ready {
- c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, h.remote, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
- c.f.closeTunnel(h, true)
+ c.l.WithField("vpnIp", h.vpnIp).WithField("udpAddr", h.remote).
+ Debug("Sending close tunnel message")
+ closed++
+ }
 
- c.l.WithField("vpnIp", h.vpnIp).WithField("udpAddr", h.remote).
- Debug("Sending close tunnel message")
- closed++
+ // Learn which hosts are being used as relays, so we can shut them down last.
+ relayingHosts := map[iputil.VpnIp]*HostInfo{}
+ // Grab the hostMap lock to access the Relays map
+ c.f.hostMap.Lock()
+ for _, relayingHost := range c.f.hostMap.Relays {
+ relayingHosts[relayingHost.vpnIp] = relayingHost
+ }
+ c.f.hostMap.Unlock()
+
+ hostInfos := []*HostInfo{}
+ // Grab the hostMap lock to access the Hosts map
+ c.f.hostMap.Lock()
+ for _, relayHost := range c.f.hostMap.Hosts {
+ if _, ok := relayingHosts[relayHost.vpnIp]; !ok {
+ hostInfos = append(hostInfos, relayHost)
  }
  }
  c.f.hostMap.Unlock()
+
+ for _, h := range hostInfos {
+ shutdown(h)
+ }
+ for _, h := range relayingHosts {
+ shutdown(h)
+ }
  return
 }
 
 func copyHostInfo(h *HostInfo, preferredRanges []*net.IPNet) ControlHostInfo {
+
  chi := ControlHostInfo{
- VpnIp: h.vpnIp.ToIP(),
- LocalIndex: h.localIndexId,
- RemoteIndex: h.remoteIndexId,
- RemoteAddrs: h.remotes.CopyAddrs(preferredRanges),
- CachedPackets: len(h.packetStore),
+ VpnIp: h.vpnIp.ToIP(),
+ LocalIndex: h.localIndexId,
+ RemoteIndex: h.remoteIndexId,
+ RemoteAddrs: h.remotes.CopyAddrs(preferredRanges),
+ CachedPackets: len(h.packetStore),
+ CurrentRelaysToMe: h.relayState.CopyRelayIps(),
+ CurrentRelaysThroughMe: h.relayState.CopyRelayForIps(),
  }
 
  if h.ConnectionState != nil {