Merge pull request yggdrasil-network#552 from yggdrasil-network/develop

Version 0.3.9

.circleci/config.yml

@@ -29,6 +29,7 @@ jobs:
       - run:
           name: Install RPM utilities
           command: |
+              sudo apt-get update
               sudo apt-get install -y rpm file
               mkdir -p ~/rpmbuild/BUILD ~/rpmbuild/RPMS ~/rpmbuild/SOURCES ~/rpmbuild/SPECS ~/rpmbuild/SRPMS
 
@@ -67,13 +68,15 @@ jobs:
               find ~/rpmbuild/SRPMS/ -name '*.rpm' -exec mv {} /tmp/upload \;
 
       - run:
-          name: Build for EdgeRouter
+          name: Build for EdgeRouter and VyOS
           command: |
               rm -f {yggdrasil,yggdrasilctl}
               git clone https://github.com/neilalexander/vyatta-yggdrasil /tmp/vyatta-yggdrasil;
               cd /tmp/vyatta-yggdrasil;
               BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-edgerouter-x $CIRCLE_BRANCH;
               BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-edgerouter-lite $CIRCLE_BRANCH;
+              BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-vyos-i386 $CIRCLE_BRANCH
+              BUILDDIR_YGG=$CIRCLE_WORKING_DIRECTORY ./build-vyos-amd64 $CIRCLE_BRANCH
               mv *.deb /tmp/upload;
 
       - persist_to_workspace:

CHANGELOG.md

@@ -25,6 +25,34 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.3.9] - 2019-09-27
+### Added
+- Yggdrasil will now complain more verbosely when a peer URI is incorrectly formatted
+- Soft-shutdown methods have been added, allowing a node to shut down gracefully when terminated
+- New multicast interval logic which sends multicast beacons more often when Yggdrasil is first started to increase the chance of finding nearby nodes quickly after startup
+
+### Changed
+- The switch now buffers packets more eagerly in an attempt to give the best link a chance to send, which appears to reduce packet reordering when crossing aggregate sets of peerings
+- Substantial amounts of the codebase have been refactored to use the actor model, which should substantially reduce the chance of deadlocks
+- Nonce tracking in sessions has been modified so that memory usage is reduced whilst still only allowing duplicate packets within a small window
+- Soft-reconfiguration support has been simplified using new actor functions
+- The garbage collector threshold has been adjusted for mobile builds
+- The maximum queue size is now managed exclusively by the switch rather than by the core
+
+### Fixed
+- The broken `hjson-go` dependency which affected builds of the previous version has now been resolved in the module manifest
+- Some minor memory leaks in the switch have been fixed, which improves memory usage on mobile builds
+- A memory leak in the add-peer loop has been fixed
+- The admin socket now reports the correct URI strings for SOCKS peers in `getPeers`
+- A race condition when dialling a remote node by both the node address and routed prefix simultaneously has been fixed
+- A race condition between the router and the dial code resulting in a panic has been fixed
+- A panic which could occur when the TUN/TAP interface disappears (e.g. during soft-shutdown) has been fixed
+- A bug in the semantic versioning script which accompanies Yggdrasil for builds has been fixed
+- A panic which could occur when the TUN/TAP interface reads an undersized/corrupted packet has been fixed
+
+### Removed
+- A number of legacy debug functions have now been removed and a number of exported API functions are now better documented
+
 ## [0.3.8] - 2019-08-21
 ### Changed
 - Yggdrasil can now send multiple packets from the switch at once, which results in improved throughput with smaller packets or lower MTUs
@@ -39,10 +67,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - New nonce tracking should help to reduce the number of packets dropped as a result of multiple/aggregate paths or congestion control in the switch
 
 ### Fixed
-- **Security vulnerability**: Address verification was not strict enough, which could result in a malicious session sending traffic with unexpected or spoofed source or destination addresses which Yggdrasil could fail to reject
+- A deadlock was fixed in the session code which could result in Yggdrasil failing to pass traffic after some time
+
+### Security
+- Address verification was not strict enough, which could result in a malicious session sending traffic with unexpected or spoofed source or destination addresses which Yggdrasil could fail to reject
   - Versions `0.3.6` and `0.3.7` are vulnerable - users of these versions should upgrade as soon as possible
   - Versions `0.3.5` and earlier are not affected
-- A deadlock was fixed in the session code which could result in Yggdrasil failing to pass traffic after some time
 
 ## [0.3.7] - 2019-08-14
 ### Changed

README.md

@@ -37,7 +37,7 @@ some of the below:
 - NetBSD
 - OpenWrt
 
-Please see our [Platforms](https://yggdrasil-network.github.io/) pages for more
+Please see our [Platforms](https://yggdrasil-network.github.io/platforms.html) pages for more
 specific information about each of our supported platforms, including
 installation steps and caveats.
 

build

@@ -34,13 +34,15 @@ if [ $IOS ]; then
   gomobile bind -target ios -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
     github.com/yggdrasil-network/yggdrasil-go/src/config \
-    github.com/yggdrasil-network/yggdrasil-extras/src/mobile
+    github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
+    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
 elif [ $ANDROID ]; then
   echo "Building aar for Android"
   gomobile bind -target android -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
     github.com/yggdrasil-network/yggdrasil-go/src/config \
-    github.com/yggdrasil-network/yggdrasil-extras/src/mobile
+    github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
+    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
 else
   for CMD in `ls cmd/` ; do
     echo "Building: $CMD"

cmd/yggdrasil/main.go

@@ -279,6 +279,7 @@ func main() {
 		case _ = <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
+				logger.Infoln("Reloading configuration from", *useconffile)
 				n.core.UpdateConfig(cfg)
 				n.tuntap.UpdateConfig(cfg)
 				n.multicast.UpdateConfig(cfg)
@@ -291,11 +292,10 @@ exit:
 }
 
 func (n *node) shutdown() {
-	n.core.Stop()
 	n.admin.Stop()
 	n.multicast.Stop()
 	n.tuntap.Stop()
-	os.Exit(0)
+	n.core.Stop()
 }
 
 func (n *node) sessionFirewall(pubkey *crypto.BoxPubKey, initiator bool) bool {

contrib/semver/name.sh

@@ -6,7 +6,7 @@ BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null)
 # Complain if the git history is not available
 if [ $? != 0 ] || [ -z "$BRANCH" ]; then
   printf "yggdrasil"
-  exit 1
+  exit 0
 fi
 
 # Remove "/" characters from the branch name if present

contrib/semver/version.sh

@@ -6,7 +6,7 @@ TAG=$(git describe --abbrev=0 --tags --match="v[0-9]*\.[0-9]*\.[0-9]*" 2>/dev/nu
 # Did getting the tag succeed?
 if [ $? != 0 ] || [ -z "$TAG" ]; then
   printf -- "unknown"
-  exit 1
+  exit 0
 fi
 
 # Get the current branch
@@ -36,7 +36,7 @@ if [ "$BRANCH" != "master" ]; then
   # Did getting the count of commits since the tag succeed?
   if [ $? != 0 ] || [ -z "$BUILD" ]; then
     printf -- "-unknown"
-    exit 1
+    exit 0
   fi
 
   # Is the build greater than zero?

contrib/systemd/yggdrasil.service

@@ -16,6 +16,7 @@ ExecStartPre=/bin/sh -ec "if ! test -s /etc/yggdrasil.conf; \
 ExecStart=/usr/bin/yggdrasil -useconffile /etc/yggdrasil.conf
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

go.mod

@@ -1,9 +1,10 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
+	github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
-	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222
+	github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible
 	github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091

go.sum

@@ -1,9 +1,11 @@
+github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6 h1:zMj5Q1V0yF4WNfV/FpXG6iXfPJ965Xc5asR2vHXanXc=
+github.com/Arceliar/phony v0.0.0-20190907031509-af5bdbeecab6/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222 h1:xmvkbxXDeN1ffWq8kvrhyqVYAO2aXuRBsbpxVTR+JyU=
-github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible h1:bLQ2Ve+eW65id3b8xEMQiAwJT4qGZeywAEMLvXjznvw=
+github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 h1:YnZmFjg0Nvk8851WTVWlqMC1ecJH07Ctz+Ezxx4u54g=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=

src/address/address.go

@@ -1,22 +1,24 @@
+// Package address contains the types used by yggdrasil to represent IPv6 addresses or prefixes, as well as functions for working with these types.
+// Of particular importance are the functions used to derive addresses or subnets from a NodeID, or to get the NodeID and bitmask of the bits visible from an address, which is needed for DHT searches.
 package address
 
 import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 
-// address represents an IPv6 address in the yggdrasil address range.
+// Address represents an IPv6 address in the yggdrasil address range.
 type Address [16]byte
 
-// subnet represents an IPv6 /64 subnet in the yggdrasil subnet range.
+// Subnet represents an IPv6 /64 subnet in the yggdrasil subnet range.
 type Subnet [8]byte
 
-// address_prefix is the prefix used for all addresses and subnets in the network.
+// GetPrefix returns the address prefix used by yggdrasil.
 // The current implementation requires this to be a muliple of 8 bits + 7 bits.
 // The 8th bit of the last byte is used to signal nodes (0) or /64 prefixes (1).
-// Nodes that configure this differently will be unable to communicate with eachother, though routing and the DHT machinery *should* still work.
+// Nodes that configure this differently will be unable to communicate with eachother using IP packets, though routing and the DHT machinery *should* still work.
 func GetPrefix() [1]byte {
 	return [...]byte{0x02}
 }
 
-// isValid returns true if an address falls within the range used by nodes in the network.
+// IsValid returns true if an address falls within the range used by nodes in the network.
 func (a *Address) IsValid() bool {
 	prefix := GetPrefix()
 	for idx := range prefix {
@@ -27,7 +29,7 @@ func (a *Address) IsValid() bool {
 	return true
 }
 
-// isValid returns true if a prefix falls within the range usable by the network.
+// IsValid returns true if a prefix falls within the range usable by the network.
 func (s *Subnet) IsValid() bool {
 	prefix := GetPrefix()
 	l := len(prefix)
@@ -39,8 +41,8 @@ func (s *Subnet) IsValid() bool {
 	return (*s)[l-1] == prefix[l-1]|0x01
 }
 
-// address_addrForNodeID takes a *NodeID as an argument and returns an *address.
-// This subnet begins with the address prefix, with the last bit set to 0 to indicate an address.
+// AddrForNodeID takes a *NodeID as an argument and returns an *Address.
+// This address begins with the contents of GetPrefix(), with the last bit set to 0 to indicate an address.
 // The following 8 bits are set to the number of leading 1 bits in the NodeID.
 // The NodeID, excluding the leading 1 bits and the first leading 0 bit, is truncated to the appropriate length and makes up the remainder of the address.
 func AddrForNodeID(nid *crypto.NodeID) *Address {
@@ -80,7 +82,7 @@ func AddrForNodeID(nid *crypto.NodeID) *Address {
 	return &addr
 }
 
-// address_subnetForNodeID takes a *NodeID as an argument and returns a *subnet.
+// SubnetForNodeID takes a *NodeID as an argument and returns an *Address.
 // This subnet begins with the address prefix, with the last bit set to 1 to indicate a prefix.
 // The following 8 bits are set to the number of leading 1 bits in the NodeID.
 // The NodeID, excluding the leading 1 bits and the first leading 0 bit, is truncated to the appropriate length and makes up the remainder of the subnet.
@@ -96,10 +98,10 @@ func SubnetForNodeID(nid *crypto.NodeID) *Subnet {
 	return &snet
 }
 
-// getNodeIDandMask returns two *NodeID.
-// The first is a NodeID with all the bits known from the address set to their correct values.
-// The second is a bitmask with 1 bit set for each bit that was known from the address.
-// This is used to look up NodeIDs in the DHT and tell if they match an address.
+// GetNodeIDandMask returns two *NodeID.
+// The first is a NodeID with all the bits known from the Address set to their correct values.
+// The second is a bitmask with 1 bit set for each bit that was known from the Address.
+// This is used to look up NodeIDs in the DHT and tell if they match an Address.
 func (a *Address) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	// Mask is a bitmask to mark the bits visible from the address
 	// This means truncated leading 1s, first leading 0, and visible part of addr
@@ -126,10 +128,10 @@ func (a *Address) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	return &nid, &mask
 }
 
-// getNodeIDandMask returns two *NodeID.
-// The first is a NodeID with all the bits known from the address set to their correct values.
-// The second is a bitmask with 1 bit set for each bit that was known from the subnet.
-// This is used to look up NodeIDs in the DHT and tell if they match a subnet.
+// GetNodeIDandMask returns two *NodeID.
+// The first is a NodeID with all the bits known from the Subnet set to their correct values.
+// The second is a bitmask with 1 bit set for each bit that was known from the Subnet.
+// This is used to look up NodeIDs in the DHT and tell if they match a Subnet.
 func (s *Subnet) GetNodeIDandMask() (*crypto.NodeID, *crypto.NodeID) {
 	// As with the address version, but visible parts of the subnet prefix instead
 	var nid crypto.NodeID