work around Docker’s non-deterministic network interface assignment (#43)

The proxy is set up with two Docker networks. This creates two network interfaces, eth0 and eth1, in the proxy container. However, the setup is not deterministic: We don't know which interface will be assigned to which network.

To restore determinism, we therefore need to determine if eth0 is facing the client and eth1 is facing the server, or vice versa.

Author: marten-seemann

integration/docker-compose.yml

@@ -10,6 +10,8 @@ services:
       - ASSIGN_ADDR=${CLIENT_ASSIGN_IP} # address to assign to the client
       - ROUTE=${ROUTE_SUBNET} # route advertised to the client (into the server's subnet)
       - FILTER_IP_PROTOCOL=${FILTER_IP_PROTOCOL}
+      - GATEWAY_IPV4=${GATEWAY_IPV4}
+      - GATEWAY_IPV6=${GATEWAY_IPV6}
       - QUIC_GO_DISABLE_RECEIVE_BUFFER_WARNING=true
       - QUIC_GO_DISABLE_GSO=true
     cap_add:

integration/proxy/proxy.go

@@ -28,7 +28,7 @@ import (
 
 var serverSocketRcv, serverSocketSend int
 
-const ifaceName = "eth1"
+var ifaceName = os.Getenv("SERVER_INTERFACE")
 
 func main() {
 	proxyPort, err := strconv.Atoi(os.Getenv("PROXY_PORT"))
@@ -57,11 +57,18 @@ func main() {
 		log.Fatalf("failed to get addresses for %s: %v", ifaceName, err)
 	}
 	if len(addrs) == 0 {
-		log.Fatalf("no IPv4 addresses found for %s", ifaceName)
+		log.Fatalf("no IP addresses found for %s", ifaceName)
 	}
-	ethAddr, ok := netip.AddrFromSlice(addrs[0].IP)
-	if !ok {
-		log.Fatalf("failed to parse %s address", ifaceName)
+	var ethAddr netip.Addr
+	for _, addr := range addrs {
+		a, ok := netip.AddrFromSlice(addr.IP)
+		if !ok {
+			log.Fatalf("failed to parse %s address", ifaceName)
+		}
+		if !a.IsLinkLocalUnicast() {
+			ethAddr = a
+			break
+		}
 	}
 
 	fdRcv, err := createReceiveSocket(ethAddr)
@@ -90,7 +97,7 @@ func createReceiveSocket(a netip.Addr) (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("creating socket: %w", err)
 	}
-	iface, err := net.InterfaceByName("eth1")
+	iface, err := net.InterfaceByName(ifaceName)
 	if err != nil {
 		return 0, fmt.Errorf("interface lookup failed: %w", err)
 	}
@@ -122,7 +129,7 @@ func createSendSocketIPv4(addr netip.Addr) (int, error) {
 	sa := &unix.SockaddrInet4{Port: 0} // raw sockets don't use ports
 	copy(sa.Addr[:], addr.AsSlice())
 	if err := unix.Bind(fd, sa); err != nil {
-		return 0, fmt.Errorf("binding socket: %w", err)
+		return 0, fmt.Errorf("binding socket to %s: %w", addr, err)
 	}
 	return fd, nil
 }
@@ -138,7 +145,7 @@ func createSendSocketIPv6(addr netip.Addr) (int, error) {
 	sa := &unix.SockaddrInet6{Port: 0} // raw sockets don't use ports
 	copy(sa.Addr[:], addr.AsSlice())
 	if err := unix.Bind(fd, sa); err != nil {
-		return 0, fmt.Errorf("binding socket: %w", err)
+		return 0, fmt.Errorf("binding socket to %s: %w", addr, err)
 	}
 	return fd, nil
 }

integration/proxy/run.sh

@@ -2,6 +2,27 @@
 
 set -e
 
+# The proxy is set up with two Docker networks.
+# This creates two network interfaces, eth0 and eth1.
+# However, the setup is not deterministic:
+# We don't know which interface will be assigned to which network.
+# We therefore need to determine which interface is facing the client, and which one the server.
+SERVER_INTERFACE="eth1"
+if [ -n "$GATEWAY_IPV4" ] && ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1 | grep -q "$GATEWAY_IPV4"; then
+  SERVER_INTERFACE="eth0"
+fi
+if [ -n "$GATEWAY_IPV6" ] && ip addr show eth0 | grep 'inet6 ' | awk '{print $2}' | cut -d/ -f1 | grep -q "$GATEWAY_IPV6"; then
+  SERVER_INTERFACE="eth0"
+fi
+
+echo "eth0:"
+ip addr show eth0
+echo "eth1:"
+ip addr show eth1
+
+echo "Server facing interface: $SERVER_INTERFACE"
+export SERVER_INTERFACE
+
 ethtool -K eth0 tx off
 ethtool -K eth1 tx off