Skip to content

Commit f9a5b3c

Browse files
plan9betterplan9better
committed
feat: use explicit routing & chainsaw e2e tests
This version introduces explicit routing instead of the non descriptive localGateway remoteGateway localRoutes remoteReoutes etc. This gives the user more control over the final setup. We have also moved from kuttl to chainsaw tests, they seem faster, have better looking output and generally work the same.
1 parent ec922a5 commit f9a5b3c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1130
-858
lines changed

‎.chainsaw.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha2
2+
kind: Configuration
3+
metadata:
4+
name: example
5+
spec:
6+
cleanup:
7+
skipDelete: false
8+
execution:
9+
failFast: true
10+
parallel: 1

‎Makefile‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@ publish:
1818
docker push $(DOCKERHUB_USER)/vlan-interface:$(VERSION)
1919

2020
test:
21-
docker build -t $(LOCAL_REGISTRY)/vlanman:dev --platform linux/amd64 --file ./operator.Dockerfile --build-arg PLATFORM=amd64 .
22-
docker push $(LOCAL_REGISTRY)/vlanman:dev
21+
docker build -t $(DOCKERHUB_USER)/vlanman:dev --platform linux/amd64 --file ./operator.Dockerfile --build-arg PLATFORM=amd64 .
22+
docker push $(DOCKERHUB_USER)/vlanman:dev
2323

24-
docker build -t $(LOCAL_REGISTRY)/vlan-manager:dev --platform linux/amd64 --file ./manager.Dockerfile --build-arg PLATFORM=amd64 .
25-
docker push $(LOCAL_REGISTRY)/vlan-manager:dev
24+
docker build -t $(DOCKERHUB_USER)/vlan-manager:dev --platform linux/amd64 --file ./manager.Dockerfile --build-arg PLATFORM=amd64 .
25+
docker push $(DOCKERHUB_USER)/vlan-manager:dev
2626

27-
docker build -t $(LOCAL_REGISTRY)/vlan-worker:dev --platform linux/amd64 --file ./worker.Dockerfile --build-arg PLATFORM=amd64 .
28-
docker push $(LOCAL_REGISTRY)/vlan-worker:dev
27+
docker build -t $(DOCKERHUB_USER)/vlan-worker:dev --platform linux/amd64 --file ./worker.Dockerfile --build-arg PLATFORM=amd64 .
28+
docker push $(DOCKERHUB_USER)/vlan-worker:dev
2929

30-
docker build -t $(LOCAL_REGISTRY)/vlan-interface:dev --platform linux/amd64 --file ./interface.Dockerfile --build-arg PLATFORM=amd64 .
31-
docker push $(LOCAL_REGISTRY)/vlan-interface:dev
30+
docker build -t $(DOCKERHUB_USER)/vlan-interface:dev --platform linux/amd64 --file ./interface.Dockerfile --build-arg PLATFORM=amd64 .
31+
docker push $(DOCKERHUB_USER)/vlan-interface:dev
3232

3333
test-local:
3434
docker build -t $(LOCAL_REGISTRY)/vlanman:$(VERSION) --platform linux/arm64 --file ./operator.Dockerfile --build-arg PLATFORM=arm64 .

‎api/v1/vlanman_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ type Route struct {
6969
// Source determines how the source IP is selected for this route. Allowed values: "self": use an IP assigned from the current VLAN pool, "none": no source IP (use default behavior)
7070
// +kubebuilder:validation:Enum=self;none
7171
Source string `json:"src"`
72+
// ScopeLink determines whether the scope of the route will be set to 'LINK', for routes to the gateway it is required.
73+
// +optional
74+
ScopeLink bool `json:"scopeLink"`
7275
}
7376

7477
type VlanNetworkPool struct {

‎cmd/manager/main.go‎

Lines changed: 89 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"syscall"
2121
"time"
2222

23+
vlanmanv1 "dialo.ai/vlanman/api/v1"
2324
"dialo.ai/vlanman/pkg/comms"
2425
errs "dialo.ai/vlanman/pkg/errors"
2526
"github.com/prometheus/client_golang/prometheus"
@@ -174,15 +175,17 @@ func addIPAddress(ctx context.Context) {
174175
})
175176
os.Exit(1)
176177
}
177-
err = ip.AddrAdd(link, &ip.Addr{IPNet: &gatewayIPNet})
178-
if err != nil && !isFileExistsErr(err) {
179-
logger.Error("Failed to add IP address to VLAN when becoming leader", "msg", &errs.UnrecoverableError{
180-
Context: "Error adding ip address to vlan on becoming leader in callback",
181-
Err: err,
182-
})
183-
os.Exit(1)
178+
for _, ipnet := range gatewayIPNets {
179+
err = ip.AddrAdd(link, &ip.Addr{IPNet: &ipnet})
180+
if err != nil && !isFileExistsErr(err) {
181+
logger.Error("Failed to add IP address to VLAN when becoming leader", "msg", &errs.UnrecoverableError{
182+
Context: "Error adding ip address to vlan on becoming leader in callback",
183+
Err: err,
184+
})
185+
os.Exit(1)
186+
}
184187
}
185-
setupRoutes(os.Getenv("REMOTE_ROUTES"), os.Getenv("LOCAL_ROUTES"))
188+
setupRoutes()
186189
}
187190
func removeIPAddress() {
188191
link, err := ip.LinkByName("macvlangw" + strconv.FormatInt(int64(envs.vlanID), 10))
@@ -193,13 +196,15 @@ func removeIPAddress() {
193196
})
194197
os.Exit(1)
195198
}
196-
err = ip.AddrDel(link, &ip.Addr{IPNet: &gatewayIPNet})
197-
if err != nil {
198-
logger.Error("msg", "Failed to delete IP address from VLAN when stopped leading", &errs.UnrecoverableError{
199-
Context: "Error deleting ip address from vlan on stopped leading in callback",
200-
Err: err,
201-
})
202-
os.Exit(1)
199+
for _, ipnet := range gatewayIPNets {
200+
err = ip.AddrAdd(link, &ip.Addr{IPNet: &ipnet})
201+
if err != nil && !isFileExistsErr(err) {
202+
logger.Error("Failed to delete IP address to VLAN when stopped leading", "msg", &errs.UnrecoverableError{
203+
Context: "Error deleting ip address from vlan on stopped leading in callback",
204+
Err: err,
205+
})
206+
os.Exit(1)
207+
}
203208
}
204209
}
205210

@@ -212,51 +217,53 @@ var (
212217
vlanWatcher *VlanWatcher = nil
213218
gatewayLink *ip.Macvlan
214219
vlanID int
215-
gatewayIPNet net.IPNet
220+
gatewayIPNets []net.IPNet
216221
remoteRoutes string
217222
leaderChanges atomic.Int64
218223
lastLeaderChange atomic.Value
219224
localRoutes string
220225
)
221226

222-
func setupRoutes(rem, loc string) error {
227+
func setupRoutes() error {
223228
var link ip.Link
224229
if gatewayLink != nil {
225230
link = gatewayLink
226231
} else {
227232
link = vlanWatcher.Link
228233
}
229-
remote := strings.SplitSeq(rem, ",")
230-
for r := range remote {
231-
_, ipnet, err := net.ParseCIDR(r)
232-
if err != nil {
233-
return err
234-
}
235-
route := ip.Route{
236-
LinkIndex: link.Attrs().Index,
237-
Dst: ipnet,
238-
}
239-
err = ip.RouteAdd(&route)
240-
if err != nil {
241-
return err
242-
}
243-
}
244-
if gatewayLink == nil {
245-
return nil
246-
}
247234

248-
for r := range strings.SplitSeq(loc, ",") {
249-
_, ipnet, err := net.ParseCIDR(r)
250-
if err != nil {
251-
return err
252-
}
253-
route := ip.Route{
254-
LinkIndex: (*gatewayLink).Attrs().Index,
255-
Dst: ipnet,
256-
}
257-
err = ip.RouteAdd(&route)
258-
if err != nil {
259-
return err
235+
for _, gw := range envs.Gateways {
236+
for _, r := range gw.Routes {
237+
_, ipnet, err := net.ParseCIDR(r.Destination)
238+
if err != nil {
239+
return err
240+
}
241+
242+
route := ip.Route{}
243+
if r.Via != nil {
244+
_, gwIpnet, err := net.ParseCIDR(*r.Via)
245+
if err != nil {
246+
return err
247+
}
248+
249+
route = ip.Route{
250+
LinkIndex: link.Attrs().Index,
251+
Dst: ipnet,
252+
Gw: gwIpnet.IP,
253+
}
254+
} else {
255+
route = ip.Route{
256+
LinkIndex: link.Attrs().Index,
257+
Dst: ipnet,
258+
}
259+
}
260+
if r.Source == "self" {
261+
route.Src = net.ParseIP(gw.Address)
262+
}
263+
err = ip.RouteAdd(&route)
264+
if err != nil {
265+
return err
266+
}
260267
}
261268
}
262269
return nil
@@ -278,12 +285,11 @@ func begin() context.Context {
278285
}
279286

280287
type Envs struct {
281-
ownerNetName string
282-
namespace string
283-
vlanID int
284-
lockName string
285-
localGatewayIP string
286-
isLocalGatewayOn bool
288+
ownerNetName string
289+
namespace string
290+
vlanID int
291+
lockName string
292+
Gateways []vlanmanv1.Gateway
287293
}
288294

289295
func getEnvs() Envs {
@@ -316,16 +322,24 @@ func getEnvs() Envs {
316322

317323
namespace := os.Getenv("NAMESPACE")
318324
lockName := os.Getenv("LOCK_NAME")
319-
localGatewayIP := os.Getenv("LOCAL_GATEWAY_IP")
320-
isLocalGatewayOn := localGatewayIP != ""
325+
gatewaysJSON := os.Getenv("GATEWAYS")
326+
327+
gateways := []vlanmanv1.Gateway{}
328+
329+
if gatewaysJSON != "" {
330+
err = json.Unmarshal([]byte(gatewaysJSON), &gateways)
331+
if err != nil {
332+
logger.Error("Couldn't unmarshal GATEWAYS env var", "err", err)
333+
os.Exit(1)
334+
}
335+
}
321336

322337
return Envs{
323-
ownerNetName: ownerNetName,
324-
namespace: namespace,
325-
lockName: lockName,
326-
localGatewayIP: localGatewayIP,
327-
isLocalGatewayOn: isLocalGatewayOn,
328-
vlanID: vlanID,
338+
ownerNetName: ownerNetName,
339+
namespace: namespace,
340+
lockName: lockName,
341+
Gateways: gateways,
342+
vlanID: vlanID,
329343
}
330344
}
331345

@@ -348,21 +362,25 @@ func interfaceSetup(ctx context.Context, e Envs) {
348362
os.Exit(1)
349363
}
350364

351-
if !e.isLocalGatewayOn {
365+
if len(e.Gateways) == 0 {
352366
logger.Info("Skipping leader election, gateway is off")
353-
setupRoutes(os.Getenv("REMOTE_ROUTES"), os.Getenv("LOCAL_ROUTES"))
367+
setupRoutes()
354368
return
355369
}
356370

357-
gatewaySubnet := os.Getenv("LOCAL_GATEWAY_SUBNET")
358-
gwInt, err := strconv.ParseInt(gatewaySubnet, 10, 64)
359-
if err != nil {
360-
gwInt = 32
361-
}
362-
gatewayIPNet = net.IPNet{
363-
IP: net.ParseIP(e.localGatewayIP),
364-
Mask: net.CIDRMask(int(gwInt), 32),
371+
gatewayIPNets = []net.IPNet{}
372+
for _, gw := range e.Gateways {
373+
gwAddr, gwSubnetStr, found := strings.Cut(gw.Address, "/")
374+
if !found {
375+
gwSubnetStr = "32"
376+
}
377+
gwSubnet, _ := strconv.ParseInt(gwSubnetStr, 10, 64)
378+
gatewayIPNets = append(gatewayIPNets, net.IPNet{
379+
IP: net.ParseIP(gwAddr),
380+
Mask: net.CIDRMask(int(gwSubnet), 32),
381+
})
365382
}
383+
366384
attrs := ip.NewLinkAttrs()
367385
attrs.Name = "macvlangw" + strconv.FormatInt(int64(e.vlanID), 10)
368386
attrs.ParentIndex = vlanWatcher.Link.Attrs().Index

‎cmd/worker/main.go‎

Lines changed: 34 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"strconv"
1313
"strings"
1414

15+
vlanmanv1 "dialo.ai/vlanman/api/v1"
1516
"dialo.ai/vlanman/pkg/comms"
1617
errs "dialo.ai/vlanman/pkg/errors"
1718
"github.com/go-logr/logr"
@@ -134,61 +135,52 @@ func main() {
134135
Err: err,
135136
})
136137
}
137-
gwIP := os.Getenv("GATEWAY_IP")
138-
gwSN := os.Getenv("GATEWAY_SUBNET")
139-
isRemoteGW := false
140-
var gwIPNet *net.IPNet
141-
if gwIP != "" {
142-
isRemoteGW = true
143-
if gwSN == "" {
144-
gwSN = "32"
138+
routesJSON := os.Getenv("ROUTES")
139+
140+
routes := []vlanmanv1.Route{}
141+
err = json.Unmarshal([]byte(routesJSON), &routes)
142+
if err != nil {
143+
fatal(&errs.ParsingError{
144+
Source: "Couldn't unmarshal ROUTES env var",
145+
Err: err,
146+
})
147+
}
148+
149+
for _, r := range routes {
150+
addr, maskStr, found := strings.Cut(r.Destination, "/")
151+
if !found {
152+
maskStr = "32"
145153
}
146-
_, gwIPNet, err = net.ParseCIDR(gwIP + "/" + gwSN)
154+
mask, _ := strconv.ParseInt(maskStr, 10, 64)
155+
147156
if err != nil {
148157
fatal(&errs.ParsingError{
149-
Source: "Parsing CIRD of remote gateway",
158+
Source: "CIDR of route",
150159
Err: err,
151160
})
152161
}
153-
gwRoute := ip.Route{
162+
163+
route := ip.Route{
154164
LinkIndex: link.Attrs().Index,
155-
Scope: ip.SCOPE_LINK,
156-
Dst: gwIPNet,
165+
Dst: &net.IPNet{
166+
IP: net.ParseIP(addr),
167+
Mask: net.CIDRMask(int(mask), 32),
168+
},
157169
}
158-
err = ip.RouteAdd(&gwRoute)
159-
if err != nil && !isAlreadyExists(err) {
160-
fatal(&errs.UnrecoverableError{
161-
Context: "Failed to add route to remote gateway",
162-
Err: err,
163-
})
170+
if r.Via != nil {
171+
route.Gw = net.ParseIP(*r.Via)
164172
}
165-
}
166-
routes := strings.SplitSeq(os.Getenv("REMOTE_ROUTES"), ",")
167-
for r := range routes {
168-
_, ipnet, err := net.ParseCIDR(r)
169-
if err != nil {
170-
fatal(&errs.UnrecoverableError{
171-
Context: fmt.Sprintf("Failed to parse route to remote '%s'", r),
172-
Err: err,
173-
})
173+
if r.ScopeLink {
174+
route.Scope = ip.SCOPE_LINK
174175
}
175-
route := ip.Route{}
176-
if isRemoteGW {
177-
route = ip.Route{
178-
LinkIndex: link.Attrs().Index,
179-
Gw: gwIPNet.IP,
180-
Dst: ipnet,
181-
}
182-
} else {
183-
route = ip.Route{
184-
LinkIndex: link.Attrs().Index,
185-
Dst: ipnet,
186-
}
176+
if r.Source == "self" {
177+
route.Src = ipnet.IP
187178
}
179+
188180
err = ip.RouteAdd(&route)
189-
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") {
181+
if err != nil {
190182
fatal(&errs.UnrecoverableError{
191-
Context: fmt.Sprintf("Failed to add route to remote '%s'", r),
183+
Context: fmt.Sprintf("Error adding route %+v", route),
192184
Err: err,
193185
})
194186
}

‎flake.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@
193193
pkgs.glow
194194
pkgs.mods
195195
pkgs.graphviz
196+
pkgs.kyverno-chainsaw
196197
];
197198
shellHook = ''
198199
go mod tidy

0 commit comments

Comments
 (0)