feat(ci): add zookeeper e2e test (#43)

Co-authored-by: soulbird <[email protected]>

Author: soulbird

ci/apisix-seed/conf.yaml

@@ -23,3 +23,9 @@ discovery:                       # service discovery center
       connect: 2000              # default 2000ms
       send: 2000                 # default 2000ms
       read: 5000                 # default 5000ms
+  zookeeper:
+    hosts:
+      - "172.50.238.40:2181"
+    prefix: /zookeeper
+    weight: 100                  # default weight for node
+    timeout: 10                  # default 10s

test/e2e/go.mod

@@ -3,6 +3,7 @@ module e2e
 go 1.17
 
 require (
+	github.com/go-zookeeper/zk v1.0.2
 	github.com/onsi/ginkgo/v2 v2.0.0
 	github.com/onsi/gomega v1.18.1
 )

test/e2e/go.sum

@@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM=
+github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=

test/e2e/regcenter/nacos_test.go

@@ -11,7 +11,7 @@ import (
 var _ = BeforeSuite(func() {
 	Expect(tools.CleanRoutes()).To(BeNil())
 	Expect(tools.NewIRegCenter("nacos").Clean()).To(BeNil())
-
+	Expect(tools.NewIRegCenter("zookeeper").Clean()).To(BeNil())
 })
 
 func TestRegcenter(t *testing.T) {

test/e2e/regcenter/regcenter_suite_test.go

@@ -0,0 +1,59 @@
+package regcenter_test
+
+import (
+	"e2e/tools"
+	"e2e/tools/common"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Normal test", Ordered, func() {
+	Context("single route, one server", func() {
+
+		type normalCase struct {
+			URI        string
+			Route      *tools.Route
+			Server     *tools.SimServer
+			Reg        tools.IRegCenter
+			ExpectBody string
+		}
+
+		DescribeTable("general logic", Ordered,
+			func(tc normalCase) {
+				Expect(tools.CreateRoutes([]*tools.Route{tc.Route})).To(BeNil())
+				//create sim server
+				Expect(tools.CreateSimServer([]*tools.SimServer{tc.Server})).To(BeNil())
+
+				// upstream server online
+				Expect(tc.Server.Register(tc.Reg)).To(BeNil())
+				time.Sleep(3 * time.Second)
+				status, body, err := common.Request(tc.URI)
+				Expect(err).To(BeNil())
+				Expect(status).To(Equal(200))
+				Expect(body).To(Equal(tc.ExpectBody))
+				// upstream server offline
+				Expect(tc.Server.LogOut(tc.Reg)).To(BeNil())
+				time.Sleep(3 * time.Second)
+				status, _, err = common.Request(tc.URI)
+				Expect(err).To(BeNil())
+				Expect(status).To(Equal(503))
+			},
+			Entry("Nacos", normalCase{
+				URI:        "/test1",
+				Route:      tools.NewRoute("1", "/test1", "APISIX-NACOS", "nacos"),
+				Server:     tools.NewSimServer("0.0.0.0", "9990", "APISIX-NACOS"),
+				Reg:        tools.NewIRegCenter("nacos"),
+				ExpectBody: "response: 0.0.0.0:9990",
+			}),
+			Entry("Zookeeper", normalCase{
+				URI:        "/test2",
+				Route:      tools.NewRoute("2", "/test2", "APISIX-ZK", "zookeeper"),
+				Server:     tools.NewSimServer("0.0.0.0", "9991", "APISIX-ZK"),
+				Reg:        tools.NewIRegCenter("zookeeper"),
+				ExpectBody: "response: 0.0.0.0:9991",
+			}),
+		)
+	})
+})

test/e2e/regcenter/regcenter_test.go

@@ -7,6 +7,7 @@ import (
 
 const (
 	NACOS_HOST     = "http://127.0.0.1:8848"
+	ZK_HOST        = "127.0.0.1:2181"
 	APISIX_HOST    = "http://127.0.0.1:9080"
 	APISIX_TOKEN   = "edd1c9f034335f136f87ad84b625c8f1"
 	DOCKER_GATEWAY = "172.50.238.1"

test/e2e/tools/common/common.go

@@ -16,6 +16,8 @@ func NewIRegCenter(name string) IRegCenter {
 	switch name {
 	case "nacos":
 		return regcenter.NewNacos()
+	case "zookeeper":
+		return regcenter.NewZookeeper()
 	}
 	return nil
 }

test/e2e/tools/regcenter.go

@@ -110,7 +110,7 @@ func (n *Nacos) deleteService(service string) error {
 	if resp.StatusCode != 200 {
 		return errors.New("delete service failed, serviceName=:" + service)
 	}
-	fmt.Println("delete service, serviceName=:" + service)
+	fmt.Println("delete service, serviceName=" + service)
 	return nil
 }
 

test/e2e/tools/regcenter/nacos.go

@@ -0,0 +1,66 @@
+package regcenter
+
+import (
+	"e2e/tools/common"
+	"fmt"
+	"time"
+
+	"github.com/go-zookeeper/zk"
+)
+
+type Zookeeper struct {
+	conn   *zk.Conn
+	prefix string
+}
+
+func NewZookeeper() *Zookeeper {
+	conn, _, err := zk.Connect([]string{common.ZK_HOST}, time.Second*5)
+	if err != nil {
+		panic(err)
+	}
+	return &Zookeeper{
+		conn:   conn,
+		prefix: "/zookeeper",
+	}
+}
+
+func (zookeeper *Zookeeper) Online(node *common.Node) error {
+	nodeStr := `{"host":"` + common.DOCKER_GATEWAY + `","port":` + node.Port + `}`
+	_, err := zookeeper.conn.Create(zookeeper.prefix+"/"+node.ServiceName, []byte(nodeStr), 0, zk.WorldACL(zk.PermAll))
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("register instance to Zookeeper: ", node.String())
+	return err
+}
+
+func (zookeeper *Zookeeper) Offline(node *common.Node) error {
+	path := zookeeper.prefix + "/" + node.ServiceName
+	_, stat, err := zookeeper.conn.Exists(path)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("offline instance to Zookeeper: ", node.String())
+	return zookeeper.conn.Delete(path, stat.Version)
+
+}
+
+func (zookeeper *Zookeeper) Clean() error {
+	fmt.Println("clean all service form zookeeper...")
+	children, stat, err := zookeeper.conn.Children(zookeeper.prefix)
+	if err != nil {
+		return err
+	}
+	for _, p := range children {
+		if p == "config" || p == "quota" {
+			continue
+		}
+		if err := zookeeper.conn.Delete(zookeeper.prefix+"/"+p, stat.Version); err != nil {
+			return err
+		}
+		fmt.Println("delete service, serviceName=", p)
+	}
+	return nil
+}