feat: implement service type NodePort

Author: lucasl0st

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -447,6 +447,18 @@ spec:
                   enable_replica_pooler_load_balancer:
                     type: boolean
                     default: false
+                  enable_master_node_port:
+                    type: boolean
+                    default: false
+                  enable_master_pooler_node_port:
+                    type: boolean
+                    default: false
+                  enable_replica_node_port:
+                    type: boolean
+                    default: false
+                  enable_replica_pooler_node_port:
+                    type: boolean
+                    default: false
                   external_traffic_policy:
                     type: string
                     enum:

charts/postgres-operator/crds/postgresqls.yaml

@@ -196,6 +196,26 @@ spec:
                 type: boolean
               enableReplicaPoolerLoadBalancer:
                 type: boolean
+              enableMasterNodePort:
+                type: boolean
+              masterNodePort:
+                type: integer
+                minimum: 0
+              enableMasterPoolerNodePort:
+                type: boolean
+              masterPoolerNodePort:
+                type: integer
+                minimum: 0
+              enableReplicaNodePort:
+                type: boolean
+              replicaNodePort:
+                type: integer
+                minimum: 0
+              enableReplicaPoolerNodePort:
+                type: boolean
+              replicaPoolerNodePort:
+                type: integer
+                minimum: 0
               enableShmVolume:
                 type: boolean
               env:

docs/administrator.md

@@ -926,6 +926,41 @@ For the `external-dns.alpha.kubernetes.io/hostname` annotation the `-pooler`
 suffix will be appended to the cluster name used in the template which is
 defined in `master|replica_dns_name_format`.
 
+## Node Ports
+
+Alternatively to Load Balancers Node Ports can be used. Kubernetes services with type
+`NodePort` redirect traffic from a specified port on your kubernetes nodes to your service.
+To expose your services to an external network with NodePorts you can set `enableMasterNodePort` and/or `enableReplicaNodePort` to `true`
+in your cluster manifest. In the case any of these variables are omitted from the manifest, the operator configuration settings `enable_master_node_port` and `enable_replica_node_port` apply.
+Note that the operator settings affect all Postgresql services running in all namespaces watched
+by the operator.
+
+**Enabling a NodePort configuration will override the corresponding LoadBalancer configuration.**
+
+There are multiple options to specify service annotations that will be merged
+with each other and override in the following order (where latter take
+precedence):
+
+1. Globally configured `custom_service_annotations`
+2. `serviceAnnotations` specified in the cluster manifest
+3. `masterServiceAnnotations` and `replicaServiceAnnotations` specified in the cluster manifest
+
+Load-Balancer specific annotations are not applied.
+
+Node port services can also be configured for the [connection pooler](user.md#connection-pooler) pods
+with the manifest flags `enableMasterPoolerNodePort` and/or `enableReplicaPoolerNodePort` or in the operator configuration with `enable_master_pooler_node_port`
+and/or `enable_replica_pooler_node_port`.
+
+To configure which ports Kubernetes should use for your NodePort service you can configure ports in your cluster manifest
+for each type:
+
+- masterNodePort
+- masterPoolerNodePort
+- replicaNodePort
+- replicaPoolerNodePort
+
+When not defined or set to 0 kubernetes will choose a port for you from [your kubernetes cluster's configured range](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport).
+
 ## Running periodic 'autorepair' scans of K8s objects
 
 The Postgres Operator periodically scans all K8s objects belonging to each

docs/reference/cluster_manifest.md

@@ -114,6 +114,48 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   this parameter. Optional, when empty the load balancer service becomes
   inaccessible from outside of the Kubernetes cluster.
 
+* **enableMasterNodePort**
+  boolean flag to override the operator defaults (set by the
+  `enable_master_node_port` parameter) to define whether to enable the node
+  port pointing to the Postgres primary. Optional. Overrides `enableMasterLoadBalancer`.
+
+* **enableMasterPoolerNodePort**
+  boolean flag to override the operator defaults (set by the
+  `enable_master_pooler_node_port` parameter) to define whether to enable
+  the node port for master pooler pods pointing to the Postgres primary.
+  Optional. Overrides `enableMasterPoolerLoadBalancer`.
+
+* **enableReplicaNodePort**
+  boolean flag to override the operator defaults (set by the
+  `enable_replica_node_port` parameter) to define whether to enable the node
+  port pointing to the Postgres standby instances. Optional. Overrides `enableReplicaLoadBalancer`.
+
+* **enableReplicaPoolerNodePort**
+  boolean flag to override the operator defaults (set by the
+  `enable_replica_pooler_node_port` parameter) to define whether to enable
+  the node port for replica pooler pods pointing to the Postgres standby
+  instances. Optional. Overrides `enableReplicaPoolerLoadBalancer`.
+
+* **masterNodePort**
+  integer flag to specify a port number for the node port to the Postgres primary.
+  Only used when `enableMasterNodePort` or `enable_master_node_port` are enabled.
+  Optional. Kubernetes will provide a port number for you if not specified.
+
+* **masterPoolerNodePort**
+  integer flag to specify a port number for the node port for the master pooler pods pointing to the Postgres primary.
+  Only used when `enableMasterPoolerNodePort` or `enable_master_pooler_node_port` are enabled.
+  Optional. Kubernetes will provide a port number for you if not specified.
+
+* **replicaNodePort**
+  integer flag to specify a port number for the node port pointing to the Postgres standby instances.
+  Only used when `enableReplicaNodePort` or `enable_replica_node_port` are enabled.
+  Optional. Kubernetes will provide a port number for you if not specified.
+
+* **replicaPoolerNodePort**
+  integer flag to specify a port number for the node port for the replica pooler pods pointing to the Postgres standby instances
+  Only used when `enableReplicaPoolerNodePort` or `enable_replica_pooler_node_port` are enabled.
+  Optional. Kubernetes will provide a port number for you if not specified.
+
 * **maintenanceWindows**
   a list which defines specific time frames when certain maintenance operations
   such as automatic major upgrades or master pod migration. Accepted formats

pkg/apis/acid.zalan.do/v1/crds.go

@@ -312,6 +312,34 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 					"enableReplicaPoolerLoadBalancer": {
 						Type: "boolean",
 					},
+					"enableMasterNodePort": {
+						Type: "boolean",
+					},
+					"masterNodePort": {
+						Type:    "integer",
+						Minimum: &min0,
+					},
+					"enableMasterPoolerNodePort": {
+						Type: "boolean",
+					},
+					"masterPoolerNodePort": {
+						Type:    "integer",
+						Minimum: &min0,
+					},
+					"enableReplicaNodePort": {
+						Type: "boolean",
+					},
+					"replicaNodePort": {
+						Type:    "integer",
+						Minimum: &min0,
+					},
+					"enableReplicaPoolerNodePort": {
+						Type: "boolean",
+					},
+					"replicaPoolerNodePort": {
+						Type:    "integer",
+						Minimum: &min0,
+					},
 					"enableShmVolume": {
 						Type: "boolean",
 					},
@@ -1661,6 +1689,18 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"enable_replica_pooler_load_balancer": {
 								Type: "boolean",
 							},
+							"enable_master_node_port": {
+								Type: "boolean",
+							},
+							"enable_master_pooler_node_port": {
+								Type: "boolean",
+							},
+							"enable_replica_node_port": {
+								Type: "boolean",
+							},
+							"enable_replica_pooler_node_port": {
+								Type: "boolean",
+							},
 							"external_traffic_policy": {
 								Type: "string",
 								Enum: []apiextv1.JSON{

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -136,17 +136,24 @@ type OperatorTimeouts struct {
 
 // LoadBalancerConfiguration defines the LB configuration
 type LoadBalancerConfiguration struct {
-	DbHostedZone                    string                `json:"db_hosted_zone,omitempty"`
-	EnableMasterLoadBalancer        bool                  `json:"enable_master_load_balancer,omitempty"`
-	EnableMasterPoolerLoadBalancer  bool                  `json:"enable_master_pooler_load_balancer,omitempty"`
-	EnableReplicaLoadBalancer       bool                  `json:"enable_replica_load_balancer,omitempty"`
-	EnableReplicaPoolerLoadBalancer bool                  `json:"enable_replica_pooler_load_balancer,omitempty"`
-	CustomServiceAnnotations        map[string]string     `json:"custom_service_annotations,omitempty"`
-	MasterDNSNameFormat             config.StringTemplate `json:"master_dns_name_format,omitempty"`
-	MasterLegacyDNSNameFormat       config.StringTemplate `json:"master_legacy_dns_name_format,omitempty"`
-	ReplicaDNSNameFormat            config.StringTemplate `json:"replica_dns_name_format,omitempty"`
-	ReplicaLegacyDNSNameFormat      config.StringTemplate `json:"replica_legacy_dns_name_format,omitempty"`
-	ExternalTrafficPolicy           string                `json:"external_traffic_policy" default:"Cluster"`
+	DbHostedZone                    string `json:"db_hosted_zone,omitempty"`
+	EnableMasterLoadBalancer        bool   `json:"enable_master_load_balancer,omitempty"`
+	EnableMasterPoolerLoadBalancer  bool   `json:"enable_master_pooler_load_balancer,omitempty"`
+	EnableReplicaLoadBalancer       bool   `json:"enable_replica_load_balancer,omitempty"`
+	EnableReplicaPoolerLoadBalancer bool   `json:"enable_replica_pooler_load_balancer,omitempty"`
+
+	// kept in LoadBalancerConfiguration because all the other parameters apply here too
+	EnableMasterNodePort        bool `json:"enable_master_node_port,omitempty"`
+	EnableMasterPoolerNodePort  bool `json:"enable_master_pooler_node_port,omitempty"`
+	EnableReplicaNodePort       bool `json:"enable_replica_node_port,omitempty"`
+	EnableReplicaPoolerNodePort bool `json:"enable_replica_pooler_node_port,omitempty"`
+
+	CustomServiceAnnotations   map[string]string     `json:"custom_service_annotations,omitempty"`
+	MasterDNSNameFormat        config.StringTemplate `json:"master_dns_name_format,omitempty"`
+	MasterLegacyDNSNameFormat  config.StringTemplate `json:"master_legacy_dns_name_format,omitempty"`
+	ReplicaDNSNameFormat       config.StringTemplate `json:"replica_dns_name_format,omitempty"`
+	ReplicaLegacyDNSNameFormat config.StringTemplate `json:"replica_legacy_dns_name_format,omitempty"`
+	ExternalTrafficPolicy      string                `json:"external_traffic_policy" default:"Cluster"`
 }
 
 // AWSGCPConfiguration defines the configuration for AWS

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -50,6 +50,18 @@ type PostgresSpec struct {
 	EnableReplicaLoadBalancer       *bool `json:"enableReplicaLoadBalancer,omitempty"`
 	EnableReplicaPoolerLoadBalancer *bool `json:"enableReplicaPoolerLoadBalancer,omitempty"`
 
+	// vars to enable and configure nodeport services
+	// set ports to 0 or nil to let kubernetes decide which port to use
+	// overrides loadbalancer configuration
+	EnableMasterNodePort        *bool  `json:"enableMasterNodePort,omitempty"`
+	MasterNodePort              *int32 `json:"masterNodePort,omitempty"`
+	EnableMasterPoolerNodePort  *bool  `json:"enableMasterPoolerNodePort,omitempty"`
+	MasterPoolerNodePort        *int32 `json:"masterPoolerNodePort,omitempty"`
+	EnableReplicaNodePort       *bool  `json:"enableReplicaNodePort,omitempty"`
+	ReplicaNodePort             *int32 `json:"replicaNodePort,omitempty"`
+	EnableReplicaPoolerNodePort *bool  `json:"enableReplicaPoolerNodePort,omitempty"`
+	ReplicaPoolerNodePort       *int32 `json:"replicaPoolerNodePort,omitempty"`
+
 	// deprecated load balancer settings maintained for backward compatibility
 	// see "Load balancers" operator docs
 	UseLoadBalancer     *bool `json:"useLoadBalancer,omitempty"`

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -849,6 +849,14 @@ func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) {
 		return false, "new service's ExternalTrafficPolicy does not match the current one"
 	}
 
+	if len(old.Spec.Ports) > 0 && len(new.Spec.Ports) > 0 {
+		// we need to check whether the new port is not zero (=user-defined)
+		// and only overwrite if it is
+		if new.Spec.Ports[0].NodePort != 0 && old.Spec.Ports[0].NodePort != new.Spec.Ports[0].NodePort {
+			return false, "new service's NodePort does not match the current one"
+		}
+	}
+
 	return true, ""
 }
 

pkg/cluster/cluster.go

@@ -508,6 +508,10 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo
 		c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges)
 	}
 
+	if ok, port := c.shouldCreateNodePortForPoolerService(poolerRole, spec); ok {
+		c.configureNodePortService(&serviceSpec, port)
+	}
+
 	service := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        connectionPooler.Name,
@@ -532,7 +536,10 @@ func (c *Cluster) generatePoolerServiceAnnotations(role PostgresRole, spec *acid
 	var dnsString string
 	annotations := c.getCustomServiceAnnotations(role, spec)
 
-	if c.shouldCreateLoadBalancerForPoolerService(role, spec) {
+	// we do not want the load balancer specific annotations on a NodePort service
+	nodePort, _ := c.shouldCreateNodePortForPoolerService(role, spec)
+
+	if c.shouldCreateLoadBalancerForPoolerService(role, spec) && !nodePort {
 		// set ELB Timeout annotation with default value
 		if _, ok := annotations[constants.ElbTimeoutAnnotationName]; !ok {
 			annotations[constants.ElbTimeoutAnnotationName] = constants.ElbTimeoutAnnotationValue
@@ -577,6 +584,37 @@ func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, sp
 	}
 }
 
+func (c *Cluster) shouldCreateNodePortForPoolerService(role PostgresRole, spec *acidv1.PostgresSpec) (bool, int32) {
+	switch role {
+	case Replica:
+		// if the value is explicitly set in a Postgresql manifest, follow this setting
+		if spec.EnableReplicaPoolerNodePort != nil {
+			port := int32(0)
+			if spec.ReplicaPoolerNodePort != nil {
+				port = *spec.ReplicaPoolerNodePort
+			}
+
+			return *spec.EnableReplicaPoolerNodePort, port
+		}
+
+		// otherwise, follow the operator configuration
+		return c.OpConfig.EnableReplicaPoolerNodePort, 0
+	case Master:
+		if spec.EnableMasterPoolerNodePort != nil {
+			port := int32(0)
+			if spec.MasterPoolerNodePort != nil {
+				port = *spec.MasterPoolerNodePort
+			}
+
+			return *spec.EnableMasterPoolerNodePort, port
+		}
+
+		return c.OpConfig.EnableMasterPoolerNodePort, 0
+	default:
+		panic(fmt.Sprintf("Unknown role %v", role))
+	}
+}
+
 func (c *Cluster) listPoolerPods(listOptions metav1.ListOptions) ([]v1.Pod, error) {
 	pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions)
 	if err != nil {