Swap

Author: bobbyiliev

README.md

@@ -203,7 +203,7 @@ No resources.
 | <a name="input_cert_manager_namespace"></a> [cert\_manager\_namespace](#input\_cert\_manager\_namespace) | The name of the namespace in which cert-manager is or will be installed. | `string` | `"cert-manager"` | no |
 | <a name="input_database_config"></a> [database\_config](#input\_database\_config) | Azure Database for PostgreSQL configuration | <pre>object({<br/>    sku_name         = optional(string, "GP_Standard_D2s_v3")<br/>    postgres_version = optional(string, "15")<br/>    password         = string<br/>    username         = optional(string, "materialize")<br/>    db_name          = optional(string, "materialize")<br/>  })</pre> | n/a | yes |
 | <a name="input_disk_setup_image"></a> [disk\_setup\_image](#input\_disk\_setup\_image) | Docker image for the disk setup script | `string` | `"materialize/ephemeral-storage-setup-image:v0.3.4"` | no |
-| <a name="input_disk_support_config"></a> [disk\_support\_config](#input\_disk\_support\_config) | Advanced configuration for disk support (only used when enable\_disk\_support = true) | <pre>object({<br/>    install_openebs       = optional(bool, true)<br/>    run_disk_setup_script = optional(bool, true)<br/>    create_storage_class  = optional(bool, true)<br/>    openebs_version       = optional(string, "4.2.0")<br/>    openebs_namespace     = optional(string, "openebs")<br/>    storage_class_name    = optional(string, "openebs-lvm-instance-store-ext4")<br/>  })</pre> | `{}` | no |
+| <a name="input_disk_support_config"></a> [disk\_support\_config](#input\_disk\_support\_config) | Advanced configuration for disk support (only used when enable\_disk\_support = true) | <pre>object({<br/>    install_openebs       = optional(bool, true)<br/>    run_disk_setup_script = optional(bool, true)<br/>    create_storage_class  = optional(bool, true)<br/>    openebs_version       = optional(string, "4.3.3")<br/>    openebs_namespace     = optional(string, "openebs")<br/>    storage_class_name    = optional(string, "openebs-lvm-instance-store-ext4")<br/>  })</pre> | `{}` | no |
 | <a name="input_enable_disk_support"></a> [enable\_disk\_support](#input\_enable\_disk\_support) | Enable disk support for Materialize using OpenEBS and local SSDs. When enabled, this configures OpenEBS, runs the disk setup script, and creates appropriate storage classes. | `bool` | `true` | no |
 | <a name="input_helm_chart"></a> [helm\_chart](#input\_helm\_chart) | Chart name from repository or local path to chart. For local charts, set the path to the chart directory. | `string` | `"materialize-operator"` | no |
 | <a name="input_helm_values"></a> [helm\_values](#input\_helm\_values) | Additional Helm values to merge with defaults | `any` | `{}` | no |
@@ -218,6 +218,7 @@ No resources.
 | <a name="input_orchestratord_version"></a> [orchestratord\_version](#input\_orchestratord\_version) | Version of the Materialize orchestrator to install | `string` | `null` | no |
 | <a name="input_prefix"></a> [prefix](#input\_prefix) | Prefix to be used for resource names | `string` | `"materialize"` | no |
 | <a name="input_resource_group_name"></a> [resource\_group\_name](#input\_resource\_group\_name) | The name of an existing resource group to use | `string` | n/a | yes |
+| <a name="input_swap_enabled"></a> [swap\_enabled](#input\_swap\_enabled) | Enable swap for Materialize. When enabled, this configures swap on a new nodepool, and adds it to the clusterd node selectors. | `bool` | `false` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | `{}` | no |
 | <a name="input_use_local_chart"></a> [use\_local\_chart](#input\_use\_local\_chart) | Whether to use a local chart instead of one from a repository | `bool` | `false` | no |
 | <a name="input_use_self_signed_cluster_issuer"></a> [use\_self\_signed\_cluster\_issuer](#input\_use\_self\_signed\_cluster\_issuer) | Whether to install and use a self-signed ClusterIssuer for TLS. To work around limitations in Terraform, this will be treated as `false` if no materialize instances are defined. | `bool` | `true` | no |

examples/simple/main.tf

@@ -91,6 +91,8 @@ module "materialize" {
 
   materialize_instances = var.materialize_instances
 
+  swap_enabled = var.swap_enabled
+
   database_config = {
     sku_name = "GP_Standard_D2s_v3"
     version  = "15"
@@ -188,6 +190,12 @@ variable "use_self_signed_cluster_issuer" {
   default     = true
 }
 
+variable "swap_enabled" {
+  description = "Enable swap for Materialize. When enabled, this configures swap on a new nodepool, and adds it to the clusterd node selectors."
+  type        = bool
+  default     = false
+}
+
 # Output the Materialize instance details
 output "aks_cluster" {
   description = "AKS cluster details"

main.tf

@@ -59,6 +59,9 @@ module "aks" {
   openebs_version   = local.disk_config.openebs_version
   disk_setup_image  = var.disk_setup_image
 
+  # Swap configuration
+  swap_enabled = var.swap_enabled
+
   tags = local.common_labels
 }
 
@@ -123,7 +126,7 @@ locals {
         region = var.location
       }
       clusters = {
-        swap_enabled = false
+        swap_enabled = var.swap_enabled
       }
     }
     observability = {

modules/aks/main.tf

@@ -1,5 +1,7 @@
 locals {
-  nodepool_name = substr(replace(var.prefix, "-", ""), 0, 12)
+  nodepool_name      = substr(replace(var.prefix, "-", ""), 0, 12)
+  swap_nodepool_name = substr(replace("${var.prefix}-swap", "-", ""), 0, 12)
+  disk_setup_name    = "disk-setup-scratchfs"
 }
 
 resource "azurerm_user_assigned_identity" "aks_identity" {
@@ -84,12 +86,46 @@ resource "azurerm_kubernetes_cluster_node_pool" "materialize" {
     "materialize.cloud/disk-config-required" = var.enable_disk_setup ? "true" : "false"
   }
 
+  upgrade_settings {
+    max_surge                     = "10%"
+    drain_timeout_in_minutes      = 0
+    node_soak_duration_in_minutes = 0
+  }
+
   # Taints can not be removed: https://github.com/Azure/AKS/issues/2934
   # node_taints = var.enable_disk_setup ? ["materialize.cloud/disk-unconfigured=true:NoSchedule"] : []
 
   tags = var.tags
 }
 
+resource "azurerm_kubernetes_cluster_node_pool" "swap" {
+  count                       = var.swap_enabled ? 1 : 0
+  name                        = local.swap_nodepool_name
+  temporary_name_for_rotation = "${local.swap_nodepool_name}2"
+  kubernetes_cluster_id       = azurerm_kubernetes_cluster.aks.id
+  vm_size                     = var.vm_size
+  auto_scaling_enabled        = true
+  min_count                   = var.min_nodes
+  max_count                   = var.max_nodes
+  vnet_subnet_id              = var.subnet_id
+  os_disk_size_gb             = var.disk_size_gb
+
+  node_labels = {
+    "materialize.cloud/scratch-fs"           = "true"
+    "materialize.cloud/swap"                 = "true"
+    "workload"                               = "materialize-instance"
+    "materialize.cloud/disk-config-required" = "true"
+  }
+
+  upgrade_settings {
+    max_surge                     = "10%"
+    drain_timeout_in_minutes      = 0
+    node_soak_duration_in_minutes = 0
+  }
+
+  tags = var.tags
+}
+
 resource "kubernetes_namespace" "openebs" {
   count = var.install_openebs ? 1 : 0
 
@@ -182,7 +218,7 @@ resource "kubernetes_namespace" "disk_setup" {
   count = var.enable_disk_setup ? 1 : 0
 
   metadata {
-    name = "disk-setup"
+    name = local.disk_setup_name
     labels = {
       "app.kubernetes.io/managed-by" = "terraform"
       "app.kubernetes.io/part-of"    = "materialize"
@@ -198,26 +234,26 @@ resource "kubernetes_daemonset" "disk_setup" {
   count = var.enable_disk_setup ? 1 : 0
 
   metadata {
-    name      = "disk-setup"
+    name      = local.disk_setup_name
     namespace = kubernetes_namespace.disk_setup[0].metadata[0].name
     labels = {
       "app.kubernetes.io/managed-by" = "terraform"
       "app.kubernetes.io/part-of"    = "materialize"
-      "app"                          = "disk-setup"
+      "app"                          = local.disk_setup_name
     }
   }
 
   spec {
     selector {
       match_labels = {
-        app = "disk-setup"
+        app = local.disk_setup_name
       }
     }
 
     template {
       metadata {
         labels = {
-          app = "disk-setup"
+          app = local.disk_setup_name
         }
       }
 
@@ -236,7 +272,7 @@ resource "kubernetes_daemonset" "disk_setup" {
             required_during_scheduling_ignored_during_execution {
               node_selector_term {
                 match_expressions {
-                  key      = "materialize.cloud/disk"
+                  key      = "materialize.cloud/scratch-fs"
                   operator = "In"
                   values   = ["true"]
                 }
@@ -346,15 +382,15 @@ resource "kubernetes_daemonset" "disk_setup" {
 resource "kubernetes_service_account" "disk_setup" {
   count = var.enable_disk_setup ? 1 : 0
   metadata {
-    name      = "disk-setup"
+    name      = local.disk_setup_name
     namespace = kubernetes_namespace.disk_setup[0].metadata[0].name
   }
 }
 
 resource "kubernetes_cluster_role" "disk_setup" {
   count = var.enable_disk_setup ? 1 : 0
   metadata {
-    name = "disk-setup"
+    name = local.disk_setup_name
   }
   rule {
     api_groups = [""]
@@ -366,7 +402,7 @@ resource "kubernetes_cluster_role" "disk_setup" {
 resource "kubernetes_cluster_role_binding" "disk_setup" {
   count = var.enable_disk_setup ? 1 : 0
   metadata {
-    name = "disk-setup"
+    name = local.disk_setup_name
   }
   role_ref {
     api_group = "rbac.authorization.k8s.io"

modules/aks/variables.tf

@@ -90,3 +90,9 @@ variable "disk_setup_image" {
   type        = string
   default     = "materialize/ephemeral-storage-setup-image:v0.1.2"
 }
+
+variable "swap_enabled" {
+  description = "Enable swap for Materialize. When enabled, this configures swap on a new nodepool."
+  type        = bool
+  default     = false
+}

variables.tf

@@ -185,6 +185,12 @@ variable "enable_disk_support" {
   default     = true
 }
 
+variable "swap_enabled" {
+  description = "Enable swap for Materialize. When enabled, this configures swap on a new nodepool, and adds it to the clusterd node selectors."
+  type        = bool
+  default     = false
+}
+
 variable "disk_support_config" {
   description = "Advanced configuration for disk support (only used when enable_disk_support = true)"
   type = object({