Merge pull request #7208 from GeorgianaElena/eks-upgrade

[projectpythia] Eks upgrade, k8s upgrade, separate nodepools

Author: GeorgianaElena

config/clusters/projectpythia/prod.values.yaml

@@ -19,6 +19,9 @@ jupyterhub:
       templateVars:
         org:
           name: ProjectPythia
+  singleuser:
+    nodeSelector:
+      2i2c/hub-name: prod
   hub:
     config:
       GitHubOAuthenticator:

config/clusters/projectpythia/pythia-binder.values.yaml

@@ -19,6 +19,7 @@ jupyterhub:
       # Schedule users on the smallest instance
       # https://github.com/2i2c-org/infrastructure/issues/4241
       node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: pythia-binder
     storage:
       type: none
       extraVolumeMounts: []

config/clusters/projectpythia/staging.values.yaml

@@ -17,6 +17,9 @@ jupyterhub:
       templateVars:
         org:
           name: projectpythia staging
+  singleuser:
+    nodeSelector:
+      2i2c/hub-name: staging
   hub:
     config:
       GitHubOAuthenticator:

config/clusters/projectpythia/support.values.yaml

@@ -35,3 +35,8 @@ cluster-autoscaler:
 
 calico:
   enabled: true
+
+# FIXME: remove this once eksctl is fixed
+nvidiaDevicePlugin:
+  aws:
+    enabled: true

eksctl/projectpythia.jsonnet

@@ -1,185 +1,24 @@
-/*
-    This file is a jsonnet template of a eksctl's cluster configuration file,
-    that is used with the eksctl CLI to both update and initialize an AWS EKS
-    based cluster.
-
-    This file has in turn been generated from eksctl/template.jsonnet which is
-    relevant to compare with for changes over time.
-
-    To use jsonnet to generate an eksctl configuration file from this, do:
-
-        jsonnet projectpythia.jsonnet > projectpythia.eksctl.yaml
-
-    References:
-    - https://eksctl.io/usage/schema/
-*/
-local ng = import './libsonnet/nodegroup.jsonnet';
-
-// place all cluster nodes here
-local clusterRegion = 'us-west-2';
-local masterAzs = ['us-west-2a', 'us-west-2b', 'us-west-2c'];
-local nodeAz = 'us-west-2a';
-
-// Node definitions for notebook nodes. Config here is merged
-// with our notebook node definition.
-// A `node.kubernetes.io/instance-type label is added, so pods
-// can request a particular kind of node with a nodeSelector
-local notebookNodes = [
-  { instanceType: 'r5.xlarge' },
-  { instanceType: 'r5.4xlarge' },
-  { instanceType: 'r5.16xlarge' },
-  {
-    instanceType: 'g4dn.xlarge',
-    minSize: 0,
-    tags+: {
-      'k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu': '1',
-      'k8s.io/cluster-autoscaler/node-template/label/k8s.amazonaws.com/accelerator': 'nvidia-tesla-t4',
-    },
-    taints+: {
-      'nvidia.com/gpu': 'present:NoSchedule',
-    },
-    labels+: {
-      '2i2c/has-gpu': 'true',
-      'k8s.amazonaws.com/accelerator': 'nvidia-tesla-t4',
-    },
-    // Allow provisioning GPUs across all AZs, to prevent situation where all
-    // GPUs in a single AZ are in use and no new nodes can be spawned
-    availabilityZones: masterAzs,
-  },
-];
-local daskNodes = [];
-
-
-{
-  apiVersion: 'eksctl.io/v1alpha5',
-  kind: 'ClusterConfig',
-  metadata+: {
-    name: 'projectpythia',
-    region: clusterRegion,
-    version: '1.32',
-    tags+: {
-      ManagedBy: '2i2c',
-      '2i2c.org/cluster-name': $.metadata.name,
-    },
-  },
-  availabilityZones: masterAzs,
-  iam: {
-    withOIDC: true,
-  },
-  // If you add an addon to this config, run the create addon command.
-  //
-  //    eksctl create addon --config-file=projectpythia.eksctl.yaml
-  //
-  addons: [
-    { version: 'latest', tags: $.metadata.tags } + addon
-    for addon in
-      [
-        { name: 'coredns' },
-        { name: 'kube-proxy' },
-        {
-          // vpc-cni is a Amazon maintained container networking interface
-          // (CNI), where a CNI is required for k8s networking. The aws-node
-          // DaemonSet in kube-system stems from installing this.
-          //
-          // Related docs: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/
-          //               https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html
-          //
-          name: 'vpc-cni',
-          attachPolicyARNs: ['arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy'],
-          // configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
-          configurationValues: |||
-            enableNetworkPolicy: "false"
-          |||,
-        },
-        {
-          // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-          // couple to AWS EBS based storage, without it expect to see pods
-          // mounting a PVC failing to schedule and PVC resources that are
-          // unbound.
-          //
-          // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-          //
-          name: 'aws-ebs-csi-driver',
-          wellKnownPolicies: {
-            ebsCSIController: true,
-          },
-          // We enable detailed metrics collection to watch for issues with
-          // jupyterhub-home-nfs~
-          // configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml
-          configurationValues: |||
-            defaultStorageClass:
-                enabled: true
-            controller:
-                enableMetrics: true
-            node:
-                enableMetrics: true
-          |||,
-        },
-      ]
+local cluster = import './libsonnet/cluster.jsonnet';
+
+local c = cluster.makeCluster(
+  name='projectpythia',
+  region='us-west-2',
+  nodeAz='us-west-2a',
+  version='1.34',
+  coreNodeInstanceType='r8i-flex.large',
+  notebookCPUInstanceTypes=[
+    'r5.xlarge',
+    'r5.4xlarge',
+    'r5.16xlarge',
   ],
-  nodeGroups: [
-    n { clusterName: $.metadata.name }
-    for n in
-      [
-        ng {
-          namePrefix: 'core',
-          nameSuffix: 'a',
-          nameIncludeInstanceType: false,
-          availabilityZones: [nodeAz],
-          instanceType: 'r5.xlarge',
-          minSize: 1,
-          maxSize: 6,
-          labels+: {
-            'hub.jupyter.org/node-purpose': 'core',
-            'k8s.dask.org/node-purpose': 'core',
-          },
-        },
-      ] + [
-        ng {
-          namePrefix: 'nb',
-          availabilityZones: [nodeAz],
-          minSize: 0,
-          maxSize: 500,
-          instanceType: n.instanceType,
-          labels+: {
-            'hub.jupyter.org/node-purpose': 'user',
-            'k8s.dask.org/node-purpose': 'scheduler',
-          },
-          taints+: {
-            'hub.jupyter.org_dedicated': 'user:NoSchedule',
-            'hub.jupyter.org/dedicated': 'user:NoSchedule',
-          },
-          tags+: {
-            '2i2c:node-purpose': 'user',
-          },
-        } + n
-        for n in notebookNodes
-      ] + (
-        if daskNodes != null then
-          [
-            ng {
-              namePrefix: 'dask',
-              availabilityZones: [nodeAz],
-              minSize: 0,
-              maxSize: 500,
-              labels+: {
-                'k8s.dask.org/node-purpose': 'worker',
-              },
-              taints+: {
-                'k8s.dask.org_dedicated': 'worker:NoSchedule',
-                'k8s.dask.org/dedicated': 'worker:NoSchedule',
-              },
-              tags+: {
-                '2i2c:node-purpose': 'worker',
-              },
-              instancesDistribution+: {
-                onDemandBaseCapacity: 0,
-                onDemandPercentageAboveBaseCapacity: 0,
-                spotAllocationStrategy: 'capacity-optimized',
-              },
-            } + n
-            for n in daskNodes
-          ] else []
-      )
+  daskInstanceTypes=[],
+  hubs=['staging', 'prod', 'pythia-binder'],
+  notebookGPUNodeGroups=[
+    {
+      instanceType: 'g4dn.xlarge',
+    },
   ],
-}
+  nodeGroupGenerations=['a'],
+);
+
+c