diff --git a/bufstream-shared-bucket.tf b/bufstream-shared-bucket.tf new file mode 100644 index 00000000..45a25bac --- /dev/null +++ b/bufstream-shared-bucket.tf @@ -0,0 +1,63 @@ +# Shared Bufstream bucket accessible from all namespaces in the EKS cluster +# Since all Kubernetes namespaces run on nodes that use the same node role, +# attaching the policy to the node role gives access from all namespaces + +# AWS S3 bucket for shared bufstream access +resource "aws_s3_bucket" "bufstream_shared" { + bucket = "${var.namespace}-bufstream-sa-demo" + force_destroy = false + + tags = { + Namespace = var.namespace + Role = "bufstream-shared-bucket" + } +} + +# IAM Policy for S3 bucket access +resource "aws_iam_policy" "bufstream_shared" { + name = "${var.namespace}-bufstream-sa-demo-policy" + description = "Policy for access to shared bufstream bucket from all namespaces" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowAllS3ActionsOnSharedBufstreamBucket" + Effect = "Allow" + Action = [ + "s3:*" + ] + Resource = [ + aws_s3_bucket.bufstream_shared.arn, + "${aws_s3_bucket.bufstream_shared.arn}/*" + ] + } + ] + }) + + tags = { + Namespace = var.namespace + } +} + +# Attach policy to the node role (used by all pods across all namespaces) +resource "aws_iam_role_policy_attachment" "bufstream_shared" { + role = module.app_eks.node_role.name + policy_arn = aws_iam_policy.bufstream_shared.arn +} + +# Outputs +output "bufstream_shared_bucket_name" { + description = "Name of the shared bufstream S3 bucket" + value = aws_s3_bucket.bufstream_shared.bucket +} + +output "bufstream_shared_bucket" { + description = "Shared bufstream S3 bucket" + value = { + id = aws_s3_bucket.bufstream_shared.id + arn = aws_s3_bucket.bufstream_shared.arn + bucket = aws_s3_bucket.bufstream_shared.bucket + policy_arn = aws_iam_policy.bufstream_shared.arn + } +} diff --git a/data.tf b/data.tf index 161a8bc7..cb179de4 100644 --- a/data.tf +++ b/data.tf @@ -10,3 +10,5 @@ data "aws_sqs_queue" "file_storage" { } data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} diff --git a/main.tf b/main.tf index 08944469..afdec5be 100644 --- a/main.tf +++ b/main.tf @@ -426,6 +426,30 @@ locals { parquet = { extraEnv = var.parquet_wandb_env } + + weave-trace-worker = { + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.app_eks.weave_worker_iam_role_arn + } + } + secretsStore = { + enabled = true + } + } + + secretsStore = { + enabled = true + provider = "aws" + secrets = [ + { + name = "weave-worker-auth" + cloudSecretName = module.app_eks.weave_worker_auth_secret_name + k8sSecretName = "weave-worker-auth" + k8sSecretKey = "key" + } + ] + } } } } diff --git a/modules/app_eks/iam-policies.tf b/modules/app_eks/iam-policies.tf index a46b9ecd..e8c693c9 100644 --- a/modules/app_eks/iam-policies.tf +++ b/modules/app_eks/iam-policies.tf @@ -63,3 +63,10 @@ resource "aws_iam_policy" "irsa" { ] }) } + +# IAM Policy for Weave Workers +resource "aws_iam_policy" "weave_worker" { + name = "${var.namespace}-weave-worker-secrets-access" + description = "Weave worker IRSA policy for accessing secrets" + policy = data.aws_iam_policy_document.weave_worker_secrets_access.json +} diff --git a/modules/app_eks/iam-policy-docs.tf b/modules/app_eks/iam-policy-docs.tf index d58322f0..af282b0c 100644 --- a/modules/app_eks/iam-policy-docs.tf +++ b/modules/app_eks/iam-policy-docs.tf @@ -127,3 +127,14 @@ data "aws_iam_policy_document" "secrets_manager" { resources = ["arn:aws:secretsmanager:*:${data.aws_caller_identity.current.account_id}:secret:${var.namespace}*"] } } + +data "aws_iam_policy_document" "weave_worker_secrets_access" { + statement { + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ] + effect = "Allow" + resources = ["arn:aws:secretsmanager:*:${data.aws_caller_identity.current.account_id}:secret:${var.namespace}-*"] + } +} diff --git a/modules/app_eks/iam-role-attachments.tf b/modules/app_eks/iam-role-attachments.tf index 92f0ff09..9d70ed50 100644 --- a/modules/app_eks/iam-role-attachments.tf +++ b/modules/app_eks/iam-role-attachments.tf @@ -59,3 +59,9 @@ resource "aws_iam_policy_attachment" "irsa" { roles = [aws_iam_role.irsa.name] policy_arn = aws_iam_policy.irsa.arn } + +# Attach Weave Worker Policy to Weave Worker Role +resource "aws_iam_role_policy_attachment" "weave_worker" { + role = aws_iam_role.weave_worker.name + policy_arn = aws_iam_policy.weave_worker.arn +} diff --git a/modules/app_eks/iam-roles.tf b/modules/app_eks/iam-roles.tf index fd2dfc4d..3993c487 100644 --- a/modules/app_eks/iam-roles.tf +++ b/modules/app_eks/iam-roles.tf @@ -27,3 +27,27 @@ resource "aws_iam_role" "irsa" { ] }) } + +# IAM Role for Weave Workers +resource "aws_iam_role" "weave_worker" { + name = "${var.namespace}-weave-worker-irsa" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${aws_iam_openid_connect_provider.eks.url}" + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringLike = { + "${aws_iam_openid_connect_provider.eks.url}:sub" = "system:serviceaccount:${var.k8s_namespace}:*" + "${aws_iam_openid_connect_provider.eks.url}:aud" = "sts.amazonaws.com" + } + } + } + ] + }) +} diff --git a/modules/app_eks/main.tf b/modules/app_eks/main.tf index f9c6355e..bf5b4f82 100644 --- a/modules/app_eks/main.tf +++ b/modules/app_eks/main.tf @@ -205,6 +205,17 @@ module "cluster_autoscaler" { ] } +module "secrets_store" { + source = "./secrets_store" + + secrets_store_csi_driver_version = var.secrets_store_csi_driver_version + secrets_store_csi_driver_provider_aws_version = var.secrets_store_csi_driver_provider_aws_version + + depends_on = [ + module.eks + ] +} + # Weave worker authentication token resource "random_password" "weave_worker_auth" { length = 32 @@ -212,7 +223,7 @@ resource "random_password" "weave_worker_auth" { } resource "aws_secretsmanager_secret" "weave_worker_auth" { - name = "${var.namespace}-weave-worker-auth" + name = "${var.namespace}-weave-worker-auth-00" recovery_window_in_days = 0 tags = { diff --git a/modules/app_eks/outputs.tf b/modules/app_eks/outputs.tf index b50376a3..2fe80949 100644 --- a/modules/app_eks/outputs.tf +++ b/modules/app_eks/outputs.tf @@ -21,3 +21,13 @@ output "primary_workers_security_group_id" { output "aws_iam_openid_connect_provider" { value = aws_iam_openid_connect_provider.eks.url } + +output "weave_worker_auth_secret_name" { + value = aws_secretsmanager_secret.weave_worker_auth.name + description = "Name of the AWS Secrets Manager secret containing the weave worker auth token (used by SecretProviderClass)" +} + +output "weave_worker_iam_role_arn" { + value = aws_iam_role.weave_worker.arn + description = "ARN of the IAM role for weave worker service accounts to access AWS Secrets Manager via CSI driver" +} diff --git a/modules/app_eks/secrets_store/secrets_store.tf b/modules/app_eks/secrets_store/secrets_store.tf new file mode 100644 index 00000000..832ff993 --- /dev/null +++ b/modules/app_eks/secrets_store/secrets_store.tf @@ -0,0 +1,40 @@ +# Install Secrets Store CSI Driver via Helm +resource "helm_release" "secrets_store_csi_driver" { + name = "secrets-store-csi-driver" + repository = "https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts" + chart = "secrets-store-csi-driver" + version = var.secrets_store_csi_driver_version + namespace = "kube-system" + + set { + name = "syncSecret.enabled" + value = "true" + } + + set { + name = "enableSecretRotation" + value = "true" + } + + set { + name = "rotationPollInterval" + value = "120s" + } +} + +# Install AWS Secrets Manager Provider for Secrets Store CSI Driver +resource "helm_release" "secrets_store_csi_driver_provider_aws" { + depends_on = [ + helm_release.secrets_store_csi_driver + ] + + name = "secrets-store-csi-driver-provider-aws" + repository = "https://aws.github.io/secrets-store-csi-driver-provider-aws" + chart = "secrets-store-csi-driver-provider-aws" + version = var.secrets_store_csi_driver_provider_aws_version + namespace = "kube-system" +} + +# NOTE: The SecretProviderClass is created by the application Helm chart (operator-wandb), +# not by Terraform. This avoids CRD timing issues and keeps application-specific configuration +# with the application deployment. diff --git a/modules/app_eks/secrets_store/variables.tf b/modules/app_eks/secrets_store/variables.tf new file mode 100644 index 00000000..016d6ebf --- /dev/null +++ b/modules/app_eks/secrets_store/variables.tf @@ -0,0 +1,9 @@ +variable "secrets_store_csi_driver_version" { + type = string + description = "The version of the Secrets Store CSI Driver Helm chart to install." +} + +variable "secrets_store_csi_driver_provider_aws_version" { + type = string + description = "The version of the AWS Secrets Manager Provider for Secrets Store CSI Driver Helm chart to install." +} diff --git a/modules/app_eks/variables.tf b/modules/app_eks/variables.tf index 0cfa8341..322bb8b3 100644 --- a/modules/app_eks/variables.tf +++ b/modules/app_eks/variables.tf @@ -254,3 +254,15 @@ variable "cluster_autoscaler_image_tag" { description = "The tag of the cluster-autoscaler to deploy." default = null } + +variable "secrets_store_csi_driver_version" { + type = string + description = "The version of the Secrets Store CSI Driver Helm chart to install." + default = "1.4.7" +} + +variable "secrets_store_csi_driver_provider_aws_version" { + type = string + description = "The version of the AWS Secrets Manager Provider for Secrets Store CSI Driver Helm chart to install." + default = "0.3.9" +}