diff --git a/Makefile b/Makefile index 7fa9b95a5..fe102e590 100644 --- a/Makefile +++ b/Makefile @@ -271,7 +271,6 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=true -f - - CONTROLLER_GEN = $(shell pwd)/bin/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. diff --git a/bundle/manifests/argoproj.io_argocds.yaml b/bundle/manifests/argoproj.io_argocds.yaml index 99f457ed9..c1db76584 100644 --- a/bundle/manifests/argoproj.io_argocds.yaml +++ b/bundle/manifests/argoproj.io_argocds.yaml @@ -20098,6 +20098,237 @@ spec: - name type: object type: array + systemCATrust: + description: Custom certificates to inject into the repo server + container and its plugins to trust source hosting sites + properties: + clusterTrustBundles: + description: ClusterTrustBundles is a list of projected ClusterTrustBundle + volume definitions from where to take the trust certs. + items: + description: |- + ClusterTrustBundleProjection describes how to select a set of + ClusterTrustBundle objects and project their contents into the pod + filesystem. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write + the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + type: array + configMaps: + description: ConfigMaps is a list of projected ConfigMap volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a ConfigMap into a projected volume. + + The contents of the target ConfigMap's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names, + unless the items element is populated with specific mappings of keys to paths. + Note that this is identical to a configmap volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + dropImageCertificates: + description: DropImageCertificates will remove all certs that + are present in the image, leaving only those explicitly + configured here. + type: boolean + secrets: + description: Secrets is a list of projected Secret volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a secret into a projected volume. + + The contents of the target Secret's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names. + Note that this is identical to a secret volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object verifytls: description: VerifyTLS defines whether repo server API should be accessed using strict TLS validation diff --git a/bundle/manifests/gitops-operator.clusterserviceversion.yaml b/bundle/manifests/gitops-operator.clusterserviceversion.yaml index db9e859fe..2f10009b6 100644 --- a/bundle/manifests/gitops-operator.clusterserviceversion.yaml +++ b/bundle/manifests/gitops-operator.clusterserviceversion.yaml @@ -180,7 +180,7 @@ metadata: capabilities: Deep Insights console.openshift.io/plugins: '["gitops-plugin"]' containerImage: quay.io/redhat-developer/gitops-operator - createdAt: "2025-10-29T14:30:25Z" + createdAt: "2025-11-04T12:43:19Z" description: Enables teams to adopt GitOps principles for managing cluster configurations and application delivery across hybrid multi-cluster Kubernetes environments. features.operators.openshift.io/disconnected: "true" diff --git a/config/crd/bases/argoproj.io_argocds.yaml b/config/crd/bases/argoproj.io_argocds.yaml index c89f73a5d..9d1450312 100644 --- a/config/crd/bases/argoproj.io_argocds.yaml +++ b/config/crd/bases/argoproj.io_argocds.yaml @@ -20087,6 +20087,237 @@ spec: - name type: object type: array + systemCATrust: + description: Custom certificates to inject into the repo server + container and its plugins to trust source hosting sites + properties: + clusterTrustBundles: + description: ClusterTrustBundles is a list of projected ClusterTrustBundle + volume definitions from where to take the trust certs. + items: + description: |- + ClusterTrustBundleProjection describes how to select a set of + ClusterTrustBundle objects and project their contents into the pod + filesystem. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write + the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + type: array + configMaps: + description: ConfigMaps is a list of projected ConfigMap volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a ConfigMap into a projected volume. + + The contents of the target ConfigMap's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names, + unless the items element is populated with specific mappings of keys to paths. + Note that this is identical to a configmap volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + dropImageCertificates: + description: DropImageCertificates will remove all certs that + are present in the image, leaving only those explicitly + configured here. + type: boolean + secrets: + description: Secrets is a list of projected Secret volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a secret into a projected volume. + + The contents of the target Secret's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names. + Note that this is identical to a secret volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object verifytls: description: VerifyTLS defines whether repo server API should be accessed using strict TLS validation diff --git a/controllers/argocd/openshift/openshift.go b/controllers/argocd/openshift/openshift.go index af37d5694..48699549e 100644 --- a/controllers/argocd/openshift/openshift.go +++ b/controllers/argocd/openshift/openshift.go @@ -4,13 +4,14 @@ import ( "context" "fmt" "os" + "slices" "strings" - "golang.org/x/mod/semver" - argoapp "github.com/argoproj-labs/argocd-operator/api/v1beta1" "github.com/argoproj-labs/argocd-operator/controllers/argocd" - + "github.com/argoproj-labs/argocd-operator/controllers/argoutil" + "github.com/go-logr/logr" + "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -59,6 +60,13 @@ func ReconcilerHook(cr *argoapp.ArgoCD, v interface{}, hint string) error { } else { o.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities = nil } + case cr.Name + "-repo-server": + + prodImage := o.Spec.Template.Spec.Containers[0].Image + usingReleasedImages := strings.Contains(prodImage, "registry.redhat.io/openshift-gitops-1/argocd-rhel") + if cr.Spec.Repo.SystemCATrust != nil && usingReleasedImages { + updateSystemCATrustBuilding(cr, o, prodImage, logv) + } } case *appsv1.StatefulSet: if o.Name == cr.Name+"-redis-ha-server" { @@ -106,6 +114,96 @@ func ReconcilerHook(cr *argoapp.ArgoCD, v interface{}, hint string) error { return nil } +// updateSystemCATrustBuilding replaces the procedure based on ubuntu container with one based on rhel containers. +// This requires changing the init container image, its script and the mount points to all consuming containers. +func updateSystemCATrustBuilding(cr *argoapp.ArgoCD, o *appsv1.Deployment, prodImage string, logv logr.Logger) { + // These volumes are created by argocd-operator + volumeSource := "argocd-ca-trust-source" + volumeTarget := "argocd-ca-trust-target" + + // Drop upstream init container and replace it with rhel specific logic + o.Spec.Template.Spec.InitContainers = slices.DeleteFunc( + o.Spec.Template.Spec.InitContainers, + func(container corev1.Container) bool { + return container.Name == "update-ca-certificates" + }, + ) + + initContainer := corev1.Container{ + Name: "update-ca-certificates", + Image: prodImage, + SecurityContext: argoutil.DefaultSecurityContext(), + VolumeMounts: []corev1.VolumeMount{ + {Name: volumeSource, MountPath: "/var/run/secrets/ca-trust-source", ReadOnly: true}, + {Name: volumeTarget, MountPath: "/etc/pki/ca-trust"}, + }, + Command: []string{"/bin/bash", "-c"}, + Args: []string{` +set -eEuo pipefail +trap 's=$?; echo >&2 "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR + +# Populate the empty volume with the expected structure +mkdir -p /etc/pki/ca-trust/{extracted/{openssl,pem,java,edk2},source/{anchors,blacklist}} + +# Copy user anchors where update-ca-trust expects it +# Using loop over 'cp *' to work well with no CA files provided (all optional, none configured, etc.) +ls /var/run/secrets/ca-trust-source/ | while read -r f; do + cp -L "/var/run/secrets/ca-trust-source/$f" /etc/pki/ca-trust/source/anchors/ +done + +echo "User defined trusted CA files:" +ls /etc/pki/ca-trust/source/anchors/ + +update-ca-trust + +echo "Trusted anchors:" +trust list + +echo "Done!" + `}, + } + + // Replace distro CA certs with empty volume when the image CA's are supposed to be dropped + if cr.Spec.Repo.SystemCATrust.DropImageCertificates { + o.Spec.Template.Spec.Volumes = append(o.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "distro-ca-trust-source", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + initContainer.VolumeMounts = append(initContainer.VolumeMounts, corev1.VolumeMount{ + Name: "distro-ca-trust-source", + MountPath: "/usr/share/pki/ca-trust-source/", + }) + } + o.Spec.Template.Spec.InitContainers = append(o.Spec.Template.Spec.InitContainers, initContainer) + + // Update where to mount for prod containers + var mountedTo []string + for ci, container := range o.Spec.Template.Spec.Containers { + // Only mount to production container or sidecars using the same image + if container.Image != prodImage { + continue + } + mountedTo = append(mountedTo, container.Name) + + // The source volume is not needed on RHEL + o.Spec.Template.Spec.Containers[ci].VolumeMounts = slices.DeleteFunc( + o.Spec.Template.Spec.Containers[ci].VolumeMounts, + func(mount corev1.VolumeMount) bool { + return mount.Name == volumeSource + }, + ) + // Use the RHEL-specific mount point for the target trust volume + for vi, volume := range o.Spec.Template.Spec.Containers[ci].VolumeMounts { + if volume.Name == volumeTarget { + o.Spec.Template.Spec.Containers[ci].VolumeMounts[vi].MountPath = "/etc/pki/ca-trust" + } + } + } + logv.Info(fmt.Sprintf("injected systemCATrust to repo-server containers: %s", strings.Join(mountedTo, ","))) +} + // BuilderHook updates the Argo CD controller builder to watch for changes to the "admin" ClusterRole func BuilderHook(_ *argoapp.ArgoCD, v interface{}, _ string) error { logv := log.WithValues("module", "builder-hook") diff --git a/go.mod b/go.mod index fe25d4240..14049ba19 100644 --- a/go.mod +++ b/go.mod @@ -219,3 +219,5 @@ replace ( k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.33.1 k8s.io/sample-controller => k8s.io/sample-controller v0.33.1 ) + +replace github.com/argoproj-labs/argocd-operator => github.com/olivergondza/argocd-operator v0.14.0-rc1.0.20251104113656-05154993f5fd diff --git a/go.sum b/go.sum index 34854cd8c..d36eb4784 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/argoproj-labs/argo-rollouts-manager v0.0.7-0.20251020065637-7f928e52c0d9 h1:WcUWvh0qIqUaY+JfVIUfBV0ACv9ep2by3YEi04dRNr4= github.com/argoproj-labs/argo-rollouts-manager v0.0.7-0.20251020065637-7f928e52c0d9/go.mod h1:iVIrf0/GJPZR3NMtJvpo1Ui6qqPjpY34Lp+5RmZo9vY= -github.com/argoproj-labs/argocd-operator v0.14.0-rc1.0.20251024105544-f7c3f5b0cc95 h1:v2J4IPd8Fab5udUD7nMZsYflqGDhkVGx30q5uenMBbE= -github.com/argoproj-labs/argocd-operator v0.14.0-rc1.0.20251024105544-f7c3f5b0cc95/go.mod h1:LTBNqNbKk9Us5xiCrK612HLOr8SJFfyxlMJQErzMghg= github.com/argoproj/argo-cd/v3 v3.1.8 h1:NkLPiRI5qGkV+q1EN3O7/0Wb9O/MVl62vadKteZqMUw= github.com/argoproj/argo-cd/v3 v3.1.8/go.mod h1:ZHb/LOz/hr88VWMJiVTd8DGYL7MheHCAT8S6DgYOBFo= github.com/argoproj/gitops-engine v0.7.1-0.20250905160054-e48120133eec h1:rNAwbRQFvRIuW/e2bU+B10mlzghYXsnwZedYeA7Drz4= @@ -262,6 +260,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olivergondza/argocd-operator v0.14.0-rc1.0.20251104113656-05154993f5fd h1:4GuLC7kpXLPMn4yG7E0q2Dzj/Ot9K7WXO4DcnVNs0dQ= +github.com/olivergondza/argocd-operator v0.14.0-rc1.0.20251104113656-05154993f5fd/go.mod h1:LTBNqNbKk9Us5xiCrK612HLOr8SJFfyxlMJQErzMghg= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= diff --git a/test/openshift/e2e/ginkgo/sequential/1-120_repo_server_system_ca_trust.go b/test/openshift/e2e/ginkgo/sequential/1-120_repo_server_system_ca_trust.go new file mode 100644 index 000000000..8a4c29aa4 --- /dev/null +++ b/test/openshift/e2e/ginkgo/sequential/1-120_repo_server_system_ca_trust.go @@ -0,0 +1,862 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sequential + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "regexp" + "strings" + "time" + + "github.com/onsi/gomega/gcustom" + matcher "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + configmapFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/configmap" + secretFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/secret" + + "k8s.io/utils/ptr" + + appFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/application" + osFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + certificatesv1alpha1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appv1alpha1 "github.com/argoproj/argo-cd/v3/pkg/apis/application/v1alpha1" + + argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1" + "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture" + argocdFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/argocd" + fixtureUtils "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/utils" +) + +var ( + // The differences between the upstream image using Ubuntu, and the downstream one using rhel. + image = "" // argocd-operator default + version = "" // argocd-operator default + caBundlePath = "/etc/ssl/certs/ca-certificates.crt" + + trustedHelmAppSource = &appv1alpha1.ApplicationSource{ + RepoURL: "https://stefanprodan.github.io/podinfo", + Chart: "podinfo", + TargetRevision: "6.5.3", + Helm: &appv1alpha1.ApplicationSourceHelm{Values: ""}, + } + + untrustedHelmAppSource = &appv1alpha1.ApplicationSource{ + RepoURL: "https://helm.nginx.com/stable", + Chart: "nginx", + TargetRevision: "1.1.0", + Helm: &appv1alpha1.ApplicationSourceHelm{Values: "service:\n type: ClusterIP"}, + } + + k8sClient client.Client + ctx context.Context + + clusterSupportsClusterTrustBundles bool +) + +var _ = Describe("GitOps Operator Sequential E2E Tests", func() { + + Context("1-120_repo_server_system_ca_trust", func() { + BeforeEach(func() { + fixture.EnsureSequentialCleanSlate() + + k8sClient, _ = fixtureUtils.GetE2ETestKubeClient() + ctx = context.Background() + + clusterSupportsClusterTrustBundles = detectClusterTrustBundleSupport(k8sClient, ctx) + + if fixture.EnvLocalRun() { + Skip("skipping test as LOCAL_RUN env is set.") + } + + if !fixture.EnvNonOLM() { + image = "registry.redhat.io/openshift-gitops-1/argocd-rhel8" + version = "sha256:8a0544c14823492165550d83a6d8ba79dd632b46144d3fdcb543793726111d76" + caBundlePath = "/etc/ssl/certs/ca-bundle.crt" + } + }) + + AfterEach(func() { + purgeCtbs() + }) + + It("ensures that missing Secret aborts startup", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with missing Secret") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + Secrets: []corev1.SecretProjection{ + {LocalObjectReference: corev1.LocalObjectReference{Name: "no-such-secret"}}, + }, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + + Eventually(argoCD, "1m", "5s").Should(argocdFixture.HaveServerStatus("Running")) + Consistently(argoCD, "20s", "5s").Should(argocdFixture.HaveRepoStatus("Pending")) + Expect(argoCD).ShouldNot(argocdFixture.BeAvailable()) + }) + + It("ensures that ClusterTrustBundles are trusted in repo-server and plugins", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + // Create a bundle with 2 CA certs in it. Ubuntu's update-ca-certificates issues a warning, but apparently it works + // It is desirable to test with multiple certs in one bundle because OpenShift permits it + combinedCtb := createCtbFromCerts(getCACert("github.com"), getCACert("github.io")) + _ = k8sClient.Delete(ctx, combinedCtb) // Exists only in case of previous failures + defer func() { _ = k8sClient.Delete(ctx, combinedCtb) }() + Expect(k8sClient.Create(ctx, combinedCtb)).To(Succeed()) + + pluginCm, pluginContainer, pluginVolumes := createGitPullingPlugin(ns) + Expect(k8sClient.Create(ctx, pluginCm)).To(Succeed()) + + By("creating Argo CD instance trusting CTBs") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // So we can test against upstream sites that would otherwise be trusted by the image + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{ + {Name: ptr.To(combinedCtb.Name), Path: "combined.crt"}, + {Name: ptr.To("nah"), Path: "no-such-ctb.crt", Optional: ptr.To(true)}, + }, + }, + // plugin containers/volumes - this is not related to CTBs + Volumes: pluginVolumes, + SidecarContainers: []corev1.Container{ + *pluginContainer, + }, + }) + + By("verifying correctly established system trust") + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + verifyCorrectlyConfiguredTrust(ns) + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(Equal(2), And( + ContainSubstring("combined.crt"), + ContainSubstring("no-such-ctb.crt"), + ))) + }) + + It("ensures that CMs and Secrets are trusted in repo-server and plugins", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + cmCert := createCmFromCert(ns, getCACert("github.com")) + Expect(k8sClient.Create(ctx, cmCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, cmCert) }() + secretCert := createSecretFromCert(ns, getCACert("github.io")) + Expect(k8sClient.Create(ctx, secretCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, secretCert) }() + + pluginCm, pluginContainer, pluginVolumes := createGitPullingPlugin(ns) + Expect(k8sClient.Create(ctx, pluginCm)).To(Succeed()) + + By("creating Argo CD instance trusting CTBs") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // So we can test against upstream sites that would otherwise be trusted by the image + Secrets: []corev1.SecretProjection{{ + // No Items, Map all + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretCert.Name, + }, + }}, + ConfigMaps: []corev1.ConfigMapProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmCert.Name, + }, + Optional: ptr.To(true), + Items: []corev1.KeyToPath{ + {Key: "ca.cm.crt", Path: "ca.cm.wrong-suffix"}, + }, + }}, + }, + // plugin containers/volumes - this is not related to Secret/CM + Volumes: pluginVolumes, + SidecarContainers: []corev1.Container{ + *pluginContainer, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + initContainerLog := getRepoCertGenerationLog(findRunningRepoServerPod(k8sClient, ns)) + Expect(initContainerLog).Should(ContainSubstring("ca.secret.crt")) + Expect(initContainerLog).Should(ContainSubstring("ca.cm.wrong-suffix.crt")) + verifyCorrectlyConfiguredTrust(ns) + }) + + It("ensures that 0 trusted certs with DropImageCertificates trusts nothing", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(Equal(0), Not(BeEmpty()))) + + trustedHelmApp := createHelmApp(ns, trustedHelmAppSource) + Expect(k8sClient.Create(ctx, trustedHelmApp)).To(Succeed()) + + // Sleep to make sure the apps sync took place - otherwise there might be no conditions _yet_ + time.Sleep(20 * time.Second) + + Expect(trustedHelmApp).Should(appFixture.HaveConditionMatching( + "ComparisonError", + ".*tls: failed to verify certificate: x509: certificate signed by unknown authority.*", + )) + Expect(trustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + }) + + It("ensures that empty trust keeps image certs in place", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: false, // Keep the image ones + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(BeNumerically(">", 100), Not(BeEmpty()))) + }) + + It("ensures that Secrets and ConfigMaps get reconciled", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust, but full of anticipation") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + Secrets: []corev1.SecretProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-trust", + }, + Optional: ptr.To(true), + }}, + ConfigMaps: []corev1.ConfigMapProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-trust", + }, + Optional: ptr.To(true), + }}, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + actualTrust := repoServerSystemCaTrust(ns) + Expect(actualTrust).Should(trustCerts(Equal(0), Not(BeEmpty()))) + + By("creating ConfigMap with 1 cert") + cmCert := createCmFromCert(ns, getCACert("github.com")) + Expect(k8sClient.Create(ctx, cmCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, cmCert) }() + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty()))) + + By("creating Secret with 1 cert") + secretCert := createSecretFromCert(ns, getCACert("github.io")) + Expect(k8sClient.Create(ctx, secretCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, secretCert) }() + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty()))) + + By("updating ConfigMap to 2 certs") + configmapFixture.Update(cmCert, func(configMap *corev1.ConfigMap) { + configMap.Data = map[string]string{ + "a.crt": getCACert("github.com"), + "b.crt": getCACert("google.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(3), Not(BeEmpty()))) + + By("updating Secret to 0 certs") + secretFixture.Update(secretCert, func(secret *corev1.Secret) { + // Albeit `.Data` is never written by the test, it is the field that holds the data after the Create/Get roundtrip. + // Erase, otherwise reducing the content of `.StringData` does not have the expected effect. + secret.Data = map[string][]byte{} + secret.StringData = map[string]string{} + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty()))) + + By("updating ConfigMap to 1 certs") + configmapFixture.Update(cmCert, func(configMap *corev1.ConfigMap) { + configMap.Data = map[string]string{ + "a.crt": getCACert("redhat.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty()))) + + By("deleting ConfigMap") + Expect(k8sClient.Delete(ctx, cmCert)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty()))) + }) + + It("ensures that ClusterTrustBundles get reconciled", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + combinedCtb := createCtbFromCerts(getCACert("github.com"), getCACert("github.io")) + _ = k8sClient.Delete(ctx, combinedCtb) // Exists only in case of previous failures, must be deleted before argo starts! + + By("creating Argo CD instance with empty system trust, but full of anticipation") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{{ + Name: ptr.To(combinedCtb.Name), Path: "ctb.crt", Optional: ptr.To(true), + }}, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + actualTrust := repoServerSystemCaTrust(ns) + Expect(actualTrust).Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + + By("creating ClusterTrustBundle with 2 certs") + defer func() { _ = k8sClient.Delete(ctx, combinedCtb) }() + Expect(k8sClient.Create(ctx, combinedCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty())), actualTrust.diagnose()) + + By("updating ClusterTrustBundle with 1 cert") + ctbUpdate(combinedCtb, func(bundle *certificatesv1alpha1.ClusterTrustBundle) { + bundle.Spec = certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: bundle.Spec.SignerName, + TrustBundle: getCACert("github.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "6m", "15s").Should(trustCerts(Equal(1), Not(BeEmpty())), actualTrust.diagnose()) + + By("deleting ClusterTrustBundle") + Expect(k8sClient.Delete(ctx, combinedCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "6m", "15s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + By("done") + }) + + It("detect only relevant ClusterTrustBundles changes", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + // Use random label value not to collide with leftover CTBs fom other tests + labelVal := rand.String(5) + signerName := "acme.com/signer" + By("creating Argo CD instance with system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + // Test CTB update detection based on CTB binding specified by labels - no real signers involved + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{ + { + SignerName: ptr.To(signerName), + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "test": labelVal, + }}, + Path: "one.crt", + Optional: ptr.To(true), + }, + }, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + By("adding ClusterTrustBundle with 1 cert") + oneCtb := createCtbFromCerts(getCACert("github.com")) + oneCtb.Labels["test"] = labelVal + oneCtb.Name = "acme.com:signer:repo-server-system-ca-trust-test-one" + oneCtb.Spec.SignerName = signerName + Expect(k8sClient.Create(ctx, oneCtb)).To(Succeed()) + actualTrust := repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty())), actualTrust.diagnose()) + + By("adding ClusterTrustBundle with other cert") + twoCtb := createCtbFromCerts(getCACert("github.io")) + twoCtb.Labels["test"] = labelVal + twoCtb.Name = "acme.com:signer:repo-server-system-ca-trust-test-two" + twoCtb.Spec.SignerName = signerName + Expect(k8sClient.Create(ctx, twoCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty())), actualTrust.diagnose()) + + By("updating Argo CD to read from ClusterTrustBundle that does not exist") + argocdFixture.Update(argoCD, func(cd *argov1beta1api.ArgoCD) { + cd.Spec.Repo.SystemCATrust.ClusterTrustBundles = []corev1.ClusterTrustBundleProjection{ + { + Name: ptr.To("no-such-ctb"), + Path: "three.crt", + Optional: ptr.To(true), + }, + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Consistently(actualTrust, "10s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + oldPodName := findRunningRepoServerPod(k8sClient, ns).Name + + By("creating unrelated ClusterTrustBundle") + fourCtb := createCtbFromCerts(getCACert("google.com")) + Expect(k8sClient.Create(ctx, fourCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Consistently(actualTrust, "10s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + newPodName := findRunningRepoServerPod(k8sClient, ns).Name + Expect(newPodName).To(Equal(oldPodName), "Pod have restarted for unrelated change") + }) + }) +}) + +func ctbUpdate(obj *certificatesv1alpha1.ClusterTrustBundle, modify func(*certificatesv1alpha1.ClusterTrustBundle)) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Retrieve the latest version of the object + err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(obj), obj) + if err != nil { + return err + } + + modify(obj) + + // Attempt to update the object + return k8sClient.Update(context.Background(), obj) + }) + Expect(err).ToNot(HaveOccurred()) +} + +func argoCDSpec(ns *corev1.Namespace, repoSpec argov1beta1api.ArgoCDRepoSpec) *argov1beta1api.ArgoCD { + return &argov1beta1api.ArgoCD{ + ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: ns.Name}, + Spec: argov1beta1api.ArgoCDSpec{ + Image: image, + Version: version, + Repo: repoSpec, + }, + } +} + +func detectClusterTrustBundleSupport(k8sClient client.Client, ctx context.Context) bool { + err := k8sClient.List(ctx, &certificatesv1alpha1.ClusterTrustBundleList{}) + if _, ok := err.(*apiutil.ErrResourceDiscoveryFailed); ok { + return false + } + Expect(err).ToNot(HaveOccurred()) // Every other error is an error + return true +} + +func createGitPullingPlugin(ns *corev1.Namespace) (*corev1.ConfigMap, *corev1.Container, []corev1.Volume) { + By("Creating ConfigManagementPlugin resources for git clone") + name := "cmp-git-https" + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + Labels: map[string]string{ + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "plugin.yaml": `apiVersion: argoproj.io/v1alpha1 +kind: ConfigManagementPlugin +metadata: + name: git-https +spec: + version: v1.0 + generate: + command: [bash, -c] + args: + - | + set -euxo pipefail + git clone --depth 1 --verbose "$ARGOCD_APP_SOURCE_REPO_URL" +`, + }, + } + + container := &corev1.Container{ + Name: name, + Command: []string{"/var/run/argocd/argocd-cmp-server"}, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/var/run/argocd", + Name: "var-files", + }, + { + MountPath: "/home/argocd/cmp-server/plugins", + Name: "plugins", + }, + { + MountPath: "/home/argocd/cmp-server/config/plugin.yaml", + SubPath: "plugin.yaml", + Name: name + "-config", + }, + { + MountPath: "/tmp", + Name: name + "-tmp", + }, + }, + } + + volumes := []corev1.Volume{ + { + Name: name + "-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + }, + }, + }, + { + Name: name + "-tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + return cm, container, volumes +} + +func createHelmApp(ns *corev1.Namespace, source *appv1alpha1.ApplicationSource) *appv1alpha1.Application { + By("creating helm Application " + source.Chart) + + return &appv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: source.Chart, + Namespace: ns.Name, + }, + Spec: appv1alpha1.ApplicationSpec{ + Project: "default", + Source: source, + Destination: appv1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: ns.Name, + }, + SyncPolicy: &appv1alpha1.SyncPolicy{ + Automated: &appv1alpha1.SyncPolicyAutomated{ + Prune: true, SelfHeal: true, + }, + }, + }, + } +} + +func createPluginApp(ns *corev1.Namespace, url string) *appv1alpha1.Application { + name := regexp.MustCompile("[^a-z]+").ReplaceAllString(url, "-") + By("creating plugin Application " + name) + return &appv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: appv1alpha1.ApplicationSpec{ + Project: "default", + Destination: appv1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: ns.Name, + }, + Source: &appv1alpha1.ApplicationSource{ + RepoURL: url, + TargetRevision: "HEAD", + Path: ".", + Plugin: &appv1alpha1.ApplicationSourcePlugin{ + Name: "git-https-v1.0", + Env: appv1alpha1.Env{ + &appv1alpha1.EnvEntry{ + Name: "ARGOCD_APP_SOURCE_REPO_URL", + Value: url, + }, + }, + }, + }, + }, + } +} + +func createCtbFromCerts(bundle ...string) *certificatesv1alpha1.ClusterTrustBundle { + return &certificatesv1alpha1.ClusterTrustBundle{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterTrustBundle", + APIVersion: "certificates.k8s.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-server-system-ca-trust", + Labels: map[string]string{ + "argocd-operator-test": "repo_server_system_ca_trust", + }, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: strings.Join(bundle, "\n"), + }, + } +} + +func createCmFromCert(ns *corev1.Namespace, bundle string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-trust", + Namespace: ns.Name, + }, + Data: map[string]string{ + "ca.cm.crt": bundle, + }, + } +} + +func createSecretFromCert(ns *corev1.Namespace, bundle string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-trust", + Namespace: ns.Name, + }, + Type: "Opaque", + StringData: map[string]string{ + "ca.secret.crt": bundle, + }, + } +} + +func getCACert(host string) string { + config := &tls.Config{MinVersion: tls.VersionTLS13} + conn, err := tls.Dial("tcp", host+":443", config) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = conn.Close() }() + + pcs := conn.ConnectionState().PeerCertificates + + // ClusterTrustBundle cannot hold leaf certificates, so testing with CA cert at least. In theory, some of the hosts + // we test against can share the same CA cert, so albeit not likely, rudimentary negative testing is needed. + return encodeCert(pcs[len(pcs)-1]) +} + +func encodeCert(cert *x509.Certificate) string { + writer := strings.Builder{} + err := pem.Encode(&writer, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + Expect(err).ToNot(HaveOccurred()) + + return writer.String() +} + +type podTrust struct { + ns *corev1.Namespace + k8sClient client.Client + + count int + log string + events string +} + +func (pt *podTrust) fetch() { + pod := findRunningRepoServerPod(pt.k8sClient, pt.ns) + pt.count = getTrustedCertCount(pod) + pt.log = getRepoCertGenerationLog(pod) + + out, err := osFixture.ExecCommandWithOutputParam(false, "kubectl", "-n", pt.ns.Name, "events") + if err != nil { + panic(err) + } + pt.events = out +} + +func (pt *podTrust) diagnose() string { + return fmt.Sprintf( + "System CA Trust init contianer log:\n%s\nProject events:\n%s\n", + pt.log, pt.events, + ) +} + +func repoServerSystemCaTrust(ns *corev1.Namespace) *podTrust { + return &podTrust{ns: ns, k8sClient: k8sClient} +} + +func trustCerts(countMatcher, logMatcher matcher.GomegaMatcher) matcher.GomegaMatcher { + // Wrap to capture and attach diagnostics + matchCount := gcustom.MakeMatcher(func(pt *podTrust) (bool, error) { + // call fetch exactly once before matchCount _and_ matchLog + pt.fetch() + + success, err := countMatcher.Match(pt.count) + if err != nil { + return false, err + } + if success { + return true, nil + } + + return false, fmt.Errorf( + "%s\n\n--- Diagnostics ---\n%s\n===", + countMatcher.FailureMessage(pt.count), + pt.diagnose(), + ) + }) + + matchLog := WithTransform(func(pt *podTrust) string { + return pt.log + }, logMatcher) + + return And(matchCount, matchLog) +} + +func getTrustedCertCount(rsPod *corev1.Pod) int { + var out string + var err error + // retry a few times, because pod can be restarting during trust source update, get Terminating between check and use. + for i := 0; i < 3; i++ { + command := []string{ + "kubectl", "-n", rsPod.Namespace, "exec", + "-c", "argocd-repo-server", rsPod.Name, "--", + "cat", caBundlePath, + } + + out, err = osFixture.ExecCommandWithOutputParam(false, command...) + if err == nil { + break + } + time.Sleep(1 * time.Second) + } + Expect(err).ToNot(HaveOccurred(), out) + + seen := make(map[string]bool) + var currentBlock strings.Builder + for line := range strings.Lines(out) { + switch { + case strings.Contains(line, "BEGIN CERTIFICATE"): + currentBlock.Reset() + case strings.Contains(line, "END CERTIFICATE"): + seen[currentBlock.String()] = true + default: + currentBlock.WriteString(line) + } + } + return len(seen) +} + +func getRepoCertGenerationLog(rsPod *corev1.Pod) string { + out, err := osFixture.ExecCommandWithOutputParam( + false, + "kubectl", "-n", rsPod.Namespace, "logs", "-c", "update-ca-certificates", rsPod.Name, + ) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("output: %s", out)) + return out +} + +func findRunningRepoServerPod(k8sClient client.Client, ns *corev1.Namespace) *corev1.Pod { + nameRegexp := regexp.MustCompile(".*-repo-server.*") + var pods []*corev1.Pod + + for j := 0; j < 10; j++ { + time.Sleep(2 * time.Second) + pods = []*corev1.Pod{} + list := &corev1.PodList{} + err := k8sClient.List(context.Background(), list, client.InNamespace(ns.Name)) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range list.Items { + if pod.Status.Phase == "Running" && nameRegexp.MatchString(pod.Name) { + pods = append(pods, &pod) + } + } + if len(pods) == 1 { + return pods[0] + } + } + + panic(fmt.Sprintf("Failed to find Running repo-server pod. have %+v", pods)) +} + +func verifyCorrectlyConfiguredTrust(ns *corev1.Namespace) { + untrustedHelmApp := createHelmApp(ns, untrustedHelmAppSource) + Expect(k8sClient.Create(ctx, untrustedHelmApp)).To(Succeed()) + + // Using some host not trusted by github's intermediate cert. Gitlab-somewhat surprisingly-is. + untrustedPluginApp := createPluginApp(ns, "https://kernel.googlesource.com/pub/scm/docs/man-pages/website.git") + Expect(k8sClient.Create(ctx, untrustedPluginApp)).To(Succeed()) + + trustedHelmApp := createHelmApp(ns, trustedHelmAppSource) + Expect(k8sClient.Create(ctx, trustedHelmApp)).To(Succeed()) + + trustedPluginApp := createPluginApp(ns, "https://github.com/argoproj-labs/argocd-operator.git") + Expect(k8sClient.Create(ctx, trustedPluginApp)).To(Succeed()) + + // Sleep to make sure the apps sync took place - otherwise there might be no conditions _yet_ + time.Sleep(20 * time.Second) + + Expect(untrustedHelmApp).Should( + appFixture.HaveConditionMatching("ComparisonError", ".*failed to fetch chart.*"), + ) + Expect(untrustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + + Expect(untrustedPluginApp).Should( + appFixture.HaveConditionMatching("ComparisonError", ".*certificate signed by unknown authority.*"), + ) + Expect(untrustedPluginApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + + Expect(trustedHelmApp).Should(appFixture.HaveNoConditions()) + Expect(trustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeSynced)) + + Expect(trustedPluginApp).Should(appFixture.HaveNoConditions()) + Expect(trustedPluginApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeSynced)) +} + +// purgeCtbs deletes all of the cluster-wide resource, that can get leaked on test failure/abort. +func purgeCtbs() { + if clusterSupportsClusterTrustBundles { + expr := client.MatchingLabels{"argocd-operator-test": "repo_server_system_ca_trust"} + Expect(k8sClient.DeleteAllOf(ctx, &certificatesv1alpha1.ClusterTrustBundle{}, expr)).To(Succeed()) + } +}