Add jitter to Default token expiration (#260)

* Adding jitter based on percent to default token expiration to avoid thundering herd

* Adding mockery configs and math/rand mocks

* Adding Jitter and updating tests cases for token expiration jitter

* Remove jitter from ISRAv1 code path

* Fixing test data with randomized token expiration

Author: notSoWiseOldMan

.mockery.yaml

@@ -0,0 +1,10 @@
+quiet: False
+disable-version-string: True
+with-expecter: True
+mockname: "{{.InterfaceName}}"
+filename: "{{.MockName}}.go"
+outpkg: pkg/mocks
+packages:
+  math/rand:
+    interfaces:
+      Source64:

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/net v0.30.0 // indirect

go.sum

@@ -261,6 +261,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/x509/pkix"
 	goflag "flag"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
@@ -46,6 +47,7 @@ import (
 )
 
 var webhookVersion = "v0.1.0"
+var random = rand.New(rand.NewSource(time.Now().UnixNano()))
 
 func main() {
 	port := flag.Int("port", 443, "Port to listen on")
@@ -208,6 +210,7 @@ func main() {
 	}
 
 	mod := handler.NewModifier(
+		random,
 		handler.WithAnnotationDomain(*annotationPrefix),
 		handler.WithMountPath(*mountPath),
 		handler.WithServiceAccountCache(saCache),

pkg/constants.go

@@ -15,9 +15,12 @@ permissions and limitations under the License.
 package pkg
 
 const (
-	// Default token expiration in seconds if none is defined,
-	// which is 24hrs as that is max for EKS
+	// 24hrs as that is max for EKS
+	MaxTokenExpiration = int64(86400)
+	// Default token expiration in seconds if none is defined, 22hrs
 	DefaultTokenExpiration = int64(86400)
+	// Used for the minimum jitter value when using the default token expiration
+	DefaultMinTokenExpiration = int64(79200)
 	// 10mins is min for kube-apiserver
 	MinTokenExpiration = int64(600)
 

pkg/handler/handler.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -85,12 +86,17 @@ func WithSALookupGraceTime(saLookupGraceTime time.Duration) ModifierOpt {
 }
 
 // NewModifier returns a Modifier with default values
-func NewModifier(opts ...ModifierOpt) *Modifier {
+func NewModifier(random *rand.Rand, opts ...ModifierOpt) *Modifier {
+	if random == nil {
+		random = rand.New(rand.NewSource(time.Now().UnixNano()))
+	}
+
 	mod := &Modifier{
 		AnnotationDomain: "eks.amazonaws.com",
 		MountPath:        "/var/run/secrets/eks.amazonaws.com/serviceaccount",
 		volName:          "aws-iam-token",
 		tokenName:        "token",
+		rand:             *random,
 	}
 	for _, opt := range opts {
 		opt(mod)
@@ -109,6 +115,7 @@ type Modifier struct {
 	volName                    string
 	tokenName                  string
 	saLookupGraceTime          time.Duration
+	rand                       rand.Rand
 }
 
 type patchOperation struct {
@@ -417,6 +424,7 @@ func (m *Modifier) buildPodPatchConfig(pod *corev1.Pod) *podPatchConfig {
 		regionalSTS, tokenExpiration := m.Cache.GetCommonConfigurations(pod.Spec.ServiceAccountName, pod.Namespace)
 		tokenExpiration, containersToSkip := m.parsePodAnnotations(pod, tokenExpiration)
 
+		tokenExpiration = m.addJitterToDefaultToken(tokenExpiration)
 		webhookPodCount.WithLabelValues("container_credentials").Inc()
 
 		return &podPatchConfig{
@@ -479,6 +487,15 @@ func (m *Modifier) buildPodPatchConfig(pod *corev1.Pod) *podPatchConfig {
 	return nil
 }
 
+func (m *Modifier) addJitterToDefaultToken(tokenExpiration int64) int64 {
+	if tokenExpiration == pkg.DefaultTokenExpiration {
+		klog.V(0).Infof("Adding jitter to default token expiration")
+		tokenExpiration = m.rand.Int63n(pkg.DefaultTokenExpiration-pkg.DefaultMinTokenExpiration+int64(1)) + pkg.DefaultMinTokenExpiration
+	}
+
+	return tokenExpiration
+}
+
 // MutatePod takes a AdmissionReview, mutates the pod, and returns an AdmissionResponse
 func (m *Modifier) MutatePod(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 	badRequest := &v1beta1.AdmissionResponse{

pkg/handler/handler_pod_test.go

@@ -62,7 +62,7 @@ const (
 // buildModifierFromPod gets values to set up test case environments with as if
 // the values were set by service account annotation/flag before the test case.
 // Test cases are defined entirely by pod yamls.
-func buildModifierFromPod(pod *corev1.Pod) *Modifier {
+func buildModifierFromPod(pod *corev1.Pod, t *testing.T) *Modifier {
 	var modifierOpts []ModifierOpt
 
 	if path, ok := pod.Annotations[handlerMountPathAnnotation]; ok {
@@ -76,7 +76,7 @@ func buildModifierFromPod(pod *corev1.Pod) *Modifier {
 	modifierOpts = append(modifierOpts, WithServiceAccountCache(buildFakeCacheFromPod(pod)))
 	modifierOpts = append(modifierOpts, WithContainerCredentialsConfig(buildFakeConfigFromPod(pod)))
 
-	return NewModifier(modifierOpts...)
+	return NewModifier(getAlwaysZeroRand(t), modifierOpts...)
 }
 
 func buildFakeCacheFromPod(pod *corev1.Pod) *cache.FakeServiceAccountCache {
@@ -157,7 +157,7 @@ func TestUpdatePodSpec(t *testing.T) {
 			pod.Spec.ServiceAccountName = "default"
 
 			t.Run(fmt.Sprintf("Pod %s in file %s", pod.Name, path), func(t *testing.T) {
-				modifier := buildModifierFromPod(pod)
+				modifier := buildModifierFromPod(pod, t)
 				patchConfig := modifier.buildPodPatchConfig(pod)
 				patch, _ := modifier.getPodSpecPatch(pod, patchConfig)
 				patchBytes, err := json.Marshal(patch)

pkg/handler/handler_test.go

@@ -19,10 +19,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"github.com/aws/amazon-eks-pod-identity-webhook/pkg/containercredentials"
+	mocks "github.com/aws/amazon-eks-pod-identity-webhook/pkg/mocks/math/rand"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"io"
 	"io/ioutil"
 	"k8s.io/apimachinery/pkg/types"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -49,9 +52,11 @@ func TestMutatePod(t *testing.T) {
 	}
 
 	modifier := NewModifier(
+		getAlwaysZeroRand(t),
 		WithServiceAccountCache(cache.NewFakeServiceAccountCache(testServiceAccount)),
 		WithContainerCredentialsConfig(&containercredentials.FakeConfig{}),
 	)
+
 	cases := []struct {
 		caseName string
 		input    *v1beta1.AdmissionReview
@@ -105,6 +110,7 @@ func TestMutatePod(t *testing.T) {
 
 func TestMutatePod_MutationNotNeeded(t *testing.T) {
 	modifier := NewModifier(
+		getAlwaysZeroRand(t),
 		WithServiceAccountCache(cache.NewFakeServiceAccountCache()),
 		WithContainerCredentialsConfig(&containercredentials.FakeConfig{}),
 	)
@@ -180,6 +186,17 @@ func serializeAdmissionReview(t *testing.T, want *v1beta1.AdmissionReview) []byt
 	return wantedBytes
 }
 
+func getAlwaysZeroRand(t *testing.T) *rand.Rand {
+	// Mock random and always return 0
+	mockRandomSource := mocks.NewSource64(t)
+	mockRandomSource.On("Int63", mock.Anything).Return(int64(0))
+
+	mockRand := rand.New(mockRandomSource)
+	mockRand.Int63()
+
+	return mockRand
+}
+
 func TestModifierHandler(t *testing.T) {
 	testServiceAccount := &corev1.ServiceAccount{}
 	testServiceAccount.Name = "default"
@@ -190,6 +207,7 @@ func TestModifierHandler(t *testing.T) {
 	}
 
 	modifier := NewModifier(
+		getAlwaysZeroRand(t),
 		WithServiceAccountCache(cache.NewFakeServiceAccountCache(testServiceAccount)),
 		WithContainerCredentialsConfig(&containercredentials.FakeConfig{}),
 	)

pkg/handler/testdata/containercredentials/rawPodSkip.pod.yaml

@@ -9,7 +9,7 @@ metadata:
     testing.eks.amazonaws.com/containercredentials/mountPath: "/con-creds-mount-path"
     testing.eks.amazonaws.com/containercredentials/volumeName: "con-creds-volume-name"
     testing.eks.amazonaws.com/containercredentials/tokenPath: "con-creds-token-path"
-    testing.eks.amazonaws.com/expectedPatch: '[{"op":"add","path":"/spec/volumes","value":[{"name":"con-creds-volume-name","projected":{"sources":[{"serviceAccountToken":{"audience":"con-creds-aud","expirationSeconds":86400,"path":"con-creds-token-path"}}]}}]},{"op":"add","path":"/spec/containers","value":[{"name":"sidecar","image":"amazonlinux","resources":{}},{"name":"balajilovesoreos","image":"amazonlinux","env":[{"name":"AWS_CONTAINER_CREDENTIALS_FULL_URI","value":"con-creds-uri"},{"name":"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE","value":"/con-creds-mount-path/con-creds-token-path"}],"resources":{},"volumeMounts":[{"name":"con-creds-volume-name","readOnly":true,"mountPath":"/con-creds-mount-path"}]}]}]'
+    testing.eks.amazonaws.com/expectedPatch: '[{"op":"add","path":"/spec/volumes","value":[{"name":"con-creds-volume-name","projected":{"sources":[{"serviceAccountToken":{"audience":"con-creds-aud","expirationSeconds":79200,"path":"con-creds-token-path"}}]}}]},{"op":"add","path":"/spec/containers","value":[{"name":"sidecar","image":"amazonlinux","resources":{}},{"name":"balajilovesoreos","image":"amazonlinux","env":[{"name":"AWS_CONTAINER_CREDENTIALS_FULL_URI","value":"con-creds-uri"},{"name":"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE","value":"/con-creds-mount-path/con-creds-token-path"}],"resources":{},"volumeMounts":[{"name":"con-creds-volume-name","readOnly":true,"mountPath":"/con-creds-mount-path"}]}]}]'
     # Pod Annotation
     eks.amazonaws.com/skip-containers: "sidecar"
 spec:

pkg/handler/testdata/containercredentials/rawPodWithVolume.pod.yaml

@@ -10,7 +10,7 @@ metadata:
     testing.eks.amazonaws.com/containercredentials/mountPath: "/con-creds-mount-path"
     testing.eks.amazonaws.com/containercredentials/volumeName: "con-creds-volume-name"
     testing.eks.amazonaws.com/containercredentials/tokenPath: "con-creds-token-path"
-    testing.eks.amazonaws.com/expectedPatch: '[{"op":"add","path":"/spec/volumes/0","value":{"name":"con-creds-volume-name","projected":{"sources":[{"serviceAccountToken":{"audience":"con-creds-aud","expirationSeconds":86400,"path":"con-creds-token-path"}}]}}},{"op":"add","path":"/spec/containers","value":[{"name":"balajilovesoreos","image":"amazonlinux","env":[{"name":"AWS_CONTAINER_CREDENTIALS_FULL_URI","value":"con-creds-uri"},{"name":"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE","value":"/con-creds-mount-path/con-creds-token-path"}],"resources":{},"volumeMounts":[{"name":"con-creds-volume-name","readOnly":true,"mountPath":"/con-creds-mount-path"}]}]}]'
+    testing.eks.amazonaws.com/expectedPatch: '[{"op":"add","path":"/spec/volumes/0","value":{"name":"con-creds-volume-name","projected":{"sources":[{"serviceAccountToken":{"audience":"con-creds-aud","expirationSeconds":79200,"path":"con-creds-token-path"}}]}}},{"op":"add","path":"/spec/containers","value":[{"name":"balajilovesoreos","image":"amazonlinux","env":[{"name":"AWS_CONTAINER_CREDENTIALS_FULL_URI","value":"con-creds-uri"},{"name":"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE","value":"/con-creds-mount-path/con-creds-token-path"}],"resources":{},"volumeMounts":[{"name":"con-creds-volume-name","readOnly":true,"mountPath":"/con-creds-mount-path"}]}]}]'
 spec:
   containers:
   - image: amazonlinux