Enable encryption in the cache proxy by experiment

Author: iain-macdonald

enterprise/server/action_cache_server_proxy/BUILD

@@ -8,6 +8,7 @@ go_library(
     importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/action_cache_server_proxy",
     visibility = ["//visibility:public"],
     deps = [
+        "//enterprise/server/remote_crypter",
         "//enterprise/server/util/proxy_util",
         "//proto:remote_execution_go_proto",
         "//proto:resource_go_proto",

enterprise/server/action_cache_server_proxy/action_cache_server_proxy.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_crypter"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/proxy_util"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
@@ -30,7 +31,7 @@ var (
 )
 
 type ActionCacheServerProxy struct {
-	supportsEncryption bool
+	supportsEncryption func(context.Context) bool
 	env                environment.Env
 	authenticator      interfaces.Authenticator
 	localCache         interfaces.Cache
@@ -52,8 +53,14 @@ func NewActionCacheServerProxy(env environment.Env) (*ActionCacheServerProxy, er
 	if remoteCache == nil {
 		return nil, fmt.Errorf("An ActionCacheClient is required to enable the ActionCacheServerProxy")
 	}
+	supportsEncryption := func(ctx context.Context) bool {
+		if env.GetCrypter() == nil {
+			return false
+		}
+		return remote_crypter.Enabled(ctx, env.GetExperimentFlagProvider())
+	}
 	return &ActionCacheServerProxy{
-		supportsEncryption: env.GetCrypter() != nil,
+		supportsEncryption: supportsEncryption,
 		env:                env,
 		authenticator:      env.GetAuthenticator(),
 		localCache:         env.GetCache(),
@@ -134,7 +141,7 @@ func (s *ActionCacheServerProxy) cacheActionResultToLocalCAS(ctx context.Context
 // request to the authoritative cache, but send a hash of the last value we
 // received to avoid transferring data on unmodified actions.
 func (s *ActionCacheServerProxy) GetActionResult(ctx context.Context, req *repb.GetActionResultRequest) (*repb.ActionResult, error) {
-	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption {
+	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption(ctx) {
 		resp, err := s.remoteACClient.GetActionResult(ctx, req)
 		labels := prometheus.Labels{
 			metrics.StatusLabel:           fmt.Sprintf("%d", gstatus.Code(err)),
@@ -222,7 +229,7 @@ func (s *ActionCacheServerProxy) GetActionResult(ctx context.Context, req *repb.
 
 func (s *ActionCacheServerProxy) UpdateActionResult(ctx context.Context, req *repb.UpdateActionResultRequest) (*repb.ActionResult, error) {
 	// Only if it's explicitly requested do we cache AC results locally.
-	if proxy_util.SkipRemote(ctx) && (!authutil.EncryptionEnabled(ctx, s.authenticator) || s.supportsEncryption) {
+	if proxy_util.SkipRemote(ctx) && (!authutil.EncryptionEnabled(ctx, s.authenticator) || s.supportsEncryption(ctx)) {
 		resp, err := s.localACServer.UpdateActionResult(ctx, req)
 
 		labels := prometheus.Labels{

enterprise/server/byte_stream_server_proxy/BUILD

@@ -8,6 +8,7 @@ go_library(
     importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/byte_stream_server_proxy",
     visibility = ["//visibility:public"],
     deps = [
+        "//enterprise/server/remote_crypter",
         "//enterprise/server/util/proxy_util",
         "//server/environment",
         "//server/interfaces",

enterprise/server/byte_stream_server_proxy/byte_stream_server_proxy.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"strconv"
 
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_crypter"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/proxy_util"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
@@ -23,7 +24,7 @@ import (
 )
 
 type ByteStreamServerProxy struct {
-	supportsEncryption bool
+	supportsEncryption func(context.Context) bool
 	atimeUpdater       interfaces.AtimeUpdater
 	authenticator      interfaces.Authenticator
 	local              interfaces.ByteStreamServer
@@ -56,8 +57,14 @@ func New(env environment.Env) (*ByteStreamServerProxy, error) {
 	if local == nil {
 		return nil, fmt.Errorf("A local ByteStreamServer is required to enable ByteStreamServerProxy")
 	}
+	supportsEncryption := func(ctx context.Context) bool {
+		if env.GetCrypter() == nil {
+			return false
+		}
+		return remote_crypter.Enabled(ctx, env.GetExperimentFlagProvider())
+	}
 	return &ByteStreamServerProxy{
-		supportsEncryption: env.GetCrypter() != nil,
+		supportsEncryption: supportsEncryption,
 		atimeUpdater:       atimeUpdater,
 		authenticator:      authenticator,
 		local:              local,
@@ -92,7 +99,7 @@ func (s *ByteStreamServerProxy) Read(req *bspb.ReadRequest, stream bspb.ByteStre
 }
 
 func (s *ByteStreamServerProxy) read(ctx context.Context, req *bspb.ReadRequest, stream *meteredReadServerStream) (string, error) {
-	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption {
+	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption(ctx) {
 		return metrics.UncacheableStatusLabel, s.readRemoteOnly(ctx, req, stream)
 	}
 
@@ -289,7 +296,7 @@ func (s *ByteStreamServerProxy) Write(stream bspb.ByteStream_WriteServer) error
 	meteredStream := &meteredServerSideClientStream{ByteStream_WriteServer: stream}
 	stream = meteredStream
 	var err error
-	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption {
+	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption(ctx) {
 		err = s.writeRemoteOnly(ctx, stream)
 	} else if proxy_util.SkipRemote(ctx) {
 		err = s.writeLocalOnly(stream)

enterprise/server/content_addressable_storage_server_proxy/BUILD

@@ -8,6 +8,7 @@ go_library(
     importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/content_addressable_storage_server_proxy",
     visibility = ["//visibility:public"],
     deps = [
+        "//enterprise/server/remote_crypter",
         "//enterprise/server/util/proxy_util",
         "//proto:remote_execution_go_proto",
         "//server/environment",

enterprise/server/content_addressable_storage_server_proxy/content_addressable_storage_server_proxy.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"strconv"
 
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_crypter"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/proxy_util"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
@@ -29,7 +30,7 @@ import (
 var enableGetTreeCaching = flag.Bool("cache_proxy.enable_get_tree_caching", false, "If true, the Cache Proxy attempts to serve GetTree requests out of the local cache. If false, GetTree requests are always proxied to the remote, authoritative cache.")
 
 type CASServerProxy struct {
-	supportsEncryption bool
+	supportsEncryption func(context.Context) bool
 	atimeUpdater       interfaces.AtimeUpdater
 	authenticator      interfaces.Authenticator
 	local              repb.ContentAddressableStorageServer
@@ -62,8 +63,14 @@ func New(env environment.Env) (*CASServerProxy, error) {
 	if remote == nil {
 		return nil, fmt.Errorf("A remote ContentAddressableStorageClient is required to enable the ContentAddressableStorageServerProxy")
 	}
+	supportsEncryption := func(ctx context.Context) bool {
+		if env.GetCrypter() == nil {
+			return false
+		}
+		return remote_crypter.Enabled(ctx, env.GetExperimentFlagProvider())
+	}
 	proxy := CASServerProxy{
-		supportsEncryption: env.GetCrypter() != nil,
+		supportsEncryption: supportsEncryption,
 		atimeUpdater:       atimeUpdater,
 		authenticator:      authenticator,
 		local:              local,
@@ -123,7 +130,7 @@ func (s *CASServerProxy) BatchUpdateBlobs(ctx context.Context, req *repb.BatchUp
 		map[string]int{metrics.MissStatusLabel: bytesInRequest(req)},
 	)
 
-	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption {
+	if authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption(ctx) {
 		return s.remote.BatchUpdateBlobs(ctx, req)
 	}
 
@@ -172,7 +179,7 @@ func (s *CASServerProxy) BatchReadBlobs(ctx context.Context, req *repb.BatchRead
 	mergedResp := repb.BatchReadBlobsResponse{}
 	mergedDigests := []*repb.Digest{}
 	localResp := &repb.BatchReadBlobsResponse{}
-	remoteOnly := authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption
+	remoteOnly := authutil.EncryptionEnabled(ctx, s.authenticator) && !s.supportsEncryption(ctx)
 	if !remoteOnly {
 		resp, err := s.local.BatchReadBlobs(ctx, req)
 		if err != nil {
@@ -290,7 +297,7 @@ func (s *CASServerProxy) batchReadBlobsRemote(ctx context.Context, readReq *repb
 			Compressor: response.Compressor,
 		})
 	}
-	if !authutil.EncryptionEnabled(ctx, s.authenticator) || s.supportsEncryption {
+	if !authutil.EncryptionEnabled(ctx, s.authenticator) || s.supportsEncryption(ctx) {
 		if _, err := s.local.BatchUpdateBlobs(ctx, &updateReq); err != nil {
 			log.CtxWarningf(ctx, "Error locally updating blobs: %s", err)
 		}

enterprise/server/remote_crypter/remote_crypter.go

@@ -21,6 +21,10 @@ import (
 	sgpb "github.com/buildbuddy-io/buildbuddy/proto/storage"
 )
 
+const (
+	remoteEncryptionEnabled = "crypter.remote_encryption_enabled"
+)
+
 var (
 	target = flag.String("crypter.remote_target", "", "The gRPC target of the remote encryption API.")
 )
@@ -32,6 +36,10 @@ type RemoteCrypter struct {
 	clientIdentityService interfaces.ClientIdentityService
 }
 
+func Enabled(ctx context.Context, experiments interfaces.ExperimentFlagProvider) bool {
+	return experiments.Boolean(ctx, remoteEncryptionEnabled, false)
+}
+
 func Register(env *real_environment.RealEnv) error {
 	if *target == "" {
 		return nil