feat(storage): add GrpcMetricsExcludedLabelsOption for gRPC metrics label filtering (#15735)

Author: shubham-up-47

google/cloud/opentelemetry/internal/monitoring_exporter.cc

@@ -31,6 +31,41 @@ std::string FormatProjectFullName(std::string const& project) {
   return absl::StrCat("projects/", project);
 }
 
+otel_internal::ResourceFilterDataFn MakeResourceFilterFn(
+    Options const& options) {
+  if (!options.has<otel_internal::ResourceFilterDataFnOption>()) {
+    return nullptr;
+  }
+
+  // Get the metric labels set to be excluded.
+  auto const& excluded =
+      options.get<otel_internal::ResourceFilterDataFnOption>();
+  if (excluded.empty()) return nullptr;
+
+  // Capture by value to avoid dangling reference in the lambda.
+  return [excluded = std::move(excluded)](std::string const& key) -> bool {
+    return excluded.count(key) > 0;
+  };
+}
+
+otel_internal::MonitoredResourceFromDataFn MakeDynamicResourceFn(
+    Options const& options, absl::optional<Project> const& project,
+    absl::optional<google::api::MonitoredResource> const& mr_proto) {
+  if (!options.has<otel_internal::ResourceFilterDataFnOption>()) {
+    return nullptr;
+  }
+
+  // `resource_filter_fn_` and `dynamic_resource_fn_` are meant to be used as a
+  // pair. Here we have a filter but no dynamic function, create a default one
+  // that returns the same project and monitored resource for all data points.
+  auto project_id = project->project_id();
+  auto monitored_resource = mr_proto.value_or(google::api::MonitoredResource{});
+  return [project_id, monitored_resource](
+             opentelemetry::sdk::metrics::PointDataAttributes const&) {
+    return std::make_pair(project_id, monitored_resource);
+  };
+}
+
 }  // namespace
 
 MonitoringExporter::MonitoringExporter(
@@ -51,6 +86,8 @@ MonitoringExporter::MonitoringExporter(
     Options const& options)
     : MonitoringExporter(std::move(conn), nullptr, nullptr, options) {
   project_ = std::move(project);
+  resource_filter_fn_ = MakeResourceFilterFn(options);
+  dynamic_resource_fn_ = MakeDynamicResourceFn(options, project_, mr_proto_);
 }
 
 opentelemetry::sdk::common::ExportResult MonitoringExporter::Export(

google/cloud/opentelemetry/internal/monitoring_exporter.h

@@ -45,6 +45,12 @@ using MonitoredResourceFromDataFn =
 // of the google::api::Metric proto.
 using ResourceFilterDataFn = std::function<bool(std::string const&)>;
 
+// Filter resource labels. A set of OpenTelemetry resource attribute keys to
+// exclude from metric labels when exporting metrics.
+struct ResourceFilterDataFnOption {
+  using Type = std::set<std::string>;
+};
+
 class MonitoringExporter final
     : public opentelemetry::sdk::metrics::PushMetricExporter {
  public:

google/cloud/opentelemetry/internal/monitoring_exporter_test.cc

@@ -159,6 +159,40 @@ TEST(MonitoringExporter, ExportSuccess) {
   EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kSuccess);
 }
 
+TEST(MonitoringExporterTest, MakeFilterNoOption) {
+  auto mock =
+      std::make_shared<monitoring_v3_mocks::MockMetricServiceConnection>();
+  Options options;
+
+  auto exporter = std::make_unique<MonitoringExporter>(Project("test-project"),
+                                                       mock, options);
+  EXPECT_NE(exporter, nullptr);
+}
+
+TEST(MonitoringExporterTest, MakeFilterEmptySet) {
+  auto mock =
+      std::make_shared<monitoring_v3_mocks::MockMetricServiceConnection>();
+  Options options;
+  options.set<otel_internal::ResourceFilterDataFnOption>(
+      std::set<std::string>{});
+
+  auto exporter = std::make_unique<MonitoringExporter>(Project("test-project"),
+                                                       mock, options);
+  EXPECT_NE(exporter, nullptr);
+}
+
+TEST(MonitoringExporterTest, MakeFilterWithExcludedKeys) {
+  auto mock =
+      std::make_shared<monitoring_v3_mocks::MockMetricServiceConnection>();
+  Options options;
+  std::set<std::string> excluded{"service_name", "service_version"};
+  options.set<otel_internal::ResourceFilterDataFnOption>(excluded);
+
+  auto exporter = std::make_unique<MonitoringExporter>(Project("test-project"),
+                                                       mock, options);
+  EXPECT_NE(exporter, nullptr);
+}
+
 }  // namespace
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace otel_internal

google/cloud/storage/grpc_plugin.h

@@ -131,6 +131,27 @@ struct GrpcMetricsExportTimeoutOption {
   using Type = std::chrono::seconds;
 };
 
+/**
+ * gRPC telemetry excluded labels.
+ *
+ * A set of OpenTelemetry resource attribute keys to exclude from metric labels
+ * when exporting gRPC telemetry. For example, to exclude the `service.name`
+ * label, configure the option with `{"service_name"}`.
+ *
+ * @par Example: Exclude specific labels from telemetry
+ * @code
+ * namespace gcs_ex = google::cloud::storage_experimental;
+ * auto client = google::cloud::storage::MakeGrpcClient(
+ *     google::cloud::Options{}
+ *         .set<gcs_ex::EnableGrpcMetricsOption>(true)
+ *         .set<gcs_ex::GrpcMetricsExcludedLabelsOption>(
+ *             std::set<std::string>{"service_name", "service_version"}));
+ * @endcode
+ */
+struct GrpcMetricsExcludedLabelsOption {
+  using Type = std::set<std::string>;
+};
+
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace storage_experimental
 }  // namespace cloud

google/cloud/storage/grpc_plugin_test.cc

@@ -138,6 +138,40 @@ TEST(GrpcPluginTest, BackwardsCompatibilityShims) {
 }
 #include "google/cloud/internal/diagnostics_pop.inc"
 
+TEST(GrpcPluginTest, GrpcMetricsExcludedLabelsOption) {
+  auto const expected =
+      std::set<std::string>{"service_name", "service_version", "custom_label"};
+  auto opts =
+      google::cloud::Options{}
+          .set<storage_experimental::GrpcMetricsExcludedLabelsOption>(expected);
+
+  EXPECT_EQ(expected,
+            opts.get<storage_experimental::GrpcMetricsExcludedLabelsOption>());
+}
+
+TEST(GrpcPluginTest, GrpcMetricsExcludedLabelsOptionEmpty) {
+  auto const expected = std::set<std::string>{};
+  auto opts =
+      google::cloud::Options{}
+          .set<storage_experimental::GrpcMetricsExcludedLabelsOption>(expected);
+
+  EXPECT_TRUE(opts.get<storage_experimental::GrpcMetricsExcludedLabelsOption>()
+                  .empty());
+}
+
+TEST(GrpcPluginTest, GrpcMetricsExcludedLabelsOptionSingle) {
+  auto const expected = std::set<std::string>{"service_name"};
+  auto opts =
+      google::cloud::Options{}
+          .set<storage_experimental::GrpcMetricsExcludedLabelsOption>(expected);
+
+  EXPECT_EQ(
+      1,
+      opts.get<storage_experimental::GrpcMetricsExcludedLabelsOption>().size());
+  EXPECT_EQ(expected,
+            opts.get<storage_experimental::GrpcMetricsExcludedLabelsOption>());
+}
+
 }  // namespace
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace storage

google/cloud/storage/internal/grpc/metrics_exporter_impl.cc

@@ -16,7 +16,7 @@
 
 #include "google/cloud/storage/internal/grpc/metrics_exporter_impl.h"
 #include "google/cloud/monitoring/v3/metric_connection.h"
-#include "google/cloud/opentelemetry/monitoring_exporter.h"
+#include "google/cloud/opentelemetry/internal/monitoring_exporter.h"
 #include "google/cloud/storage/grpc_plugin.h"
 #include "google/cloud/storage/internal/grpc/metrics_exporter_options.h"
 #include "google/cloud/storage/internal/grpc/metrics_meter_provider.h"
@@ -91,6 +91,11 @@ absl::optional<ExporterConfig> MakeMeterProviderConfig(
   if (!project) return absl::nullopt;
 
   auto exporter_options = MetricsExporterOptions(*project, resource);
+  if (options.has<storage_experimental::GrpcMetricsExcludedLabelsOption>()) {
+    exporter_options.set<otel_internal::ResourceFilterDataFnOption>(
+        options.get<storage_experimental::GrpcMetricsExcludedLabelsOption>());
+  }
+
   auto exporter_connection_options = MetricsExporterConnectionOptions(options);
   return ExporterConfig{std::move(*project), std::move(exporter_options),
                         std::move(exporter_connection_options),

google/cloud/storage/internal/grpc/metrics_exporter_impl_test.cc

@@ -15,6 +15,7 @@
 #ifdef GOOGLE_CLOUD_CPP_STORAGE_WITH_OTEL_METRICS
 
 #include "google/cloud/storage/internal/grpc/metrics_exporter_impl.h"
+#include "google/cloud/opentelemetry/internal/monitoring_exporter.h"
 #include "google/cloud/opentelemetry/monitoring_exporter.h"
 #include "google/cloud/storage/grpc_plugin.h"
 #include "google/cloud/storage/internal/grpc/default_options.h"
@@ -156,6 +157,64 @@ TEST(GrpcMetricsExporter, ReaderOptionsAreSetFromConfig) {
             std::chrono::milliseconds(expected_timeout));
 }
 
+TEST(MakeMeterProviderConfigTest, NoExcludedLabels) {
+  auto resource = opentelemetry::sdk::resource::Resource::Create(
+      {{"service.name", "test-service"}, {"service.version", "1.0.0"}});
+
+  Options options;
+  options.set<storage_experimental::EnableGrpcMetricsOption>(true);
+  options.set<storage::ProjectIdOption>("test-project");
+
+  auto config = MakeMeterProviderConfig(resource, options);
+
+  ASSERT_TRUE(config.has_value());
+  EXPECT_FALSE(config->exporter_options
+                   .has<otel_internal::ResourceFilterDataFnOption>());
+}
+
+TEST(MakeMeterProviderConfigTest, WithExcludedLabels) {
+  auto resource = opentelemetry::sdk::resource::Resource::Create(
+      {{"service.name", "test-service"}, {"service.version", "1.0.0"}});
+
+  std::set<std::string> excluded_labels{"service_name", "service_version"};
+  Options options;
+  options.set<storage_experimental::EnableGrpcMetricsOption>(true);
+  options.set<storage::ProjectIdOption>("test-project");
+  options.set<storage_experimental::GrpcMetricsExcludedLabelsOption>(
+      excluded_labels);
+
+  auto config = MakeMeterProviderConfig(resource, options);
+
+  ASSERT_TRUE(config.has_value());
+  EXPECT_TRUE(config->exporter_options
+                  .has<otel_internal::ResourceFilterDataFnOption>());
+
+  auto actual_excluded =
+      config->exporter_options.get<otel_internal::ResourceFilterDataFnOption>();
+  EXPECT_EQ(excluded_labels, actual_excluded);
+}
+
+TEST(MakeMeterProviderConfigTest, EmptyExcludedLabels) {
+  auto resource = opentelemetry::sdk::resource::Resource::Create(
+      {{"service.name", "test-service"}});
+
+  Options options;
+  options.set<storage_experimental::EnableGrpcMetricsOption>(true);
+  options.set<storage::ProjectIdOption>("test-project");
+  options.set<storage_experimental::GrpcMetricsExcludedLabelsOption>(
+      std::set<std::string>{});
+
+  auto config = MakeMeterProviderConfig(resource, options);
+
+  ASSERT_TRUE(config.has_value());
+  EXPECT_TRUE(config->exporter_options
+                  .has<otel_internal::ResourceFilterDataFnOption>());
+
+  auto actual_excluded =
+      config->exporter_options.get<otel_internal::ResourceFilterDataFnOption>();
+  EXPECT_TRUE(actual_excluded.empty());
+}
+
 }  // namespace
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace storage_internal