feat: add the export logic for per-connection error rate metric (#2121)

rkaregar · web-flow · commit d053f2dc98eb · 2024-02-21T09:17:52.000-05:00
* Add the metric export logic.

* Fix integration test failure by updating set of metrics.

* Refactor the creation of MonitoredResource.

* Use StackdriverStatsConfiguration to get GCE/GKE monitoring resource.

* Use preconditions to check null values.

* Remove unnecesary checks.
diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml
@@ -77,6 +77,12 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <!-- Other opencensus packages' versions are pulled through com.google.cloud:third-party-dependencies, but has to be manually specified for this one. -->
+      <dependency>
+        <groupId>io.opencensus</groupId>
+        <artifactId>opencensus-contrib-resource-util</artifactId>
+        <version>0.31.1</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml
@@ -38,6 +38,10 @@
             <groupId>io.opencensus</groupId>
             <artifactId>opencensus-exporter-stats-stackdriver</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.opencensus</groupId>
+            <artifactId>opencensus-contrib-resource-util</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.opencensus</groupId>
             <artifactId>opencensus-impl</artifactId>
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java
@@ -33,26 +33,22 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter {
   private static final Logger logger =
       Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName());
   private final MetricServiceClient metricServiceClient;
-  private final MonitoredResource monitoredResource;
+  private final MonitoredResource gceOrGkeMonitoredResource;
   private final String clientId;
 
   BigtableCreateTimeSeriesExporter(
-      MetricServiceClient metricServiceClient, MonitoredResource monitoredResource) {
+      MetricServiceClient metricServiceClient, MonitoredResource gceOrGkeMonitoredResource) {
     this.metricServiceClient = metricServiceClient;
-    this.monitoredResource = monitoredResource;
     this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue();
+    this.gceOrGkeMonitoredResource = gceOrGkeMonitoredResource;
   }
 
   public void export(Collection<Metric> metrics) {
     Map<String, List<com.google.monitoring.v3.TimeSeries>> projectToTimeSeries = new HashMap<>();
 
     for (Metric metric : metrics) {
       // only export bigtable metrics
-      if (!metric.getMetricDescriptor().getName().contains("bigtable")) {
-        continue;
-      }
-      // TODO: temporarily skip exporting per connection metrics.
-      if (metric.getMetricDescriptor().getName().contains("per_connection_error_count")) {
+      if (!BigtableStackdriverExportUtils.shouldExportMetric(metric.getMetricDescriptor())) {
         continue;
       }
 
@@ -69,7 +65,7 @@ public void export(Collection<Metric> metrics) {
                                   metric.getMetricDescriptor(),
                                   timeSeries,
                                   clientId,
-                                  monitoredResource),
+                                  gceOrGkeMonitoredResource),
                           Collectors.toList())));
 
       for (Map.Entry<String, List<com.google.monitoring.v3.TimeSeries>> entry :
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java
@@ -15,6 +15,8 @@
  */
 package com.google.cloud.bigtable.stats;
 
+import static com.google.cloud.bigtable.stats.BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW;
+
 import com.google.api.Distribution.BucketOptions;
 import com.google.api.Distribution.BucketOptions.Explicit;
 import com.google.api.Metric;
@@ -52,7 +54,7 @@
 import javax.annotation.Nullable;
 
 class BigtableStackdriverExportUtils {
-
+  private static final String BIGTABLE_RESOURCE_TYPE = "bigtable_client_raw";
   private static final Logger logger =
       Logger.getLogger(BigtableStackdriverExportUtils.class.getName());
 
@@ -90,8 +92,8 @@ class BigtableStackdriverExportUtils {
         return builder.build();
       };
 
-  // promote the following metric labels to monitored resource labels
-  private static final Set<String> PROMOTED_RESOURCE_LABELS =
+  // promote the following metric labels to Bigtable monitored resource labels
+  private static final Set<String> PROMOTED_BIGTABLE_RESOURCE_LABELS =
       ImmutableSet.of(
           BuiltinMeasureConstants.PROJECT_ID.getName(),
           BuiltinMeasureConstants.INSTANCE_ID.getName(),
@@ -102,25 +104,67 @@ class BigtableStackdriverExportUtils {
   private static final LabelKey CLIENT_UID_LABEL_KEY =
       LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client uid");
 
+  static boolean isBigtableTableMetric(MetricDescriptor metricDescriptor) {
+    return metricDescriptor.getName().contains("bigtable")
+        && !metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString());
+  }
+
+  static boolean shouldExportMetric(MetricDescriptor metricDescriptor) {
+    return isBigtableTableMetric(metricDescriptor)
+        || (metricDescriptor.getName().equals(PER_CONNECTION_ERROR_COUNT_VIEW.getName().asString())
+            && (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke()));
+  }
+
   static com.google.monitoring.v3.TimeSeries convertTimeSeries(
       MetricDescriptor metricDescriptor,
       TimeSeries timeSeries,
       String clientId,
-      MonitoredResource monitoredResource) {
-    String metricName = metricDescriptor.getName();
-    List<LabelKey> labelKeys = metricDescriptor.getLabelKeys();
+      MonitoredResource gceOrGkeMonitoredResource) {
     Type metricType = metricDescriptor.getType();
 
-    MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder();
+    com.google.monitoring.v3.TimeSeries.Builder builder;
+    if (isBigtableTableMetric(metricDescriptor)) {
+      builder =
+          setupBuilderForBigtableResource(
+              metricDescriptor,
+              MonitoredResource.newBuilder().setType(BIGTABLE_RESOURCE_TYPE),
+              timeSeries,
+              clientId);
+    } else if (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke()) {
+      builder =
+          setupBuilderForGceOrGKEResource(
+              metricDescriptor, gceOrGkeMonitoredResource, timeSeries, clientId);
+    } else {
+      logger.warning(
+          "Trying to export metric "
+              + metricDescriptor.getName()
+              + " in a non-GCE/GKE environment.");
+      return com.google.monitoring.v3.TimeSeries.newBuilder().build();
+    }
+    builder.setMetricKind(createMetricKind(metricType));
+    builder.setValueType(createValueType(metricType));
+    Timestamp startTimeStamp = timeSeries.getStartTimestamp();
+    for (Point point : timeSeries.getPoints()) {
+      builder.addPoints(createPoint(point, startTimeStamp));
+    }
+    return builder.build();
+  }
 
+  private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForBigtableResource(
+      MetricDescriptor metricDescriptor,
+      MonitoredResource.Builder monitoredResourceBuilder,
+      TimeSeries timeSeries,
+      String clientId) {
+    List<LabelKey> labelKeys = metricDescriptor.getLabelKeys();
+    String metricName = metricDescriptor.getName();
     List<LabelKey> metricTagKeys = new ArrayList<>();
     List<LabelValue> metricTagValues = new ArrayList<>();
 
     List<LabelValue> labelValues = timeSeries.getLabelValues();
     for (int i = 0; i < labelValues.size(); i++) {
       // If the label is defined in the monitored resource, convert it to
       // a monitored resource label. Otherwise, keep it as a metric label.
-      if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) {
+      if (PROMOTED_BIGTABLE_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) {
         monitoredResourceBuilder.putLabels(
             labelKeys.get(i).getKey(), labelValues.get(i).getValue());
       } else {
@@ -135,13 +179,34 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries(
         com.google.monitoring.v3.TimeSeries.newBuilder();
     builder.setResource(monitoredResourceBuilder.build());
     builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues));
-    builder.setMetricKind(createMetricKind(metricType));
-    builder.setValueType(createValueType(metricType));
-    Timestamp startTimeStamp = timeSeries.getStartTimestamp();
-    for (Point point : timeSeries.getPoints()) {
-      builder.addPoints(createPoint(point, startTimeStamp));
+
+    return builder;
+  }
+
+  private static com.google.monitoring.v3.TimeSeries.Builder setupBuilderForGceOrGKEResource(
+      MetricDescriptor metricDescriptor,
+      MonitoredResource gceOrGkeMonitoredResource,
+      TimeSeries timeSeries,
+      String clientId) {
+    List<LabelKey> labelKeys = metricDescriptor.getLabelKeys();
+    String metricName = metricDescriptor.getName();
+    List<LabelKey> metricTagKeys = new ArrayList<>();
+    List<LabelValue> metricTagValues = new ArrayList<>();
+
+    List<LabelValue> labelValues = timeSeries.getLabelValues();
+    for (int i = 0; i < labelValues.size(); i++) {
+      metricTagKeys.add(labelKeys.get(i));
+      metricTagValues.add(labelValues.get(i));
     }
-    return builder.build();
+    metricTagKeys.add(CLIENT_UID_LABEL_KEY);
+    metricTagValues.add(LabelValue.create(clientId));
+
+    com.google.monitoring.v3.TimeSeries.Builder builder =
+        com.google.monitoring.v3.TimeSeries.newBuilder();
+    builder.setResource(gceOrGkeMonitoredResource);
+    builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues));
+
+    return builder;
   }
 
   static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSeries) {
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java
@@ -28,6 +28,7 @@
 import io.opencensus.common.Duration;
 import io.opencensus.exporter.metrics.util.IntervalMetricReader;
 import io.opencensus.exporter.metrics.util.MetricReader;
+import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration;
 import io.opencensus.metrics.Metrics;
 import java.io.IOException;
 import javax.annotation.Nullable;
@@ -43,7 +44,6 @@ public class BigtableStackdriverStatsExporter {
 
   // Default export interval is 1 minute
   private static final Duration EXPORT_INTERVAL = Duration.create(60, 0);
-  private static final String RESOURCE_TYPE = "bigtable_client_raw";
 
   private static final String MONITORING_ENDPOINT =
       MoreObjects.firstNonNull(
@@ -55,13 +55,13 @@ public class BigtableStackdriverStatsExporter {
   private BigtableStackdriverStatsExporter(
       MetricServiceClient metricServiceClient,
       Duration exportInterval,
-      MonitoredResource monitoredResource) {
+      MonitoredResource gceOrGkeMonitoredResource) {
     IntervalMetricReader.Options.Builder intervalMetricReaderOptionsBuilder =
         IntervalMetricReader.Options.builder();
     intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval);
     this.intervalMetricReader =
         IntervalMetricReader.create(
-            new BigtableCreateTimeSeriesExporter(metricServiceClient, monitoredResource),
+            new BigtableCreateTimeSeriesExporter(metricServiceClient, gceOrGkeMonitoredResource),
             MetricReader.create(
                 MetricReader.Options.builder()
                     .setMetricProducerManager(
@@ -76,9 +76,13 @@ public static void register(Credentials credentials) throws IOException {
           instance == null, "Bigtable Stackdriver stats exporter is already created");
       // Default timeout for creating a client is 1 minute
       MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0));
-      MonitoredResource resourceType =
-          MonitoredResource.newBuilder().setType(RESOURCE_TYPE).build();
-      instance = new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL, resourceType);
+      MonitoredResource gceOrGkeMonitoredResource = null;
+      if (ConsumerEnvironmentUtils.isEnvGce() || ConsumerEnvironmentUtils.isEnvGke()) {
+        gceOrGkeMonitoredResource =
+            StackdriverStatsConfiguration.builder().build().getMonitoredResource();
+      }
+      instance =
+          new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL, gceOrGkeMonitoredResource);
     }
   }
 
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java
@@ -15,7 +15,25 @@
  */
 package com.google.cloud.bigtable.stats;
 
-import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.*;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APPLICATION_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APP_PROFILE;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ATTEMPT_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLIENT_NAME;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLUSTER;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.INSTANCE_ID;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.METHOD;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.OPERATION_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PER_CONNECTION_ERROR_COUNT;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PROJECT_ID;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.RETRY_COUNT;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.SERVER_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STATUS;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STREAMING;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.TABLE;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.THROTTLING_LATENCIES;
+import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ZONE;
 import static io.opencensus.stats.Aggregation.Distribution;
 import static io.opencensus.stats.Aggregation.Sum;
 
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java
@@ -37,18 +37,28 @@ public class BuiltinViews {
           BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW,
           BuiltinViewConstants.APPLICATION_LATENCIES_VIEW,
           BuiltinViewConstants.THROTTLING_LATENCIES_VIEW);
+  // We store views that don't use the Bigtable schema and need different tags in a separate set to
+  // simplify testing.
+  static final ImmutableSet<View> NON_BIGTABLE_BUILTIN_VIEWS =
+      ImmutableSet.of(BuiltinViewConstants.PER_CONNECTION_ERROR_COUNT_VIEW);
 
   @VisibleForTesting
   void registerPrivateViews(ViewManager viewManager) {
     for (View view : BIGTABLE_BUILTIN_VIEWS) {
       viewManager.registerView(view);
     }
+    for (View view : NON_BIGTABLE_BUILTIN_VIEWS) {
+      viewManager.registerView(view);
+    }
   }
 
   public static void registerBigtableBuiltinViews() {
     ViewManager viewManager = Stats.getViewManager();
     for (View view : BIGTABLE_BUILTIN_VIEWS) {
       viewManager.registerView(view);
     }
+    for (View view : NON_BIGTABLE_BUILTIN_VIEWS) {
+      viewManager.registerView(view);
+    }
   }
 }
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/ConsumerEnvironmentUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.stats;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.opencensus.contrib.resource.util.CloudResource;
+import io.opencensus.contrib.resource.util.ContainerResource;
+import io.opencensus.contrib.resource.util.HostResource;
+import io.opencensus.contrib.resource.util.ResourceUtils;
+import io.opencensus.resource.Resource;
+import java.util.Objects;
+
+/** A class for extracting details about consumer environments (GCE and GKE) for metrics. */
+class ConsumerEnvironmentUtils {
+
+  private static ResourceUtilsWrapper resourceUtilsWrapper = new ResourceUtilsWrapper();
+
+  @VisibleForTesting
+  public static void setResourceUtilsWrapper(ResourceUtilsWrapper newResourceUtilsWrapper) {
+    resourceUtilsWrapper = newResourceUtilsWrapper;
+  }
+
+  public static boolean isEnvGce() {
+    Resource resource = resourceUtilsWrapper.detectResource();
+    return Objects.equals(resource.getType(), HostResource.TYPE)
+        && Objects.equals(
+            resource.getLabels().get(CloudResource.PROVIDER_KEY), CloudResource.PROVIDER_GCP);
+  }
+
+  public static boolean isEnvGke() {
+    Resource resource = resourceUtilsWrapper.detectResource();
+    return Objects.equals(resource.getType(), ContainerResource.TYPE)
+        && Objects.equals(
+            resource.getLabels().get(CloudResource.PROVIDER_KEY), CloudResource.PROVIDER_GCP);
+  }
+
+  // We wrap the static ResourceUtils.detectResource() method in a non-static method for mocking.
+  @VisibleForTesting
+  public static class ResourceUtilsWrapper {
+    public Resource detectResource() {
+      return ResourceUtils.detectResource();
+    }
+  }
+}
diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java
@@ -61,7 +61,7 @@ public static List<String> getOperationLatencyViewTagValueStrings() {
   // the packaging step. Opencensus classes will be relocated when they are packaged but the
   // integration test files will not be. So the integration tests can't reference any transitive
   // dependencies that have been relocated.
-  static Map<String, List<String>> getViewToTagMap() {
+  static Map<String, List<String>> getBigtableViewToTagMap() {
     Map<String, List<String>> map = new HashMap<>();
     for (View view : BuiltinViews.BIGTABLE_BUILTIN_VIEWS) {
       List<TagKey> tagKeys = view.getColumns();
diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java
diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java
