Add Xds Server metric provider (#997)

* add indis observer latency metric provider

* rename the interface for OSS

* track latency for d2 node type resource as well

* track latency only for non-initial update

* deprecate old constructor and fix some styles

* add more latency metrics and address comments

* update version in gradle.properties

* cleanup

CHANGELOG.md

@@ -14,6 +14,9 @@ and what APIs have changed, if applicable.
 
 ## [Unreleased]
 
+## [29.53.0] - 2024-04-09
+- add xDS server latency metric provider
+
 ## [29.52.1] - 2024-04-03
 - fix concurrent configuration resolution issue in the Gradle plugin in Gradle 8 and above
 
@@ -5680,7 +5683,8 @@ patch operations can re-use these classes for generating patch messages.
 
 ## [0.14.1]
 
-[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.52.1...master
+[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.53.0...master
+[29.53.0]: https://github.com/linkedin/rest.li/compare/v29.52.1...v29.53.0
 [29.52.1]: https://github.com/linkedin/rest.li/compare/v29.52.0...v29.52.1
 [29.52.0]: https://github.com/linkedin/rest.li/compare/v29.51.14...v29.52.0
 [29.51.14]: https://github.com/linkedin/rest.li/compare/v29.51.13...v29.51.14

d2/src/main/java/com/linkedin/d2/balancer/D2ClientBuilder.java

@@ -48,6 +48,7 @@
 import com.linkedin.d2.discovery.event.ServiceDiscoveryEventEmitter;
 import com.linkedin.d2.discovery.stores.zk.ZKPersistentConnection;
 import com.linkedin.d2.discovery.stores.zk.ZooKeeper;
+import com.linkedin.d2.jmx.XdsServerMetricsProvider;
 import com.linkedin.d2.jmx.JmxManager;
 import com.linkedin.d2.jmx.NoOpJmxManager;
 import com.linkedin.r2.transport.common.TransportClientFactory;
@@ -215,7 +216,8 @@ public D2Client build()
                   _config.dualReadNewLbExecutor,
                   _config.xdsChannelLoadBalancingPolicy,
                   _config.xdsChannelLoadBalancingPolicyConfig,
-                  _config.subscribeToUriGlobCollection
+                  _config.subscribeToUriGlobCollection,
+                  _config._xdsServerMetricsProvider
     );
 
     final LoadBalancerWithFacilitiesFactory loadBalancerFactory = (_config.lbWithFacilitiesFactory == null) ?
@@ -717,6 +719,11 @@ public D2ClientBuilder setSubscribeToUriGlobCollection(boolean subscribeToUriGlo
     return this;
   }
 
+  public D2ClientBuilder setXdsServerMetricsProvider(XdsServerMetricsProvider xdsServerMetricsProvider) {
+    _config._xdsServerMetricsProvider = xdsServerMetricsProvider;
+    return this;
+  }
+
   private Map<String, TransportClientFactory> createDefaultTransportClientFactories()
   {
     final Map<String, TransportClientFactory> clientFactories = new HashMap<>();

d2/src/main/java/com/linkedin/d2/balancer/D2ClientConfig.java

@@ -37,7 +37,9 @@
 import com.linkedin.d2.discovery.stores.zk.ZKPersistentConnection;
 import com.linkedin.d2.discovery.stores.zk.ZooKeeper;
 import com.linkedin.d2.discovery.stores.zk.ZooKeeperStore;
+import com.linkedin.d2.jmx.XdsServerMetricsProvider;
 import com.linkedin.d2.jmx.JmxManager;
+import com.linkedin.d2.jmx.NoOpXdsServerMetricsProvider;
 import com.linkedin.d2.jmx.NoOpJmxManager;
 import com.linkedin.r2.transport.common.TransportClientFactory;
 import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
@@ -135,6 +137,7 @@ public class D2ClientConfig
   public String xdsChannelLoadBalancingPolicy = null;
   public Map<String, ?> xdsChannelLoadBalancingPolicyConfig = null;
   public boolean subscribeToUriGlobCollection = false;
+  public XdsServerMetricsProvider _xdsServerMetricsProvider = new NoOpXdsServerMetricsProvider();
 
   public D2ClientConfig()
   {
@@ -210,7 +213,8 @@ public D2ClientConfig()
                  ExecutorService dualReadNewLbExecutor,
                  String xdsChannelLoadBalancingPolicy,
                  Map<String, ?> xdsChannelLoadBalancingPolicyConfig,
-                 boolean subscribeToUriGlobCollection
+                 boolean subscribeToUriGlobCollection,
+                 XdsServerMetricsProvider xdsServerMetricsProvider
       )
   {
     this.zkHosts = zkHosts;
@@ -284,5 +288,6 @@ public D2ClientConfig()
     this.xdsChannelLoadBalancingPolicy = xdsChannelLoadBalancingPolicy;
     this.xdsChannelLoadBalancingPolicyConfig = xdsChannelLoadBalancingPolicyConfig;
     this.subscribeToUriGlobCollection = subscribeToUriGlobCollection;
+    this._xdsServerMetricsProvider = xdsServerMetricsProvider;
   }
 }

d2/src/main/java/com/linkedin/d2/jmx/NoOpXdsServerMetricsProvider.java

@@ -0,0 +1,41 @@
+package com.linkedin.d2.jmx;
+
+
+/**
+ * NoOp implementation of {@link XdsServerMetricsProvider}
+ */
+public class NoOpXdsServerMetricsProvider implements XdsServerMetricsProvider {
+  @Override
+  public long getLatencyMin() {
+    return 0;
+  }
+
+  @Override
+  public double getLatencyAverage() {
+    return 0;
+  }
+
+  @Override
+  public long getLatency50Pct() {
+    return 0;
+  }
+
+  @Override
+  public long getLatency99Pct() {
+    return 0;
+  }
+
+  @Override
+  public long getLatency99_9Pct() {
+    return 0;
+  }
+
+  @Override
+  public long getLatencyMax() {
+    return 0;
+  }
+
+  @Override
+  public void trackLatency(long latency) {
+  }
+}

d2/src/main/java/com/linkedin/d2/jmx/XdsClientJmx.java

@@ -20,14 +20,28 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 
-public class XdsClientJmx implements XdsClientJmxMBean {
+public class XdsClientJmx implements XdsClientJmxMBean
+{
 
   private final AtomicInteger _connectionLostCount = new AtomicInteger();
   private final AtomicInteger _connectionClosedCount = new AtomicInteger();
   private final AtomicInteger _reconnectionCount = new AtomicInteger();
 
   private final AtomicBoolean _isConnected = new AtomicBoolean();
   private final AtomicInteger _resourceNotFoundCount = new AtomicInteger();
+  private final XdsServerMetricsProvider _xdsServerMetricsProvider;
+
+  @Deprecated
+  public XdsClientJmx()
+  {
+    this(new NoOpXdsServerMetricsProvider());
+  }
+
+  public XdsClientJmx(XdsServerMetricsProvider xdsServerMetricsProvider)
+  {
+    _xdsServerMetricsProvider = xdsServerMetricsProvider == null ?
+        new NoOpXdsServerMetricsProvider() : xdsServerMetricsProvider;
+  }
 
   @Override
   public int getConnectionLostCount()
@@ -53,6 +67,39 @@ public int getResourceNotFoundCount()
     return _resourceNotFoundCount.get();
   }
 
+  @Override
+  public long getXdsServerLatencyMin() {
+    return _xdsServerMetricsProvider.getLatencyMin();
+  }
+
+  @Override
+  public double getXdsServerLatencyAverage()
+  {
+    return _xdsServerMetricsProvider.getLatencyAverage();
+  }
+
+  @Override
+  public long getXdsServerLatency50Pct()
+  {
+    return _xdsServerMetricsProvider.getLatency50Pct();
+  }
+
+  @Override
+  public long getXdsServerLatency99Pct()
+  {
+    return _xdsServerMetricsProvider.getLatency99Pct();
+  }
+
+  @Override
+  public long getXdsServerLatency99_9Pct() {
+    return _xdsServerMetricsProvider.getLatency99_9Pct();
+  }
+
+  @Override
+  public long getXdsServerLatencyMax() {
+    return _xdsServerMetricsProvider.getLatencyMax();
+  }
+
   @Override
   public int isDisconnected()
   {

d2/src/main/java/com/linkedin/d2/jmx/XdsClientJmxMBean.java

@@ -34,4 +34,40 @@ public interface XdsClientJmxMBean {
 
   // when the resource is not found.
   int getResourceNotFoundCount();
+
+  /**
+   * Get minimum of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getXdsServerLatencyMin();
+
+  /**
+   * Get Avg of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  double getXdsServerLatencyAverage();
+
+  /**
+   * Get 50 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getXdsServerLatency50Pct();
+
+  /**
+   * Get 90 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getXdsServerLatency99Pct();
+
+  /**
+   * Get 99.9 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getXdsServerLatency99_9Pct();
+
+  /**
+   * Get maximum of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getXdsServerLatencyMax();
 }

d2/src/main/java/com/linkedin/d2/jmx/XdsServerMetricsProvider.java

@@ -0,0 +1,49 @@
+package com.linkedin.d2.jmx;
+
+
+/**
+ * Interface for providing metrics for Xds Server
+ */
+public interface XdsServerMetricsProvider {
+  /**
+   * Get minimum of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getLatencyMin();
+
+  /**
+   * Get Avg of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  double getLatencyAverage();
+
+  /**
+   * Get 50 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getLatency50Pct();
+
+  /**
+   * Get 90 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getLatency99Pct();
+
+  /**
+   * Get 99.9 Percentile of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getLatency99_9Pct();
+
+  /**
+   * Get maximum of Xds server latency, which is from when the resource is updated on the Xds server to when the
+   * client receives it.
+   */
+  long getLatencyMax();
+
+  /**
+   * Track the latency of the Xds server.
+   * @param latency the latency to track
+   */
+  void trackLatency(long latency);
+}