Skip to content
This repository has been archived by the owner on May 14, 2022. It is now read-only.

Commit

Permalink
Use the GRPC assistant API in ClusterMembership client/server (#1251)
Browse files Browse the repository at this point in the history
* Use the GRPC assistant API in ClusterMembership client/server

* Code review requests
  • Loading branch information
tbak authored Apr 7, 2022
1 parent 083db0e commit 74de0bf
Show file tree
Hide file tree
Showing 16 changed files with 355 additions and 130 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,10 @@
import reactor.core.publisher.Mono;

/**
* Cluster membership Spring-Reactor client interface. To be used with
* {@link com.netflix.titus.runtime.connector.common.reactor.client.ReactorToGrpcClientBuilder}.
* This API maps to the GRPC ClusterMembershipService documented at:
* Reactor wrapper around the GRPC ClusterMembershipService documented at:
* https://github.com/Netflix/titus-api-definitions/blob/master/src/main/proto/netflix/titus/titus_cluster_membership_api.proto.
*/
public interface ReactorClusterMembershipClient {
public interface ClusterMembershipClient {

Mono<ClusterMembershipRevisions> getMembers();

Expand All @@ -47,4 +45,6 @@ public interface ReactorClusterMembershipClient {
Mono<Void> stopBeingLeader();

Flux<ClusterMembershipEvent> events();

void shutdown();
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;

import com.google.common.annotations.VisibleForTesting;
import com.netflix.titus.api.clustermembership.model.ClusterMember;
Expand All @@ -30,21 +29,18 @@
import com.netflix.titus.api.clustermembership.model.ClusterMembershipFunctions;
import com.netflix.titus.api.clustermembership.model.ClusterMembershipRevision;
import com.netflix.titus.api.clustermembership.model.ClusterMembershipSnapshot;
import com.netflix.titus.api.model.callmetadata.CallMetadata;
import com.netflix.titus.client.clustermembership.grpc.ReactorClusterMembershipClient;
import com.netflix.titus.client.clustermembership.grpc.ClusterMembershipClient;
import com.netflix.titus.common.runtime.TitusRuntime;
import com.netflix.titus.common.util.CollectionsExt;
import com.netflix.titus.common.util.ExceptionExt;
import com.netflix.titus.common.util.grpc.reactor.client.ReactorToGrpcClientBuilder;
import com.netflix.titus.common.util.closeable.CloseableReference;
import com.netflix.titus.common.util.rx.ReactorExt;
import com.netflix.titus.common.util.rx.ReactorRetriers;
import com.netflix.titus.common.util.rx.RetryHandlerBuilder;
import com.netflix.titus.common.util.time.Clock;
import com.netflix.titus.grpc.protogen.ClusterMember.LeadershipState;
import com.netflix.titus.grpc.protogen.ClusterMembershipEvent;
import com.netflix.titus.grpc.protogen.ClusterMembershipServiceGrpc;
import com.netflix.titus.runtime.common.grpc.GrpcClientErrorUtils;
import io.grpc.ManagedChannel;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import org.slf4j.Logger;
Expand All @@ -55,7 +51,6 @@
import reactor.core.publisher.Mono;
import reactor.core.publisher.ReplayProcessor;
import reactor.core.publisher.SignalType;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;

import static com.netflix.titus.api.clustermembership.model.ClusterMembershipFunctions.hasIpAddress;
Expand All @@ -74,14 +69,13 @@ public class SingleClusterMemberResolver implements DirectClusterMemberResolver
*/
private static final Duration GRPC_REQUEST_TIMEOUT = Duration.ofSeconds(1);

private final CloseableReference<ClusterMembershipClient> clientRef;
private final String name;
private final ClusterMembershipResolverConfiguration configuration;
private final ClusterMemberAddress address;
private final ClusterMemberVerifier clusterMemberVerifier;
private final Clock clock;

private final ManagedChannel channel;
private final ReactorClusterMembershipClient client;
private final Disposable eventStreamDisposable;

private volatile String rejectedMemberError;
Expand All @@ -95,33 +89,20 @@ public class SingleClusterMemberResolver implements DirectClusterMemberResolver
private final AtomicReference<Long> disconnectTimeRef;

public SingleClusterMemberResolver(ClusterMembershipResolverConfiguration configuration,
Function<ClusterMemberAddress, ManagedChannel> channelProvider,
CloseableReference<ClusterMembershipClient> clientRef,
ClusterMemberAddress address,
ClusterMemberVerifier clusterMemberVerifier,
Scheduler scheduler,
TitusRuntime titusRuntime) {
this.clientRef = clientRef;
this.name = "member@" + ClusterMembershipFunctions.toStringUri(address);
this.configuration = configuration;
this.address = address;
this.clusterMemberVerifier = clusterMemberVerifier;
this.clock = titusRuntime.getClock();

this.channel = channelProvider.apply(address);
this.disconnectTimeRef = new AtomicReference<>(clock.wallTime());
this.client = ReactorToGrpcClientBuilder
.newBuilderWithDefaults(
ReactorClusterMembershipClient.class,
ClusterMembershipServiceGrpc.newStub(channel),
ClusterMembershipServiceGrpc.getServiceDescriptor(),
CallMetadata.class
)
// FIXME Once call metadata interceptor is moved into common module
// .withGrpcStubDecorator(AnonymousCallMetadataResolver.getInstance())
.withTimeout(GRPC_REQUEST_TIMEOUT)
.withStreamingTimeout(Duration.ofMillis(configuration.getSingleMemberReconnectIntervalMs()))
.build();

this.eventStreamDisposable = client.events()
this.eventStreamDisposable = clientRef.get().events()
.materialize()
.flatMap(signal -> {
if (signal.getType() == SignalType.ON_COMPLETE) {
Expand Down Expand Up @@ -189,7 +170,7 @@ private Optional<String> checkSnapshot(ClusterMembershipEvent.Snapshot snapshot)
public void shutdown() {
ReactorExt.safeDispose(eventStreamDisposable);
ExceptionExt.silent(eventSink::complete);
ExceptionExt.silent(channel::shutdownNow);
clientRef.close();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright 2022 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.titus.client.clustermembership.resolver;

import com.google.protobuf.Empty;
import com.netflix.titus.client.clustermembership.grpc.ClusterMembershipClient;
import com.netflix.titus.grpc.protogen.ClusterMembershipEvent;
import com.netflix.titus.grpc.protogen.ClusterMembershipRevision;
import com.netflix.titus.grpc.protogen.ClusterMembershipRevisions;
import com.netflix.titus.grpc.protogen.ClusterMembershipServiceGrpc;
import com.netflix.titus.grpc.protogen.ClusterMembershipServiceGrpc.ClusterMembershipServiceStub;
import com.netflix.titus.grpc.protogen.DeleteMemberLabelsRequest;
import com.netflix.titus.grpc.protogen.EnableMemberRequest;
import com.netflix.titus.grpc.protogen.MemberId;
import com.netflix.titus.grpc.protogen.UpdateMemberLabelsRequest;
import io.grpc.ManagedChannel;
import io.grpc.stub.StreamObserver;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.MonoSink;

/**
* FIXME We need this implementation for testing as DefaultClusterMembershipClient is located in titus-common-server package.
* To resolve this issue we have to move CallMetadata classes to titus-common-client.
*/
class SimpleClusterMembershipClient implements ClusterMembershipClient {

private final ManagedChannel channel;
private final ClusterMembershipServiceStub stub;

public SimpleClusterMembershipClient(ManagedChannel channel) {
this.channel = channel;
this.stub = ClusterMembershipServiceGrpc.newStub(channel);
}

@Override
public Mono<ClusterMembershipRevisions> getMembers() {
return Mono.create(sink -> stub.getMembers(Empty.getDefaultInstance(), connectSink(sink)));
}

@Override
public Mono<ClusterMembershipRevision> getMember(MemberId request) {
return Mono.create(sink -> stub.getMember(request, connectSink(sink)));
}

@Override
public Mono<ClusterMembershipRevision> updateMemberLabels(UpdateMemberLabelsRequest request) {
return Mono.create(sink -> stub.updateMemberLabels(request, connectSink(sink)));
}

@Override
public Mono<ClusterMembershipRevision> deleteMemberLabels(DeleteMemberLabelsRequest request) {
return Mono.create(sink -> stub.deleteMemberLabels(request, connectSink(sink)));
}

@Override
public Mono<ClusterMembershipRevision> enableMember(EnableMemberRequest request) {
return Mono.create(sink -> stub.enableMember(request, connectSink(sink)));
}

@Override
public Mono<Void> stopBeingLeader() {
return Mono.<Empty>create(sink -> stub.stopBeingLeader(Empty.getDefaultInstance(), connectSink(sink)))
.ignoreElement()
.cast(Void.class);
}

@Override
public Flux<ClusterMembershipEvent> events() {
return Flux.create(sink -> {
stub.events(Empty.getDefaultInstance(), new StreamObserver<ClusterMembershipEvent>() {
@Override
public void onNext(ClusterMembershipEvent value) {
sink.next(value);
}

@Override
public void onError(Throwable t) {
sink.error(t);
}

@Override
public void onCompleted() {
sink.complete();
}
});
});
}

@Override
public void shutdown() {
channel.shutdownNow();
}

private <T> StreamObserver<T> connectSink(MonoSink<T> sink) {
return new StreamObserver<T>() {
@Override
public void onNext(T value) {
sink.success(value);
}

@Override
public void onError(Throwable t) {
sink.error(t);
}

@Override
public void onCompleted() {
sink.success();
}
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@

import com.netflix.titus.api.clustermembership.model.ClusterMemberAddress;
import com.netflix.titus.api.clustermembership.model.ClusterMembershipSnapshot;
import com.netflix.titus.client.clustermembership.grpc.ClusterMembershipClient;
import com.netflix.titus.common.runtime.TitusRuntime;
import com.netflix.titus.common.runtime.TitusRuntimes;
import com.netflix.titus.common.util.ExceptionExt;
import com.netflix.titus.common.util.archaius2.Archaius2Ext;
import com.netflix.titus.common.util.closeable.CloseableReference;
import com.netflix.titus.grpc.protogen.ClusterMember;
import com.netflix.titus.grpc.protogen.ClusterMembershipRevision;
import io.grpc.Server;
Expand All @@ -34,7 +36,6 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import reactor.core.scheduler.Schedulers;

import static com.jayway.awaitility.Awaitility.await;
import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -91,12 +92,12 @@ private void setupServerAndResolver(ClusterMemberVerifier clusterMemberVerifier)
.build()
.start();

SimpleClusterMembershipClient client = new SimpleClusterMembershipClient(InProcessChannelBuilder.forName(serviceName).directExecutor().build());
this.resolver = new SingleClusterMemberResolver(
configuration,
address -> InProcessChannelBuilder.forName(serviceName).directExecutor().build(),
CloseableReference.referenceOf(client, ClusterMembershipClient::shutdown),
ADDRESS,
clusterMemberVerifier,
Schedulers.parallel(),
titusRuntime
);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright 2022 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.titus.runtime.clustermembership;

import com.google.protobuf.Empty;
import com.netflix.titus.client.clustermembership.grpc.ClusterMembershipClient;
import com.netflix.titus.common.util.ExceptionExt;
import com.netflix.titus.grpc.protogen.ClusterMembershipEvent;
import com.netflix.titus.grpc.protogen.ClusterMembershipRevision;
import com.netflix.titus.grpc.protogen.ClusterMembershipRevisions;
import com.netflix.titus.grpc.protogen.ClusterMembershipServiceGrpc;
import com.netflix.titus.grpc.protogen.ClusterMembershipServiceGrpc.ClusterMembershipServiceStub;
import com.netflix.titus.grpc.protogen.DeleteMemberLabelsRequest;
import com.netflix.titus.grpc.protogen.EnableMemberRequest;
import com.netflix.titus.grpc.protogen.MemberId;
import com.netflix.titus.grpc.protogen.UpdateMemberLabelsRequest;
import com.netflix.titus.runtime.endpoint.common.grpc.assistant.GrpcClientCallAssistant;
import com.netflix.titus.runtime.endpoint.common.grpc.assistant.GrpcClientCallAssistantFactory;
import io.grpc.ManagedChannel;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

public class DefaultClusterMembershipClient implements ClusterMembershipClient {

private final GrpcClientCallAssistant<ClusterMembershipServiceStub> assistant;
private final ManagedChannel managedChannel;

public DefaultClusterMembershipClient(GrpcClientCallAssistantFactory assistantFactory,
ManagedChannel managedChannel) {
this.managedChannel = managedChannel;
this.assistant = assistantFactory.create(ClusterMembershipServiceGrpc.newStub(managedChannel));
}

@Override
public Mono<ClusterMembershipRevisions> getMembers() {
return assistant.asMono((stub, responseStream) -> stub.getMembers(Empty.getDefaultInstance(), responseStream));
}

@Override
public Mono<ClusterMembershipRevision> getMember(MemberId request) {
return assistant.asMono((stub, responseStream) -> stub.getMember(request, responseStream));
}

@Override
public Mono<ClusterMembershipRevision> updateMemberLabels(UpdateMemberLabelsRequest request) {
return assistant.asMono((stub, responseStream) -> stub.updateMemberLabels(request, responseStream));
}

@Override
public Mono<ClusterMembershipRevision> deleteMemberLabels(DeleteMemberLabelsRequest request) {
return assistant.asMono((stub, responseStream) -> stub.deleteMemberLabels(request, responseStream));
}

@Override
public Mono<ClusterMembershipRevision> enableMember(EnableMemberRequest request) {
return assistant.asMono((stub, responseStream) -> stub.enableMember(request, responseStream));
}

@Override
public Mono<Void> stopBeingLeader() {
return assistant.asMonoEmpty((stub, responseStream) -> stub.stopBeingLeader(Empty.getDefaultInstance(), responseStream));
}

@Override
public Flux<ClusterMembershipEvent> events() {
return assistant.asFlux((stub, responseStream) -> stub.events(Empty.getDefaultInstance(), responseStream));
}

@Override
public void shutdown() {
ExceptionExt.silent(managedChannel::shutdownNow);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@

import com.netflix.titus.api.clustermembership.service.ClusterMembershipService;
import com.netflix.titus.common.runtime.TitusRuntime;
import com.netflix.titus.runtime.endpoint.common.grpc.assistant.GrpcServerCallAssistant;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ClusterMembershipGrpcEndpointComponent {

@Bean
public ReactorClusterMembershipGrpcService getReactorClusterMembershipGrpcService(ClusterMembershipService service, TitusRuntime titusRuntime) {
return new ReactorClusterMembershipGrpcService(service, titusRuntime);
public GrpcClusterMembershipService getGrpcClusterMembershipService(ClusterMembershipService service,
GrpcServerCallAssistant assistant,
TitusRuntime titusRuntime) {
return new GrpcClusterMembershipService(service, assistant, titusRuntime);
}
}
Loading

0 comments on commit 74de0bf

Please sign in to comment.