Visitar URL original
rls: implement stale_header_data caching and propagation in RLS (#12972) · grpc/grpc-java@7843bd4 · GitHub
Skip to content

Commit 7843bd4

Browse files
AgraVatorAgraVator
authored
rls: implement stale_header_data caching and propagation in RLS (#12972)
This PR implements stale_header_data caching and propagation for Route Lookup Service (RLS) in :grpc-rls, addressing gRPC Java does not handle RLS stale_header_data properly).
1 parent 3cb7007 commit 7843bd4

5 files changed

Lines changed: 237 additions & 9 deletions

File tree

‎rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ private void periodicClean() {
324324
@GuardedBy("lock")
325325
private CachedRouteLookupResponse asyncRlsCall(
326326
RouteLookupRequestKey routeLookupRequestKey, @Nullable BackoffPolicy backoffPolicy,
327-
RouteLookupRequest.Reason routeLookupReason) {
327+
RouteLookupRequest.Reason routeLookupReason, @Nullable String staleHeaderData) {
328328
if (throttler.shouldThrottle()) {
329329
logger.log(ChannelLogLevel.DEBUG, "[RLS Entry {0}] Throttled RouteLookup",
330330
routeLookupRequestKey);
@@ -336,7 +336,8 @@ private CachedRouteLookupResponse asyncRlsCall(
336336
}
337337
final SettableFuture<RouteLookupResponse> response = SettableFuture.create();
338338
io.grpc.lookup.v1.RouteLookupRequest routeLookupRequest = REQUEST_CONVERTER.convert(
339-
RouteLookupRequest.create(routeLookupRequestKey.keyMap(), routeLookupReason));
339+
RouteLookupRequest.create(
340+
routeLookupRequestKey.keyMap(), routeLookupReason, staleHeaderData));
340341
logger.log(ChannelLogLevel.DEBUG,
341342
"[RLS Entry {0}] Starting RouteLookup: {1}", routeLookupRequestKey, routeLookupRequest);
342343
rlsStub.withDeadlineAfter(callTimeoutNanos, TimeUnit.NANOSECONDS)
@@ -386,7 +387,7 @@ final CachedRouteLookupResponse get(final RouteLookupRequestKey routeLookupReque
386387
}
387388
return asyncRlsCall(routeLookupRequestKey, cacheEntry instanceof BackoffCacheEntry
388389
? ((BackoffCacheEntry) cacheEntry).backoffPolicy : null,
389-
RouteLookupRequest.Reason.REASON_MISS);
390+
RouteLookupRequest.Reason.REASON_MISS, /* staleHeaderData= */ null);
390391
}
391392

392393
if (cacheEntry instanceof DataCacheEntry) {
@@ -717,7 +718,7 @@ void maybeRefresh() {
717718
logger.log(ChannelLogLevel.DEBUG,
718719
"[RLS Entry {0}] Cache entry is stale, refreshing", routeLookupRequestKey);
719720
asyncRlsCall(routeLookupRequestKey, /* backoffPolicy= */ null,
720-
RouteLookupRequest.Reason.REASON_STALE);
721+
RouteLookupRequest.Reason.REASON_STALE, getHeaderData());
721722
}
722723
}
723724

‎rls/src/main/java/io/grpc/rls/RlsProtoConverters.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,18 +65,22 @@ static final class RouteLookupRequestConverter
6565
protected RlsProtoData.RouteLookupRequest doForward(RouteLookupRequest routeLookupRequest) {
6666
return RlsProtoData.RouteLookupRequest.create(
6767
ImmutableMap.copyOf(routeLookupRequest.getKeyMapMap()),
68-
RlsProtoData.RouteLookupRequest.Reason.valueOf(routeLookupRequest.getReason().name())
68+
RlsProtoData.RouteLookupRequest.Reason.valueOf(routeLookupRequest.getReason().name()),
69+
Strings.emptyToNull(routeLookupRequest.getStaleHeaderData())
6970
);
7071
}
7172

7273
@Override
7374
protected RouteLookupRequest doBackward(RlsProtoData.RouteLookupRequest routeLookupRequest) {
74-
return
75+
RouteLookupRequest.Builder builder =
7576
RouteLookupRequest.newBuilder()
7677
.setTargetType("grpc")
7778
.setReason(RouteLookupRequest.Reason.valueOf(routeLookupRequest.reason().name()))
78-
.putAllKeyMap(routeLookupRequest.keyMap())
79-
.build();
79+
.putAllKeyMap(routeLookupRequest.keyMap());
80+
if (routeLookupRequest.staleHeaderData() != null) {
81+
builder.setStaleHeaderData(routeLookupRequest.staleHeaderData());
82+
}
83+
return builder.build();
8084
}
8185
}
8286

‎rls/src/main/java/io/grpc/rls/RlsProtoData.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,16 @@ enum Reason {
6161
/** Returns a map of key values extracted via key builders for the gRPC or HTTP request. */
6262
abstract ImmutableMap<String, String> keyMap();
6363

64+
@Nullable
65+
abstract String staleHeaderData();
66+
67+
static RouteLookupRequest create(
68+
ImmutableMap<String, String> keyMap, Reason reason, @Nullable String staleHeaderData) {
69+
return new AutoValue_RlsProtoData_RouteLookupRequest(reason, keyMap, staleHeaderData);
70+
}
71+
6472
static RouteLookupRequest create(ImmutableMap<String, String> keyMap, Reason reason) {
65-
return new AutoValue_RlsProtoData_RouteLookupRequest(reason, keyMap);
73+
return create(keyMap, reason, null);
6674
}
6775
}
6876

‎rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,160 @@ public void get_noError_lifeCycle() throws Exception {
277277
inOrder.verifyNoMoreInteractions();
278278
}
279279

280+
@Test
281+
public void asyncRefresh_sendsStaleHeaderData() throws Exception {
282+
setUpRlsLbClient();
283+
RlsProtoData.RouteLookupRequestKey routeLookupRequestKey =
284+
RlsProtoData.RouteLookupRequestKey.create(
285+
ImmutableMap.of(
286+
"server", "bigtable.googleapis.com", "service-key", "foo", "method-key", "bar"));
287+
rlsServerImpl.setLookupTable(
288+
ImmutableMap.of(
289+
routeLookupRequestKey,
290+
RouteLookupResponse.create(ImmutableList.of("target"), "stale-header-v1")));
291+
292+
// Initial lookup: cache miss
293+
CachedRouteLookupResponse resp = getInSyncContext(routeLookupRequestKey);
294+
assertThat(resp.isPending()).isTrue();
295+
296+
// RLS server response arrives
297+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
298+
resp = getInSyncContext(routeLookupRequestKey);
299+
assertThat(resp.hasData()).isTrue();
300+
assertThat(resp.getHeaderData()).isEqualTo("stale-header-v1");
301+
assertThat(rlsServerImpl.routeLookupReason).isEqualTo(
302+
io.grpc.lookup.v1.RouteLookupRequest.Reason.REASON_MISS);
303+
304+
// Advance fake clock past staleAge
305+
fakeClock.forwardTime(ROUTE_LOOKUP_CONFIG.staleAgeInNanos(), TimeUnit.NANOSECONDS);
306+
307+
rlsServerImpl.routeLookupReason = null;
308+
rlsServerImpl.routeLookupStaleHeaderData = null;
309+
310+
// Lookup on stale entry: returns cached response immediately
311+
resp = getInSyncContext(routeLookupRequestKey);
312+
assertThat(resp.hasData()).isTrue();
313+
assertThat(resp.getHeaderData()).isEqualTo("stale-header-v1");
314+
315+
// Async refresh finishes
316+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
317+
318+
assertThat(rlsServerImpl.routeLookupReason).isEqualTo(
319+
io.grpc.lookup.v1.RouteLookupRequest.Reason.REASON_STALE);
320+
assertThat(rlsServerImpl.routeLookupStaleHeaderData).isEqualTo("stale-header-v1");
321+
}
322+
323+
@Test
324+
public void updatedHeaderData_replacesCachedHeaderData() throws Exception {
325+
setUpRlsLbClient();
326+
RlsProtoData.RouteLookupRequestKey routeLookupRequestKey =
327+
RlsProtoData.RouteLookupRequestKey.create(
328+
ImmutableMap.of(
329+
"server", "bigtable.googleapis.com", "service-key", "foo", "method-key", "bar"));
330+
rlsServerImpl.setLookupTable(
331+
ImmutableMap.of(
332+
routeLookupRequestKey,
333+
RouteLookupResponse.create(ImmutableList.of("target"), "stale-header-v1")));
334+
335+
// Initial lookup
336+
getInSyncContext(routeLookupRequestKey);
337+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
338+
339+
// RLS server updated to return new header data
340+
rlsServerImpl.setLookupTable(
341+
ImmutableMap.of(
342+
routeLookupRequestKey,
343+
RouteLookupResponse.create(ImmutableList.of("target"), "updated-header-v2")));
344+
345+
// Advance fake clock past staleAge
346+
fakeClock.forwardTime(ROUTE_LOOKUP_CONFIG.staleAgeInNanos(), TimeUnit.NANOSECONDS);
347+
348+
// Stale lookup triggers background refresh
349+
getInSyncContext(routeLookupRequestKey);
350+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
351+
352+
// Verify cache entry is updated with new header data
353+
CachedRouteLookupResponse resp = getInSyncContext(routeLookupRequestKey);
354+
assertThat(resp.getHeaderData()).isEqualTo("updated-header-v2");
355+
356+
// Advance past staleAge again
357+
fakeClock.forwardTime(ROUTE_LOOKUP_CONFIG.staleAgeInNanos(), TimeUnit.NANOSECONDS);
358+
rlsServerImpl.routeLookupStaleHeaderData = null;
359+
360+
// Next stale refresh sends updated stale_header_data
361+
getInSyncContext(routeLookupRequestKey);
362+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
363+
364+
assertThat(rlsServerImpl.routeLookupStaleHeaderData).isEqualTo("updated-header-v2");
365+
}
366+
367+
@Test
368+
public void expiredEntry_cacheMiss_clearsStaleHeaderData() throws Exception {
369+
setUpRlsLbClient();
370+
RlsProtoData.RouteLookupRequestKey routeLookupRequestKey =
371+
RlsProtoData.RouteLookupRequestKey.create(
372+
ImmutableMap.of(
373+
"server", "bigtable.googleapis.com", "service-key", "foo", "method-key", "bar"));
374+
rlsServerImpl.setLookupTable(
375+
ImmutableMap.of(
376+
routeLookupRequestKey,
377+
RouteLookupResponse.create(ImmutableList.of("target"), "stale-header-v1")));
378+
379+
// Initial lookup
380+
getInSyncContext(routeLookupRequestKey);
381+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
382+
383+
// Advance fake clock past maxAge (expiration)
384+
fakeClock.forwardTime(ROUTE_LOOKUP_CONFIG.maxAgeInNanos(), TimeUnit.NANOSECONDS);
385+
386+
rlsServerImpl.routeLookupReason = null;
387+
rlsServerImpl.routeLookupStaleHeaderData = null;
388+
389+
// Expired entry triggers cache miss
390+
CachedRouteLookupResponse resp = getInSyncContext(routeLookupRequestKey);
391+
assertThat(resp.isPending()).isTrue();
392+
393+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
394+
395+
assertThat(rlsServerImpl.routeLookupReason).isEqualTo(
396+
io.grpc.lookup.v1.RouteLookupRequest.Reason.REASON_MISS);
397+
assertThat(rlsServerImpl.routeLookupStaleHeaderData).isEmpty();
398+
}
399+
400+
@Test
401+
public void rlsPicker_attachesHeaderDataToPickedRpcs() throws Exception {
402+
setUpRlsLbClient();
403+
RlsProtoData.RouteLookupRequestKey routeLookupRequestKey =
404+
RlsProtoData.RouteLookupRequestKey.create(
405+
ImmutableMap.of(
406+
"server", "bigtable.googleapis.com", "service-key", "service1",
407+
"method-key", "create"));
408+
rlsServerImpl.setLookupTable(
409+
ImmutableMap.of(
410+
routeLookupRequestKey,
411+
RouteLookupResponse.create(
412+
ImmutableList.of("primary.cloudbigtable.googleapis.com"),
413+
"header-rls-data-value")));
414+
415+
// Populate cache and wait for server response
416+
getInSyncContext(routeLookupRequestKey);
417+
fakeClock.forwardTime(SERVER_LATENCY_MILLIS, TimeUnit.MILLISECONDS);
418+
419+
ArgumentCaptor<SubchannelPicker> pickerCaptor =
420+
ArgumentCaptor.forClass(SubchannelPicker.class);
421+
verify(helper, times(3))
422+
.updateBalancingState(any(ConnectivityState.class), pickerCaptor.capture());
423+
424+
Metadata headers = new Metadata();
425+
headers.put(RLS_DATA_KEY, "old-header-data");
426+
427+
PickResult pickResult = getPickResultForCreate(pickerCaptor, headers);
428+
429+
assertThat(pickResult.getStatus().isOk()).isTrue();
430+
assertThat(headers.get(RLS_DATA_KEY)).isEqualTo("header-rls-data-value");
431+
}
432+
433+
280434
@Test
281435
public void rls_withCustomRlsChannelServiceConfig() throws Exception {
282436
Map<String, ?> routeLookupChannelServiceConfig =
@@ -1120,6 +1274,7 @@ private static final class StaticFixedDelayRlsServerImpl
11201274
private Map<RlsProtoData.RouteLookupRequestKey, RouteLookupResponse> lookupTable =
11211275
ImmutableMap.of();
11221276
io.grpc.lookup.v1.RouteLookupRequest.Reason routeLookupReason;
1277+
String routeLookupStaleHeaderData;
11231278

11241279
public StaticFixedDelayRlsServerImpl(
11251280
long responseDelayNano, ScheduledExecutorService scheduledExecutorService) {
@@ -1143,6 +1298,7 @@ public void routeLookup(final io.grpc.lookup.v1.RouteLookupRequest request,
11431298
@Override
11441299
public void run() {
11451300
routeLookupReason = request.getReason();
1301+
routeLookupStaleHeaderData = request.getStaleHeaderData();
11461302
RouteLookupResponse response =
11471303
lookupTable.get(
11481304
RlsProtoData.RouteLookupRequestKey.create(

‎rls/src/test/java/io/grpc/rls/RlsProtoConvertersTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,65 @@ public void convert_toRequestObject() {
7171
assertThat(proto.getReason()).isEqualTo(RouteLookupRequest.Reason.REASON_MISS);
7272
}
7373

74+
@Test
75+
public void convert_toRequestProto_staleHeaderData() {
76+
Converter<RouteLookupRequest, RlsProtoData.RouteLookupRequest> converter =
77+
new RouteLookupRequestConverter();
78+
79+
// Non-null value
80+
RouteLookupRequest protoWithStaleHeader = RouteLookupRequest.newBuilder()
81+
.putKeyMap("key1", "val1")
82+
.setStaleHeaderData("stale-header-v1")
83+
.build();
84+
RlsProtoData.RouteLookupRequest objectWithStaleHeader =
85+
converter.convert(protoWithStaleHeader);
86+
assertThat(objectWithStaleHeader.staleHeaderData()).isEqualTo("stale-header-v1");
87+
88+
// Null value (unset)
89+
RouteLookupRequest protoUnset = RouteLookupRequest.newBuilder()
90+
.putKeyMap("key1", "val1")
91+
.build();
92+
RlsProtoData.RouteLookupRequest objectUnset = converter.convert(protoUnset);
93+
assertThat(objectUnset.staleHeaderData()).isNull();
94+
95+
// Empty string
96+
RouteLookupRequest protoEmpty = RouteLookupRequest.newBuilder()
97+
.putKeyMap("key1", "val1")
98+
.setStaleHeaderData("")
99+
.build();
100+
RlsProtoData.RouteLookupRequest objectEmpty = converter.convert(protoEmpty);
101+
assertThat(objectEmpty.staleHeaderData()).isNull();
102+
}
103+
104+
@Test
105+
public void convert_toRequestObject_staleHeaderData() {
106+
Converter<RlsProtoData.RouteLookupRequest, RouteLookupRequest> converter =
107+
new RouteLookupRequestConverter().reverse();
108+
109+
// Non-null value
110+
RlsProtoData.RouteLookupRequest objectWithStaleHeader =
111+
RlsProtoData.RouteLookupRequest.create(
112+
ImmutableMap.of("key1", "val1"),
113+
RlsProtoData.RouteLookupRequest.Reason.REASON_STALE,
114+
"stale-header-v1");
115+
RouteLookupRequest protoWithStaleHeader = converter.convert(objectWithStaleHeader);
116+
assertThat(protoWithStaleHeader.getStaleHeaderData()).isEqualTo("stale-header-v1");
117+
assertThat(protoWithStaleHeader.getReason())
118+
.isEqualTo(RouteLookupRequest.Reason.REASON_STALE);
119+
120+
// Null value
121+
RlsProtoData.RouteLookupRequest objectNull =
122+
RlsProtoData.RouteLookupRequest.create(
123+
ImmutableMap.of("key1", "val1"),
124+
RlsProtoData.RouteLookupRequest.Reason.REASON_MISS,
125+
null);
126+
RouteLookupRequest protoNull = converter.convert(objectNull);
127+
assertThat(protoNull.getStaleHeaderData()).isEmpty();
128+
assertThat(protoNull.getReason())
129+
.isEqualTo(RouteLookupRequest.Reason.REASON_MISS);
130+
}
131+
132+
74133
@Test
75134
public void convert_toResponseProto() {
76135
Converter<RouteLookupResponse, RlsProtoData.RouteLookupResponse> converter =

0 commit comments

Comments
 (0)