Visitar URL original
Cherry-pick recent changes from master (#9240) · grpc/grpc-go@e393849 · GitHub
Skip to content

Commit e393849

Browse files
authored
Cherry-pick recent changes from master (#9240)
RELEASE NOTES: - server: Stop reading from the connection when flooded by HTTP/2 frames. The default value for this limit is 100 frames, excluding DATA and HEADERS, and may be changed by setting environment variable `GRPC_GO_EXPERIMENTAL_CONTROL_BUFFER_THROTTLE_LIMIT`. - xds/rbac: Support `Metadata` and `RequestedServerName` permissions matcher fields. If present in a DENY rule, previously these would be ignored and fail-open. - xds/rbac: Fix panic when parsing unsupported fields in `NotRule`/`NotId` permissions. - xds/rbac: Support the deprecated `source_ip` principal identifier by treating it as equivalent to `direct_remote_ip`.
1 parent 2a112a8 commit e393849

9 files changed

Lines changed: 655 additions & 179 deletions

File tree

‎internal/envconfig/envconfig.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,17 @@ var (
141141
// feature if unforeseen issues arise, and it will be removed in a future
142142
// release.
143143
EnableHTTPFramerReadBufferPooling = boolFromEnv("GRPC_GO_EXPERIMENTAL_HTTP_FRAMER_READ_BUFFER_POOLING", true)
144+
145+
// ControlBufferThrottleLimit is the maximum number of control frames that can
146+
// be queued in the control buffer before throttling is applied. The value
147+
// must be between 1 and 10,000, and is set to 100 by default.
148+
//
149+
// This environment variable serves as an escape hatch to increase the
150+
// throttling limit if unforeseen issues arise, and it will be removed in a
151+
// future release.
152+
//
153+
// TODO: Remove this env var once v1.83.0 is release.
154+
ControlBufferThrottleLimit = uint64FromEnv("GRPC_GO_EXPERIMENTAL_CONTROL_BUFFER_THROTTLE_LIMIT", 100, 1, 10000)
144155
)
145156

146157
func boolFromEnv(envVar string, def bool) bool {

‎internal/transport/controlbuf.go‎

Lines changed: 84 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
"golang.org/x/net/http2"
3131
"golang.org/x/net/http2/hpack"
32+
"google.golang.org/grpc/internal/envconfig"
3233
"google.golang.org/grpc/internal/grpclog"
3334
"google.golang.org/grpc/mem"
3435
)
@@ -96,61 +97,70 @@ func (il *itemList) isEmpty() bool {
9697
return il.head == nil
9798
}
9899

99-
// The following defines various control items which could flow through
100-
// the control buffer of transport. They represent different aspects of
101-
// control tasks, e.g., flow control, settings, streaming resetting, etc.
102-
103-
// maxQueuedTransportResponseFrames is the most queued "transport response"
104-
// frames we will buffer before preventing new reads from occurring on the
105-
// transport. These are control frames sent in response to client requests,
106-
// such as RST_STREAM due to bad headers or settings acks.
107-
const maxQueuedTransportResponseFrames = 50
100+
// maxQueuedControlBufferItems is the maximum number of frames (other than
101+
// HEADERS and DATA) that we will buffer before preventing new reads from
102+
// occurring on the transport. These are control frames sent in response to
103+
// client requests, or frames that result in work being scheduled, such as
104+
// RST_STREAM due to bad headers or settings acks.
105+
var maxQueuedControlBufferItems = int(envconfig.ControlBufferThrottleLimit)
108106

109107
type cbItem interface {
110-
isTransportResponseFrame() bool
108+
isThrottled() bool
111109
}
112110

111+
// throttledItem represents every item in the controlBuffer to which the overall
112+
// throttling limit applies, other than outgoing HEADERS and DATA frames.
113+
type throttledItem struct{}
114+
115+
func (throttledItem) isThrottled() bool { return true }
116+
117+
// The following defines various control items which could flow through
118+
// the control buffer of transport. They represent different aspects of
119+
// control tasks, e.g., flow control, settings, streaming resetting, etc.
120+
113121
// registerStream is used to register an incoming stream with loopy writer.
114122
type registerStream struct {
123+
throttledItem
115124
streamID uint32
116125
wq *writeQuota
117126
}
118127

119-
func (*registerStream) isTransportResponseFrame() bool { return false }
120-
121-
// headerFrame is also used to register stream on the client-side.
122-
type headerFrame struct {
128+
type clientHeaders struct {
123129
streamID uint32
124130
hf []hpack.HeaderField
125-
endStream bool // Valid on server side.
126-
initStream func(uint32) error // Used only on the client side.
131+
initStream func(uint32) error
127132
onWrite func()
128-
wq *writeQuota // write quota for the stream created.
129-
cleanup *cleanupStream // Valid on the server side.
130-
onOrphaned func(error) // Valid on client-side
133+
wq *writeQuota
134+
onOrphaned func(error)
131135
}
132136

133-
func (h *headerFrame) isTransportResponseFrame() bool {
134-
return h.cleanup != nil && h.cleanup.rst // Results in a RST_STREAM
137+
func (*clientHeaders) isThrottled() bool { return false }
138+
139+
type serverHeaders struct {
140+
streamID uint32
141+
hf []hpack.HeaderField
142+
endStream bool
143+
onWrite func()
144+
cleanup *cleanupStream
135145
}
136146

147+
func (h *serverHeaders) isThrottled() bool { return false }
148+
137149
type cleanupStream struct {
150+
throttledItem
138151
streamID uint32
139152
rst bool
140153
rstCode http2.ErrCode
141154
onWrite func()
142155
}
143156

144-
func (c *cleanupStream) isTransportResponseFrame() bool { return c.rst } // Results in a RST_STREAM
145-
146157
type earlyAbortStream struct {
158+
throttledItem
147159
streamID uint32
148160
rst bool
149161
hf []hpack.HeaderField // Pre-built header fields
150162
}
151163

152-
func (*earlyAbortStream) isTransportResponseFrame() bool { return false }
153-
154164
type dataFrame struct {
155165
streamID uint32
156166
endStream bool
@@ -162,70 +172,60 @@ type dataFrame struct {
162172
onEachWrite func()
163173
}
164174

165-
func (*dataFrame) isTransportResponseFrame() bool { return false }
175+
func (*dataFrame) isThrottled() bool { return false }
166176

167177
type incomingWindowUpdate struct {
178+
throttledItem
168179
streamID uint32
169180
increment uint32
170181
}
171182

172-
func (*incomingWindowUpdate) isTransportResponseFrame() bool { return false }
173-
174183
type outgoingWindowUpdate struct {
184+
throttledItem
175185
streamID uint32
176186
increment uint32
177187
}
178188

179-
func (*outgoingWindowUpdate) isTransportResponseFrame() bool {
180-
return false // window updates are throttled by thresholds
181-
}
182-
183189
type incomingSettings struct {
190+
throttledItem
184191
ss []http2.Setting
185192
}
186193

187-
func (*incomingSettings) isTransportResponseFrame() bool { return true } // Results in a settings ACK
188-
189194
type outgoingSettings struct {
195+
throttledItem
190196
ss []http2.Setting
191197
}
192198

193-
func (*outgoingSettings) isTransportResponseFrame() bool { return false }
194-
195199
type incomingGoAway struct {
200+
throttledItem
196201
}
197202

198-
func (*incomingGoAway) isTransportResponseFrame() bool { return false }
199-
200203
type goAway struct {
204+
throttledItem
201205
code http2.ErrCode
202206
debugData []byte
203207
headsUp bool
204208
closeConn error // if set, loopyWriter will exit with this error
205209
}
206210

207-
func (*goAway) isTransportResponseFrame() bool { return false }
208-
209211
type ping struct {
212+
throttledItem
210213
ack bool
211214
data [8]byte
212215
}
213216

214-
func (*ping) isTransportResponseFrame() bool { return true }
215-
216217
type outFlowControlSizeRequest struct {
218+
throttledItem
217219
resp chan uint32
218220
}
219221

220-
func (*outFlowControlSizeRequest) isTransportResponseFrame() bool { return false }
221-
222222
// closeConnection is an instruction to tell the loopy writer to flush the
223223
// framer and exit, which will cause the transport's connection to be closed
224224
// (by the client or server). The transport itself will close after the reader
225225
// encounters the EOF caused by the connection closure.
226-
type closeConnection struct{}
227-
228-
func (closeConnection) isTransportResponseFrame() bool { return false }
226+
type closeConnection struct {
227+
throttledItem
228+
}
229229

230230
type outStreamState int
231231

@@ -379,9 +379,9 @@ func (c *controlBuffer) executeAndPut(f func() bool, it cbItem) (bool, error) {
379379
c.consumerWaiting = false
380380
}
381381
c.list.enqueue(it)
382-
if it.isTransportResponseFrame() {
382+
if it.isThrottled() {
383383
c.transportResponseFrames++
384-
if c.transportResponseFrames == maxQueuedTransportResponseFrames {
384+
if c.transportResponseFrames == maxQueuedControlBufferItems {
385385
// We are adding the frame that puts us over the threshold; create
386386
// a throttling channel.
387387
ch := make(chan struct{})
@@ -436,8 +436,8 @@ func (c *controlBuffer) getOnceLocked() (any, error) {
436436
return nil, nil
437437
}
438438
h := c.list.dequeue().(cbItem)
439-
if h.isTransportResponseFrame() {
440-
if c.transportResponseFrames == maxQueuedTransportResponseFrames {
439+
if h.isThrottled() {
440+
if c.transportResponseFrames == maxQueuedControlBufferItems {
441441
// We are removing the frame that put us over the
442442
// threshold; close and clear the throttling channel.
443443
ch := c.trfChan.Swap(nil)
@@ -464,10 +464,8 @@ func (c *controlBuffer) finish() {
464464
// is still not aware of these yet.
465465
for head := c.list.dequeueAll(); head != nil; head = head.next {
466466
switch v := head.it.(type) {
467-
case *headerFrame:
468-
if v.onOrphaned != nil { // It will be nil on the server-side.
469-
v.onOrphaned(ErrConnClosing)
470-
}
467+
case *clientHeaders:
468+
v.onOrphaned(ErrConnClosing)
471469
case *dataFrame:
472470
if !v.processing {
473471
v.data.Free()
@@ -680,42 +678,38 @@ func (l *loopyWriter) registerStreamHandler(h *registerStream) {
680678
l.estdStreams[h.streamID] = str
681679
}
682680

683-
func (l *loopyWriter) headerHandler(h *headerFrame) error {
684-
if l.side == serverSide {
685-
str, ok := l.estdStreams[h.streamID]
686-
if !ok {
687-
if l.logger.V(logLevel) {
688-
l.logger.Infof("Unrecognized streamID %d in loopyWriter", h.streamID)
689-
}
690-
return nil
691-
}
692-
// Case 1.A: Server is responding back with headers.
693-
if !h.endStream {
694-
return l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite)
681+
func (l *loopyWriter) serverHeaderHandler(hdr *serverHeaders) error {
682+
str, ok := l.estdStreams[hdr.streamID]
683+
if !ok {
684+
if l.logger.V(logLevel) {
685+
l.logger.Infof("Unrecognized streamID %d in loopyWriter", hdr.streamID)
695686
}
696-
// else: Case 1.B: Server wants to close stream.
687+
return nil
688+
}
697689

698-
if str.state != empty { // either active or waiting on stream quota.
699-
// add it str's list of items.
700-
str.itl.enqueue(h)
701-
return nil
702-
}
703-
if err := l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite); err != nil {
704-
return err
705-
}
706-
return l.cleanupStreamHandler(h.cleanup)
690+
// Case 1: Server is responding back with headers.
691+
if !hdr.endStream {
692+
return l.writeHeader(hdr.streamID, hdr.endStream, hdr.hf, hdr.onWrite)
693+
}
694+
695+
// Case 2: Server is closing the stream.
696+
if str.state != empty { // either active or waiting on stream quota.
697+
str.itl.enqueue(hdr)
698+
return nil
699+
}
700+
if err := l.writeHeader(hdr.streamID, hdr.endStream, hdr.hf, hdr.onWrite); err != nil {
701+
return err
707702
}
708-
// Case 2: Client wants to originate stream.
703+
return l.cleanupStreamHandler(hdr.cleanup)
704+
}
705+
706+
func (l *loopyWriter) clientHeaderHandler(hdr *clientHeaders) error {
709707
str := &outStream{
710-
id: h.streamID,
708+
id: hdr.streamID,
711709
state: empty,
712710
itl: &itemList{},
713-
wq: h.wq,
711+
wq: hdr.wq,
714712
}
715-
return l.originateStream(str, h)
716-
}
717-
718-
func (l *loopyWriter) originateStream(str *outStream, hdr *headerFrame) error {
719713
// l.draining is set when handling GoAway. In which case, we want to avoid
720714
// creating new streams.
721715
if l.draining {
@@ -726,7 +720,7 @@ func (l *loopyWriter) originateStream(str *outStream, hdr *headerFrame) error {
726720
if err := hdr.initStream(str.id); err != nil {
727721
return err
728722
}
729-
if err := l.writeHeader(str.id, hdr.endStream, hdr.hf, hdr.onWrite); err != nil {
723+
if err := l.writeHeader(str.id, false, hdr.hf, hdr.onWrite); err != nil {
730724
return err
731725
}
732726
l.estdStreams[str.id] = str
@@ -882,8 +876,10 @@ func (l *loopyWriter) handle(i any) error {
882876
return l.incomingSettingsHandler(i)
883877
case *outgoingSettings:
884878
return l.outgoingSettingsHandler(i)
885-
case *headerFrame:
886-
return l.headerHandler(i)
879+
case *clientHeaders:
880+
return l.clientHeaderHandler(i)
881+
case *serverHeaders:
882+
return l.serverHeaderHandler(i)
887883
case *registerStream:
888884
l.registerStreamHandler(i)
889885
case *cleanupStream:
@@ -1022,7 +1018,7 @@ func (l *loopyWriter) processData() (bool, error) {
10221018
func (l *loopyWriter) updateStreamAfterWrite(str *outStream) error {
10231019
if str.itl.isEmpty() {
10241020
str.state = empty
1025-
} else if trailer, ok := str.itl.peek().(*headerFrame); ok { // the next item is trailers.
1021+
} else if trailer, ok := str.itl.peek().(*serverHeaders); ok { // the next item is trailers.
10261022
if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil {
10271023
return err
10281024
}

0 commit comments

Comments
 (0)