Skip to content

Commit 6fa5b4a

Browse files
meiliang86meiliang86
authored
Add dynamic config for per instance visibility API rate limit and burst (#2585)
1 parent 5fb4f1c commit 6fa5b4a

File tree

5 files changed

+61
-39
lines changed

5 files changed

+61
-39
lines changed

‎common/dynamicconfig/constants.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,12 @@ const (
152152
FrontendMaxNamespaceBurstPerInstance = "frontend.namespaceBurst"
153153
// FrontendMaxNamespaceCountPerInstance is workflow namespace count limit per second
154154
FrontendMaxNamespaceCountPerInstance = "frontend.namespaceCount"
155+
// FrontendMaxNamespaceVisibilityRPSPerInstance is namespace rate limit per second for visibility APIs.
156+
// This config is EXPERIMENTAL and may be changed or removed in a later release.
157+
FrontendMaxNamespaceVisibilityRPSPerInstance = "frontend.namespaceRPS.visibility"
158+
// FrontendMaxNamespaceVisibilityBurstPerInstance is namespace burst limit for visibility APIs.
159+
// This config is EXPERIMENTAL and may be changed or removed in a later release.
160+
FrontendMaxNamespaceVisibilityBurstPerInstance = "frontend.namespaceBurst.visibility"
155161
// FrontendGlobalNamespaceRPS is workflow namespace rate limit per second for the whole cluster
156162
FrontendGlobalNamespaceRPS = "frontend.globalNamespacerps"
157163
// FrontendThrottledLogRPS is the rate limit on number of log messages emitted per second for throttled logger

‎host/archival_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ import (
5151

5252
const (
5353
retryLimit = 20
54-
retryBackoffTime = 200 * time.Millisecond
54+
retryBackoffTime = 500 * time.Millisecond
5555
)
5656

5757
func (s *integrationSuite) TestArchival_TimerQueueProcessor() {

‎service/frontend/configs/quotas.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,13 +141,15 @@ func (c *NamespaceRateBurstImpl) Burst() int {
141141
}
142142

143143
func NewRequestToRateLimiter(
144-
rateBurstFn quotas.RateBurst,
144+
executionRateBurstFn quotas.RateBurst,
145+
visibilityRateBurstFn quotas.RateBurst,
146+
otherRateBurstFn quotas.RateBurst,
145147
) quotas.RequestRateLimiter {
146148
mapping := make(map[string]quotas.RequestRateLimiter)
147149

148-
executionRateLimiter := NewExecutionPriorityRateLimiter(rateBurstFn)
149-
visibilityRateLimiter := NewVisibilityPriorityRateLimiter(rateBurstFn)
150-
otherRateLimiter := NewOtherAPIPriorityRateLimiter(rateBurstFn)
150+
executionRateLimiter := NewExecutionPriorityRateLimiter(executionRateBurstFn)
151+
visibilityRateLimiter := NewVisibilityPriorityRateLimiter(visibilityRateBurstFn)
152+
otherRateLimiter := NewOtherAPIPriorityRateLimiter(otherRateBurstFn)
151153

152154
for api := range ExecutionAPIToPriority {
153155
mapping[api] = executionRateLimiter

‎service/frontend/fx.go‎

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -234,11 +234,12 @@ func TelemetryInterceptorProvider(
234234
func RateLimitInterceptorProvider(
235235
serviceConfig *Config,
236236
) *interceptor.RateLimitInterceptor {
237+
rateFn := func() float64 { return float64(serviceConfig.RPS()) }
237238
return interceptor.NewRateLimitInterceptor(
238239
configs.NewRequestToRateLimiter(
239-
quotas.NewDefaultIncomingRateLimiter(
240-
func() float64 { return float64(serviceConfig.RPS()) },
241-
),
240+
quotas.NewDefaultIncomingRateLimiter(rateFn),
241+
quotas.NewDefaultIncomingRateLimiter(rateFn),
242+
quotas.NewDefaultIncomingRateLimiter(rateFn),
242243
),
243244
map[string]int{},
244245
)
@@ -249,25 +250,33 @@ func NamespaceRateLimitInterceptorProvider(
249250
namespaceRegistry namespace.Registry,
250251
frontendServiceResolver membership.ServiceResolver,
251252
) *interceptor.NamespaceRateLimitInterceptor {
252-
return interceptor.NewNamespaceRateLimitInterceptor(
253-
namespaceRegistry,
254-
quotas.NewNamespaceRateLimiter(
255-
func(req quotas.Request) quotas.RequestRateLimiter {
256-
return configs.NewRequestToRateLimiter(configs.NewNamespaceRateBurst(
257-
req.Caller,
258-
func(namespace string) float64 {
259-
return namespaceRPS(
260-
serviceConfig,
261-
frontendServiceResolver,
262-
namespace,
263-
)
264-
},
265-
serviceConfig.MaxNamespaceBurstPerInstance,
266-
))
267-
},
268-
),
269-
map[string]int{},
253+
rateFn := func(namespace string) float64 {
254+
return namespaceRPS(
255+
serviceConfig.MaxNamespaceRPSPerInstance,
256+
serviceConfig.GlobalNamespaceRPS,
257+
frontendServiceResolver,
258+
namespace,
259+
)
260+
}
261+
262+
visibilityRateFn := func(namespace string) float64 {
263+
return namespaceRPS(
264+
serviceConfig.MaxNamespaceVisibilityRPSPerInstance,
265+
serviceConfig.GlobalNamespaceRPS,
266+
frontendServiceResolver,
267+
namespace,
268+
)
269+
}
270+
namespaceRateLimiter := quotas.NewNamespaceRateLimiter(
271+
func(req quotas.Request) quotas.RequestRateLimiter {
272+
return configs.NewRequestToRateLimiter(
273+
configs.NewNamespaceRateBurst(req.Caller, rateFn, serviceConfig.MaxNamespaceBurstPerInstance),
274+
configs.NewNamespaceRateBurst(req.Caller, visibilityRateFn, serviceConfig.MaxNamespaceVisibilityBurstPerInstance),
275+
configs.NewNamespaceRateBurst(req.Caller, rateFn, serviceConfig.MaxNamespaceBurstPerInstance),
276+
)
277+
},
270278
)
279+
return interceptor.NewNamespaceRateLimitInterceptor(namespaceRegistry, namespaceRateLimiter, map[string]int{})
271280
}
272281

273282
func NamespaceCountLimitInterceptorProvider(

‎service/frontend/service.go‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,16 +67,18 @@ type Config struct {
6767
EnableReadFromSecondaryAdvancedVisibility dynamicconfig.BoolPropertyFnWithNamespaceFilter
6868
ESIndexMaxResultWindow dynamicconfig.IntPropertyFn
6969

70-
HistoryMaxPageSize dynamicconfig.IntPropertyFnWithNamespaceFilter
71-
RPS dynamicconfig.IntPropertyFn
72-
MaxNamespaceRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
73-
MaxNamespaceBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
74-
MaxNamespaceCountPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
75-
GlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
76-
MaxIDLengthLimit dynamicconfig.IntPropertyFn
77-
EnableClientVersionCheck dynamicconfig.BoolPropertyFn
78-
DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter
79-
ShutdownDrainDuration dynamicconfig.DurationPropertyFn
70+
HistoryMaxPageSize dynamicconfig.IntPropertyFnWithNamespaceFilter
71+
RPS dynamicconfig.IntPropertyFn
72+
MaxNamespaceRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
73+
MaxNamespaceBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
74+
MaxNamespaceCountPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
75+
MaxNamespaceVisibilityRPSPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
76+
MaxNamespaceVisibilityBurstPerInstance dynamicconfig.IntPropertyFnWithNamespaceFilter
77+
GlobalNamespaceRPS dynamicconfig.IntPropertyFnWithNamespaceFilter
78+
MaxIDLengthLimit dynamicconfig.IntPropertyFn
79+
EnableClientVersionCheck dynamicconfig.BoolPropertyFn
80+
DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter
81+
ShutdownDrainDuration dynamicconfig.DurationPropertyFn
8082

8183
MaxBadBinaries dynamicconfig.IntPropertyFnWithNamespaceFilter
8284

@@ -153,6 +155,8 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, esIndexName
153155
MaxNamespaceRPSPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceRPSPerInstance, 2400),
154156
MaxNamespaceBurstPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceBurstPerInstance, 4800),
155157
MaxNamespaceCountPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceCountPerInstance, 1200),
158+
MaxNamespaceVisibilityRPSPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance, 10),
159+
MaxNamespaceVisibilityBurstPerInstance: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxNamespaceVisibilityBurstPerInstance, 10),
156160
GlobalNamespaceRPS: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendGlobalNamespaceRPS, 0),
157161
MaxIDLengthLimit: dc.GetIntProperty(dynamicconfig.MaxIDLengthLimit, 1000),
158162
MaxBadBinaries: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.FrontendMaxBadBinaries, namespace.MaxBadBinaries),
@@ -310,12 +314,13 @@ func (s *Service) Stop() {
310314
}
311315

312316
func namespaceRPS(
313-
config *Config,
317+
perInstanceRPSFn dynamicconfig.IntPropertyFnWithNamespaceFilter,
318+
globalRPSFn dynamicconfig.IntPropertyFnWithNamespaceFilter,
314319
frontendResolver membership.ServiceResolver,
315320
namespace string,
316321
) float64 {
317-
hostRPS := float64(config.MaxNamespaceRPSPerInstance(namespace))
318-
globalRPS := float64(config.GlobalNamespaceRPS(namespace))
322+
hostRPS := float64(perInstanceRPSFn(namespace))
323+
globalRPS := float64(globalRPSFn(namespace))
319324
hosts := float64(numFrontendHosts(frontendResolver))
320325

321326
rps := hostRPS + globalRPS*math.Exp((1.0-hosts)/8.0)

0 commit comments

Comments
 (0)