Skip to content

Commit e6c4db0

Browse files
bborehambboreham
authored
Fix auto scaling - don't set inactive write scaling on active tables (#750)
* Refactor TableManager test To allow multiple tests to use the same helper function * Add auto-scaling values to TableManager test * Add test with auto-scaling of inactive tables only * Don't set inactive write scaling on active tables * fix typo in log message
1 parent a181b71 commit e6c4db0

File tree

3 files changed

+132
-40
lines changed

3 files changed

+132
-40
lines changed

‎pkg/chunk/schema_config.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -241,11 +241,6 @@ func (cfg *PeriodicTableConfig) periodicTables(beginGrace, endGrace time.Duratio
241241
Tags: cfg.GetTags(),
242242
}
243243

244-
// Autoscale last N tables (excluding lastTable which is active).
245-
if cfg.InactiveWriteScale.Enabled && i >= (lastTable-cfg.InactiveWriteScaleLastN) && i < lastTable {
246-
table.WriteScale = cfg.InactiveWriteScale
247-
}
248-
249244
// if now is within table [start - grace, end + grace), then we need some write throughput
250245
if (i*periodSecs)-beginGraceSecs <= now && now < (i*periodSecs)+periodSecs+endGraceSecs {
251246
table.ProvisionedRead = cfg.ProvisionedReadThroughput
@@ -254,7 +249,11 @@ func (cfg *PeriodicTableConfig) periodicTables(beginGrace, endGrace time.Duratio
254249
if cfg.WriteScale.Enabled {
255250
table.WriteScale = cfg.WriteScale
256251
}
252+
} else if cfg.InactiveWriteScale.Enabled && i >= (lastTable-cfg.InactiveWriteScaleLastN) {
253+
// Autoscale last N tables
254+
table.WriteScale = cfg.InactiveWriteScale
257255
}
256+
258257
result = append(result, table)
259258
}
260259
return result

‎pkg/chunk/table_manager.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ func (m *TableManager) updateTables(ctx context.Context, descriptions []TableDes
292292
continue
293293
}
294294

295-
level.Info(util.Logger).Log("msg", "updating provisioned throughput on table", "table", expected.Name, "old_read", current.ProvisionedRead, "old_write", current.ProvisionedWrite, "new_read", expected.ProvisionedRead, "old_read", expected.ProvisionedWrite)
295+
level.Info(util.Logger).Log("msg", "updating provisioned throughput on table", "table", expected.Name, "old_read", current.ProvisionedRead, "old_write", current.ProvisionedWrite, "new_read", expected.ProvisionedRead, "new_write", expected.ProvisionedWrite)
296296
err = m.client.UpdateTable(ctx, current, expected)
297297
if err != nil {
298298
return err

‎pkg/chunk/table_manager_test.go‎

Lines changed: 127 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ const (
2424
inactiveRead = 2
2525
write = 200
2626
read = 100
27+
autoScaleLastN = 2
28+
autoScaleMin = 50
29+
autoScaleMax = 500
30+
autoScaleTarget = 80
2731
)
2832

2933
type mockTableClient struct {
@@ -71,6 +75,31 @@ func (m *mockTableClient) UpdateTable(_ context.Context, current, expected Table
7175
return nil
7276
}
7377

78+
func tmTest(t *testing.T, client *mockTableClient, tableManager *TableManager, name string, tm time.Time, expected []TableDesc) {
79+
t.Run(name, func(t *testing.T) {
80+
ctx := context.Background()
81+
mtime.NowForce(tm)
82+
if err := tableManager.SyncTables(ctx); err != nil {
83+
t.Fatal(err)
84+
}
85+
err := ExpectTables(ctx, client, expected)
86+
require.NoError(t, err)
87+
})
88+
}
89+
90+
var activeScalingConfig = AutoScalingConfig{
91+
Enabled: true,
92+
MinCapacity: autoScaleMin * 2,
93+
MaxCapacity: autoScaleMax * 2,
94+
TargetValue: autoScaleTarget,
95+
}
96+
var inactiveScalingConfig = AutoScalingConfig{
97+
Enabled: true,
98+
MinCapacity: autoScaleMin,
99+
MaxCapacity: autoScaleMax,
100+
TargetValue: autoScaleTarget,
101+
}
102+
74103
func TestTableManager(t *testing.T) {
75104
client := newMockTableClient()
76105

@@ -84,6 +113,9 @@ func TestTableManager(t *testing.T) {
84113
ProvisionedReadThroughput: read,
85114
InactiveWriteThroughput: inactiveWrite,
86115
InactiveReadThroughput: inactiveRead,
116+
WriteScale: activeScalingConfig,
117+
InactiveWriteScale: inactiveScalingConfig,
118+
InactiveWriteScaleLastN: autoScaleLastN,
87119
},
88120

89121
ChunkTables: PeriodicTableConfig{
@@ -103,108 +135,169 @@ func TestTableManager(t *testing.T) {
103135
t.Fatal(err)
104136
}
105137

106-
test := func(name string, tm time.Time, expected []TableDesc) {
107-
t.Run(name, func(t *testing.T) {
108-
ctx := context.Background()
109-
mtime.NowForce(tm)
110-
if err := tableManager.SyncTables(ctx); err != nil {
111-
t.Fatal(err)
112-
}
113-
err := ExpectTables(ctx, client, expected)
114-
require.NoError(t, err)
115-
})
116-
}
117-
118138
// Check at time zero, we have the base table and one weekly table
119-
test(
139+
tmTest(t, client, tableManager,
120140
"Initial test",
121141
time.Unix(0, 0),
122142
[]TableDesc{
123-
{Name: "", ProvisionedRead: read, ProvisionedWrite: write},
124-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
143+
{Name: "", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
144+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
125145
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
126146
},
127147
)
128148

129149
// Check running twice doesn't change anything
130-
test(
150+
tmTest(t, client, tableManager,
131151
"Nothing changed",
132152
time.Unix(0, 0),
133153
[]TableDesc{
134-
{Name: "", ProvisionedRead: read, ProvisionedWrite: write},
135-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
154+
{Name: "", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
155+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
136156
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
137157
},
138158
)
139159

140160
// Fast forward grace period, check we still have write throughput on base table
141-
test(
161+
tmTest(t, client, tableManager,
142162
"Move forward by grace period",
143163
time.Unix(0, 0).Add(gracePeriod),
144164
[]TableDesc{
145-
{Name: "", ProvisionedRead: read, ProvisionedWrite: write},
146-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
165+
{Name: "", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
166+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
147167
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
148168
},
149169
)
150170

151171
// Fast forward max chunk age + grace period, check write throughput on base table has gone
152-
test(
172+
// (and we don't put inactive auto-scaling on base table)
173+
tmTest(t, client, tableManager,
153174
"Move forward by max chunk age + grace period",
154175
time.Unix(0, 0).Add(maxChunkAge).Add(gracePeriod),
155176
[]TableDesc{
156177
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
157-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
178+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
158179
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
159180
},
160181
)
161182

162183
// Fast forward table period - grace period, check we add another weekly table
163-
test(
184+
tmTest(t, client, tableManager,
164185
"Move forward by table period - grace period",
165186
time.Unix(0, 0).Add(tablePeriod).Add(-gracePeriod),
166187
[]TableDesc{
167188
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
168-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
169-
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
189+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
190+
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
170191
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
171192
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
172193
},
173194
)
174195

175196
// Fast forward table period + grace period, check we still have provisioned throughput
176-
test(
197+
tmTest(t, client, tableManager,
177198
"Move forward by table period + grace period",
178199
time.Unix(0, 0).Add(tablePeriod).Add(gracePeriod),
179200
[]TableDesc{
180201
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
181-
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
182-
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
202+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
203+
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
183204
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
184205
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
185206
},
186207
)
187208

188209
// Fast forward table period + max chunk age + grace period, check we remove provisioned throughput
189-
test(
210+
tmTest(t, client, tableManager,
190211
"Move forward by table period + max chunk age + grace period",
191212
time.Unix(0, 0).Add(tablePeriod).Add(maxChunkAge).Add(gracePeriod),
192213
[]TableDesc{
193214
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
194-
{Name: tablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
195-
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
215+
{Name: tablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite, WriteScale: inactiveScalingConfig},
216+
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
196217
{Name: chunkTablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
197218
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
198219
},
199220
)
200221

201222
// Check running twice doesn't change anything
202-
test(
223+
tmTest(t, client, tableManager,
203224
"Nothing changed",
204225
time.Unix(0, 0).Add(tablePeriod).Add(maxChunkAge).Add(gracePeriod),
205226
[]TableDesc{
206227
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
207-
{Name: tablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
228+
{Name: tablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite, WriteScale: inactiveScalingConfig},
229+
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write, WriteScale: activeScalingConfig},
230+
{Name: chunkTablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
231+
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
232+
},
233+
)
234+
}
235+
236+
func TestTableManagerAutoscaleInactiveOnly(t *testing.T) {
237+
client := newMockTableClient()
238+
239+
cfg := SchemaConfig{
240+
UsePeriodicTables: true,
241+
IndexTables: PeriodicTableConfig{
242+
Prefix: tablePrefix,
243+
Period: tablePeriod,
244+
From: util.NewDayValue(model.TimeFromUnix(0)),
245+
ProvisionedWriteThroughput: write,
246+
ProvisionedReadThroughput: read,
247+
InactiveWriteThroughput: inactiveWrite,
248+
InactiveReadThroughput: inactiveRead,
249+
InactiveWriteScale: inactiveScalingConfig,
250+
InactiveWriteScaleLastN: autoScaleLastN,
251+
},
252+
253+
ChunkTables: PeriodicTableConfig{
254+
Prefix: chunkTablePrefix,
255+
Period: tablePeriod,
256+
From: util.NewDayValue(model.TimeFromUnix(0)),
257+
ProvisionedWriteThroughput: write,
258+
ProvisionedReadThroughput: read,
259+
InactiveWriteThroughput: inactiveWrite,
260+
InactiveReadThroughput: inactiveRead,
261+
},
262+
263+
CreationGracePeriod: gracePeriod,
264+
}
265+
tableManager, err := NewTableManager(cfg, maxChunkAge, client)
266+
if err != nil {
267+
t.Fatal(err)
268+
}
269+
270+
// Check at time zero, we have the base table and one weekly table
271+
tmTest(t, client, tableManager,
272+
"Initial test",
273+
time.Unix(0, 0),
274+
[]TableDesc{
275+
{Name: "", ProvisionedRead: read, ProvisionedWrite: write},
276+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
277+
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
278+
},
279+
)
280+
281+
// Fast forward table period + grace period, check we still have provisioned throughput
282+
tmTest(t, client, tableManager,
283+
"Move forward by table period + grace period",
284+
time.Unix(0, 0).Add(tablePeriod).Add(gracePeriod),
285+
[]TableDesc{
286+
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
287+
{Name: tablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
288+
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
289+
{Name: chunkTablePrefix + "0", ProvisionedRead: read, ProvisionedWrite: write},
290+
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
291+
},
292+
)
293+
294+
// Fast forward table period + max chunk age + grace period, check we remove provisioned throughput
295+
tmTest(t, client, tableManager,
296+
"Move forward by table period + max chunk age + grace period",
297+
time.Unix(0, 0).Add(tablePeriod).Add(maxChunkAge).Add(gracePeriod),
298+
[]TableDesc{
299+
{Name: "", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
300+
{Name: tablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite, WriteScale: inactiveScalingConfig},
208301
{Name: tablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},
209302
{Name: chunkTablePrefix + "0", ProvisionedRead: inactiveRead, ProvisionedWrite: inactiveWrite},
210303
{Name: chunkTablePrefix + "1", ProvisionedRead: read, ProvisionedWrite: write},

0 commit comments

Comments
 (0)