Skip to content

Commit 8ccd2e6

Browse files
geoffxygeoffxy
authored
Display VDBE state on the dashboard (#515)
This PR also includes other various improvements.
1 parent a0c617f commit 8ccd2e6

File tree

20 files changed

+653
-125
lines changed

20 files changed

+653
-125
lines changed

‎config/system_config_demo.yml‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# This file contains configurations that are used by BRAD. These are default
2+
# values and should be customized for specific situations.
3+
4+
# BRAD's front end servers will listen for client connections on this interface
5+
# and port. If `num_front_ends` is greater than one, subsequent front ends will
6+
# listen on successive ports (e.g., 6584, 6585, etc.).
7+
front_end_interface: "0.0.0.0"
8+
front_end_port: 6583
9+
num_front_ends: 1
10+
11+
# If installed and enabled, BRAD will serve its UI from a webserver that listens
12+
# for connections on this network interface and port.
13+
ui_interface: "0.0.0.0"
14+
ui_port: 7583
15+
16+
# Logging paths. If the value is in ALL_CAPS (with underscores), it is
17+
# interpreted as an environment variable (BRAD will log to the path stored in
18+
# the environment variable).
19+
20+
# Where BRAD's daemon process will write its logs.
21+
daemon_log_file: /tmp
22+
23+
# Where BRAD's front end processes will write their logs.
24+
front_end_log_path: /tmp
25+
26+
# Where BRAD's blueprint planner will write debug logs.
27+
planner_log_path: /tmp
28+
29+
# Where BRAD's metrics loggers will write their logs.
30+
metrics_log_path: /tmp
31+
32+
# Probability that each transactional query will be logged.
33+
txn_log_prob: 0.01
34+
35+
# Set to a non-zero value enable automatic data syncing. When this is set to 0,
36+
# automatic syncing is disabled.
37+
data_sync_period_seconds: 0
38+
39+
# BRAD's front end servers will report their metrics at regular intervals.
40+
front_end_metrics_reporting_period_seconds: 30
41+
front_end_query_latency_buffer_size: 100
42+
43+
# `default` means to use the policy encoded in the blueprint. Other values will
44+
# override the blueprint.
45+
routing_policy: default
46+
47+
# Whether to disable table movement for benchmark purposes (i.e., keep all
48+
# tables on all engines.)
49+
disable_table_movement: true
50+
51+
# Epoch length for metrics and forecasting. This is the granularity at which
52+
# metrics/forecasting will be performed.
53+
epoch_length:
54+
weeks: 0
55+
days: 0
56+
hours: 0
57+
minutes: 1
58+
59+
# Blueprint planning strategy.
60+
strategy: fp_query_based_beam
61+
62+
# Used to specify the period of time over which to use data for planning.
63+
# Currrently, this is a "look behind" window for the workload.
64+
planning_window:
65+
weeks: 0
66+
days: 0
67+
hours: 1
68+
minutes: 0
69+
70+
# Used to aggregate metrics collected in the planning window.
71+
metrics_agg:
72+
method: ewm # 'mean' is another option
73+
alpha: 0.86466472 # 1 - 1 / e^2
74+
75+
# Used during planning.
76+
reinterpret_second_as: 1
77+
78+
# The query distribution must change by at least this much for a new blueprint
79+
# to be accepted.
80+
query_dist_change_frac: 0.1
81+
82+
# The search bound for the provisioning.
83+
max_provisioning_multiplier: 2.5
84+
85+
# Flag options for blueprint planning.
86+
use_io_optimized_aurora: true
87+
use_recorded_routing_if_available: true
88+
ensure_tables_together_on_one_engine: true
89+
90+
# Loads used to prime the system when no information is available.
91+
aurora_initialize_load_fraction: 0.25
92+
redshift_initialize_load_fraction: 0.25
93+
94+
# BRAD will not reduce predicted load lower than these values. Raise these
95+
# values to be more conservative against mispredictions.
96+
aurora_min_load_removal_fraction: 0.8
97+
redshift_min_load_removal_fraction: 0.8
98+
99+
# Blueprint planning performance ceilings.
100+
query_latency_p90_ceiling_s: 30.0
101+
txn_latency_p90_ceiling_s: 0.030
102+
103+
# If set to true, BRAD will attempt to use the specified preset Redshift
104+
# clusters instead of resizing the main Redshift cluster.
105+
use_preset_redshift_clusters: false
106+
107+
# Used for ordering blueprints during planning.
108+
comparator:
109+
type: benefit_perf_ceiling # or `perf_ceiling`
110+
111+
benefit_horizon: # Only used by the `benefit_perf_ceiling` comparator
112+
weeks: 0
113+
days: 0
114+
hours: 1
115+
minutes: 0
116+
117+
penalty_threshold: 0.8 # Only used by the `benefit_perf_ceiling` comparator
118+
penalty_power: 8 # Only used by the `benefit_perf_ceiling` comparator
119+
120+
# Used for precomputed predictions.
121+
std_datasets:
122+
- name: regular
123+
path: workloads/IMDB_100GB/regular_test/
124+
- name: adhoc
125+
path: workloads/IMDB_100GB/adhoc_test/
126+
127+
bootstrap_vdbe_path: config/vdbe_demo/imdb_extended_vdbes.json
128+
129+
# Blueprint planning trigger configs.
130+
131+
triggers:
132+
enabled: false
133+
check_period_s: 90 # Triggers are checked every X seconds.
134+
check_period_offset_s: 360 # Wait 6 mins before starting.
135+
136+
# Triggers will not fire for at least this many minutes after a new blueprint
137+
# takes effect. Usually this should be greater than zero to give BRAD
138+
# sufficient time to observe the effect of the blueprint on the workload. BRAD
139+
# may wait longer to ensure metrics are also available for this many minutes.
140+
observe_new_blueprint_mins: 3
141+
142+
elapsed_time:
143+
disabled: true
144+
multiplier: 60 # Multiplier over `planning_window`.
145+
146+
redshift_cpu:
147+
lo: 15
148+
hi: 85
149+
sustained_epochs: 3
150+
151+
aurora_cpu:
152+
lo: 15
153+
hi: 85
154+
sustained_epochs: 3
155+
156+
variable_costs:
157+
disabled: true
158+
threshold: 1.0
159+
160+
query_latency_ceiling:
161+
ceiling_s: 30.0
162+
sustained_epochs: 3
163+
164+
txn_latency_ceiling:
165+
ceiling_s: 0.030
166+
sustained_epochs: 3
167+
168+
recent_change:
169+
delay_epochs: 5

‎config/vdbe_demo/imdb_extended_vdbes.json‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
{
2+
"schema_name": "imdb_extended",
3+
"engines": [
4+
{
5+
"name": "VDBE (T)",
6+
"max_staleness_ms": 0,
7+
"p90_latency_slo_ms": 30,
8+
"interface": "postgresql",
9+
"tables": [
10+
{
11+
"name": "theatres",
12+
"writable": true
13+
},
14+
{
15+
"name": "showings",
16+
"writable": true
17+
},
18+
{
19+
"name": "ticket_orders",
20+
"writable": true
21+
},
22+
{
23+
"name": "movie_info",
24+
"writable": true
25+
},
26+
{
27+
"name": "aka_title",
28+
"writable": true
29+
}
30+
],
31+
"mapped_to": "aurora"
32+
},
33+
{
34+
"name": "VDBE (A)",
35+
"max_staleness_ms": 3600000,
36+
"p90_latency_slo_ms": 30000,
37+
"interface": "postgresql",
38+
"tables": [
39+
{
40+
"name": "homes",
41+
"writable": false
42+
},
43+
{
44+
"name": "theatres",
45+
"writable": false
46+
},
47+
{
48+
"name": "showings",
49+
"writable": false
50+
},
51+
{
52+
"name": "ticket_orders",
53+
"writable": false
54+
},
55+
{
56+
"name": "aka_name",
57+
"writable": false
58+
},
59+
{
60+
"name": "aka_title",
61+
"writable": false
62+
},
63+
{
64+
"name": "cast_info",
65+
"writable": false
66+
},
67+
{
68+
"name": "char_name",
69+
"writable": false
70+
},
71+
{
72+
"name": "comp_cast_type",
73+
"writable": false
74+
},
75+
{
76+
"name": "company_name",
77+
"writable": false
78+
},
79+
{
80+
"name": "company_type",
81+
"writable": false
82+
},
83+
{
84+
"name": "complete_cast",
85+
"writable": false
86+
},
87+
{
88+
"name": "info_type",
89+
"writable": false
90+
},
91+
{
92+
"name": "keyword",
93+
"writable": false
94+
},
95+
{
96+
"name": "kind_type",
97+
"writable": false
98+
},
99+
{
100+
"name": "link_type",
101+
"writable": false
102+
},
103+
{
104+
"name": "movie_companies",
105+
"writable": false
106+
},
107+
{
108+
"name": "movie_info_idx",
109+
"writable": false
110+
},
111+
{
112+
"name": "movie_keyword",
113+
"writable": false
114+
},
115+
{
116+
"name": "movie_link",
117+
"writable": false
118+
},
119+
{
120+
"name": "name",
121+
"writable": false
122+
},
123+
{
124+
"name": "role_type",
125+
"writable": false
126+
},
127+
{
128+
"name": "title",
129+
"writable": false
130+
},
131+
{
132+
"name": "movie_info",
133+
"writable": false
134+
},
135+
{
136+
"name": "person_info",
137+
"writable": false
138+
}
139+
],
140+
"mapped_to": "redshift"
141+
}
142+
],
143+
"tables": [
144+
{
145+
"name": "homes"
146+
},
147+
{
148+
"name": "theatres"
149+
},
150+
{
151+
"name": "showings"
152+
},
153+
{
154+
"name": "ticket_orders"
155+
},
156+
{
157+
"name": "aka_name"
158+
},
159+
{
160+
"name": "aka_title"
161+
},
162+
{
163+
"name": "cast_info"
164+
},
165+
{
166+
"name": "char_name"
167+
},
168+
{
169+
"name": "comp_cast_type"
170+
},
171+
{
172+
"name": "company_name"
173+
},
174+
{
175+
"name": "company_type"
176+
},
177+
{
178+
"name": "complete_cast"
179+
},
180+
{
181+
"name": "info_type"
182+
},
183+
{
184+
"name": "keyword"
185+
},
186+
{
187+
"name": "kind_type"
188+
},
189+
{
190+
"name": "link_type"
191+
},
192+
{
193+
"name": "movie_companies"
194+
},
195+
{
196+
"name": "movie_info_idx"
197+
},
198+
{
199+
"name": "movie_keyword"
200+
},
201+
{
202+
"name": "movie_link"
203+
},
204+
{
205+
"name": "name"
206+
},
207+
{
208+
"name": "role_type"
209+
},
210+
{
211+
"name": "title"
212+
},
213+
{
214+
"name": "movie_info"
215+
},
216+
{
217+
"name": "person_info"
218+
}
219+
]
220+
}

0 commit comments

Comments
 (0)