Skip to content

Commit fef8616

Browse files
karencfvkarencfv
authored
[reconfigurator] Call clickhouse-admin API from SMF services (#6533)
## Overview This commit replaces the old replicated ClickHouse server and keeper configuration templates with calls to the `clickhouse-admin` API that generate said configuration files. ## Purpose While the end goal is to have Nexus make the API calls to generate the configuration files, we'd like to have a working implementation of the `clickhouse-admin` API via the SMF services. Using `curl` is not what the finished work will look like, but rather it is the simplest way to have a working implementation in the mean time. ## Testing Deployed this branch on a Helios machine with the following results Replica 1 ```console root@oxz_clickhouse_server_a9d02cd3:~# /opt/oxide/clickhouse_server/clickhouse client --host fd00:1122:3344:101::f ClickHouse client version 23.8.7.1. Connecting to fd00:1122:3344:101::f:9000 as user default. Connected to ClickHouse server version 23.8.7 revision 54465. oximeter_cluster_1 :) SHOW TABLES FROM oximeter SHOW TABLES FROM oximeter Query id: 06867649-f49e-451f-b9f1-5a574e12ce5b ┌─name─────────────────────────────┐ │ fields_bool │ │ fields_bool_local │ │ fields_i16 │ │ fields_i16_local │ │ fields_i32 │ │ fields_i32_local │ │ fields_i64 │ │ fields_i64_local │ │ fields_i8 │ │ fields_i8_local │ │ fields_ipaddr │ │ fields_ipaddr_local │ │ fields_string │ │ fields_string_local │ │ fields_u16 │ │ fields_u16_local │ │ fields_u32 │ │ fields_u32_local │ │ fields_u64 │ │ fields_u64_local │ │ fields_u8 │ │ fields_u8_local │ │ fields_uuid │ │ fields_uuid_local │ │ measurements_bool │ │ measurements_bool_local │ │ measurements_bytes │ │ measurements_bytes_local │ │ measurements_cumulativef32 │ │ measurements_cumulativef32_local │ │ measurements_cumulativef64 │ │ measurements_cumulativef64_local │ │ measurements_cumulativei64 │ │ measurements_cumulativei64_local │ │ measurements_cumulativeu64 │ │ measurements_cumulativeu64_local │ │ measurements_f32 │ │ measurements_f32_local │ │ measurements_f64 │ │ measurements_f64_local │ │ measurements_histogramf32 │ │ measurements_histogramf32_local │ │ measurements_histogramf64 │ │ measurements_histogramf64_local │ │ measurements_histogrami16 │ │ measurements_histogrami16_local │ │ measurements_histogrami32 │ │ measurements_histogrami32_local │ │ measurements_histogrami64 │ │ measurements_histogrami64_local │ │ measurements_histogrami8 │ │ measurements_histogrami8_local │ │ measurements_histogramu16 │ │ measurements_histogramu16_local │ │ measurements_histogramu32 │ │ measurements_histogramu32_local │ │ measurements_histogramu64 │ │ measurements_histogramu64_local │ │ measurements_histogramu8 │ │ measurements_histogramu8_local │ │ measurements_i16 │ │ measurements_i16_local │ │ measurements_i32 │ │ measurements_i32_local │ │ measurements_i64 │ │ measurements_i64_local │ │ measurements_i8 │ │ measurements_i8_local │ │ measurements_string │ │ measurements_string_local │ │ measurements_u16 │ │ measurements_u16_local │ │ measurements_u32 │ │ measurements_u32_local │ │ measurements_u64 │ │ measurements_u64_local │ │ measurements_u8 │ │ measurements_u8_local │ │ timeseries_schema │ │ timeseries_schema_local │ │ version │ └──────────────────────────────────┘ 81 rows in set. Elapsed: 0.005 sec. oximeter_cluster_1 :) SELECT * FROM oximeter.measurements_u64 SELECT * FROM oximeter.measurements_u64 Query id: 2e13d330-8f0b-4346-afc0-ba3c21ea7674 ┌─timeseries_name─────────────────────────┬───────timeseries_key─┬─────────────────────timestamp─┬─datum─┐ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:16:47.241835734 │ 0 │ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:16:48.241091831 │ 0 │ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:16:49.241294398 │ 0 │ <...> ``` Replica 2 ```console root@oxz_clickhouse_server_ba1601d3:~# /opt/oxide/clickhouse_server/clickhouse client --host fd00:1122:3344:101::e ClickHouse client version 23.8.7.1. Connecting to fd00:1122:3344:101::e:9000 as user default. Connected to ClickHouse server version 23.8.7 revision 54465. oximeter_cluster_2 :) SHOW TABLES FROM oximeter SHOW TABLES FROM oximeter Query id: 33dd1d4d-1596-44e3-90ea-c755a1e3ae24 ┌─name─────────────────────────────┐ │ fields_bool │ │ fields_bool_local │ │ fields_i16 │ │ fields_i16_local │ │ fields_i32 │ │ fields_i32_local │ │ fields_i64 │ │ fields_i64_local │ │ fields_i8 │ │ fields_i8_local │ │ fields_ipaddr │ │ fields_ipaddr_local │ │ fields_string │ │ fields_string_local │ │ fields_u16 │ │ fields_u16_local │ │ fields_u32 │ │ fields_u32_local │ │ fields_u64 │ │ fields_u64_local │ │ fields_u8 │ │ fields_u8_local │ │ fields_uuid │ │ fields_uuid_local │ │ measurements_bool │ │ measurements_bool_local │ │ measurements_bytes │ │ measurements_bytes_local │ │ measurements_cumulativef32 │ │ measurements_cumulativef32_local │ │ measurements_cumulativef64 │ │ measurements_cumulativef64_local │ │ measurements_cumulativei64 │ │ measurements_cumulativei64_local │ │ measurements_cumulativeu64 │ │ measurements_cumulativeu64_local │ │ measurements_f32 │ │ measurements_f32_local │ │ measurements_f64 │ │ measurements_f64_local │ │ measurements_histogramf32 │ │ measurements_histogramf32_local │ │ measurements_histogramf64 │ │ measurements_histogramf64_local │ │ measurements_histogrami16 │ │ measurements_histogrami16_local │ │ measurements_histogrami32 │ │ measurements_histogrami32_local │ │ measurements_histogrami64 │ │ measurements_histogrami64_local │ │ measurements_histogrami8 │ │ measurements_histogrami8_local │ │ measurements_histogramu16 │ │ measurements_histogramu16_local │ │ measurements_histogramu32 │ │ measurements_histogramu32_local │ │ measurements_histogramu64 │ │ measurements_histogramu64_local │ │ measurements_histogramu8 │ │ measurements_histogramu8_local │ │ measurements_i16 │ │ measurements_i16_local │ │ measurements_i32 │ │ measurements_i32_local │ │ measurements_i64 │ │ measurements_i64_local │ │ measurements_i8 │ │ measurements_i8_local │ │ measurements_string │ │ measurements_string_local │ │ measurements_u16 │ │ measurements_u16_local │ │ measurements_u32 │ │ measurements_u32_local │ │ measurements_u64 │ │ measurements_u64_local │ │ measurements_u8 │ │ measurements_u8_local │ │ timeseries_schema │ │ timeseries_schema_local │ │ version │ └──────────────────────────────────┘ 81 rows in set. Elapsed: 0.010 sec. oximeter_cluster_2 :) SELECT * FROM oximeter.measurements_u64 SELECT * FROM oximeter.measurements_u64 Query id: 06da0f16-3055-47cb-9984-94dc78f99afc ┌─timeseries_name─────────────────────────┬───────timeseries_key─┬─────────────────────timestamp─┬─datum─┐ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:22:02.443983562 │ 0 │ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:22:03.444346219 │ 0 │ │ ddm_router:originated_tunnel_endpoints │ 2085026407707057203 │ 2024-09-09 07:22:04.444356384 │ 0 │ <...> ``` Keeper 1 ```console root@oxz_clickhouse_keeper_8cb0de91:~# echo mntr | nc fd00:1122:3344:101::12 9181 zk_version v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64 zk_avg_latency 4 zk_max_latency 103 zk_min_latency 0 zk_packets_received 27769 zk_packets_sent 29290 zk_num_alive_connections 1 zk_outstanding_requests 0 zk_server_state leader zk_znode_count 6535 zk_watch_count 83 zk_ephemerals_count 82 zk_approximate_data_size 2330794 zk_key_arena_size 1044480 zk_latest_snapshot_size 0 zk_followers 2 zk_synced_followers 2 ``` Keeper 2 ```console root@oxz_clickhouse_keeper_a6c18bd2:~# echo mntr | nc fd00:1122:3344:101::10 9181 zk_version v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64 zk_avg_latency 10 zk_max_latency 139 zk_min_latency 0 zk_packets_received 22278 zk_packets_sent 23922 zk_num_alive_connections 1 zk_outstanding_requests 0 zk_server_state follower zk_znode_count 7015 zk_watch_count 83 zk_ephemerals_count 82 zk_approximate_data_size 2512980 zk_key_arena_size 1044480 zk_latest_snapshot_size 0 ``` Keeper 3 ```console root@oxz_clickhouse_keeper_45d3e6ef:~# echo mntr | nc fd00:1122:3344:101::11 9181 zk_version v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64 zk_avg_latency 0 zk_max_latency 0 zk_min_latency 0 zk_packets_received 0 zk_packets_sent 0 zk_num_alive_connections 0 zk_outstanding_requests 0 zk_server_state follower zk_znode_count 7188 zk_watch_count 0 zk_ephemerals_count 82 zk_approximate_data_size 2575631 zk_key_arena_size 1044480 zk_latest_snapshot_size 0 ``` Related: #5999 Closes: #3824
1 parent ba49693 commit fef8616

File tree

14 files changed

+125
-220
lines changed

14 files changed

+125
-220
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎clickhouse-admin/types/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ workspace = true
99

1010
[dependencies]
1111
anyhow.workspace = true
12+
atomicwrites.workspace = true
1213
camino.workspace = true
1314
camino-tempfile.workspace = true
1415
derive_more.workspace = true

‎clickhouse-admin/types/src/config.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ impl ReplicaConfig {
9797
let keepers = keepers.to_xml();
9898
let remote_servers = remote_servers.to_xml();
9999
let user_files_path = data_path.clone().join("user_files");
100+
let temp_files_path = data_path.clone().join("tmp");
100101
let format_schema_path = data_path.clone().join("format_schemas");
101102
format!(
102103
"
@@ -135,6 +136,7 @@ impl ReplicaConfig {
135136
</default>
136137
</quotas>
137138
139+
<tmp_path>{temp_files_path}</tmp_path>
138140
<user_files_path>{user_files_path}</user_files_path>
139141
<default_profile>default</default_profile>
140142
<format_schema_path>{format_schema_path}</format_schema_path>
@@ -472,6 +474,9 @@ pub struct KeeperConfig {
472474
pub coordination_settings: KeeperCoordinationSettings,
473475
/// Settings for each server in the keeper cluster
474476
pub raft_config: RaftServers,
477+
/// Directory for all files generated by ClickHouse itself
478+
#[schemars(schema_with = "path_schema")]
479+
pub datastore_path: Utf8PathBuf,
475480
}
476481

477482
impl KeeperConfig {
@@ -496,6 +501,7 @@ impl KeeperConfig {
496501
snapshot_storage_path,
497502
coordination_settings,
498503
raft_config,
504+
datastore_path,
499505
}
500506
}
501507

@@ -509,6 +515,7 @@ impl KeeperConfig {
509515
snapshot_storage_path,
510516
coordination_settings,
511517
raft_config,
518+
datastore_path,
512519
} = self;
513520
let logger = logger.to_xml();
514521
let KeeperCoordinationSettings {
@@ -522,6 +529,7 @@ impl KeeperConfig {
522529
<clickhouse>
523530
{logger}
524531
<listen_host>{listen_host}</listen_host>
532+
<path>{datastore_path}</path>
525533
<keeper_server>
526534
<enable_reconfiguration>false</enable_reconfiguration>
527535
<tcp_port>{tcp_port}</tcp_port>

‎clickhouse-admin/types/src/lib.rs

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@
22
// License, v. 2.0. If a copy of the MPL was not distributed with this
33
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
44

5-
use anyhow::Result;
5+
use anyhow::{Context, Result};
6+
use atomicwrites::AtomicFile;
67
use camino::Utf8PathBuf;
7-
use camino_tempfile::NamedUtf8TempFile;
88
use derive_more::{Add, AddAssign, Display, From};
99
use schemars::JsonSchema;
1010
use serde::{Deserialize, Serialize};
11-
use std::fs::rename;
12-
use std::io::Write;
11+
use std::fs::create_dir;
12+
use std::io::{ErrorKind, Write};
1313
use std::net::Ipv6Addr;
1414

1515
pub mod config;
@@ -121,12 +121,20 @@ impl ServerSettings {
121121
self.datastore_path.clone(),
122122
);
123123

124-
// Writing to a temporary file and then renaming it will ensure we
125-
// don't end up with a partially written file after a crash
126-
let mut f = NamedUtf8TempFile::new()?;
127-
f.write_all(config.to_xml().as_bytes())?;
128-
f.flush()?;
129-
rename(f.path(), self.config_dir.join("replica-server-config.xml"))?;
124+
match create_dir(self.config_dir.clone()) {
125+
Ok(_) => (),
126+
Err(e) if e.kind() == ErrorKind::AlreadyExists => (),
127+
Err(e) => return Err(e.into()),
128+
};
129+
130+
let path = self.config_dir.join("replica-server-config.xml");
131+
AtomicFile::new(
132+
path.clone(),
133+
atomicwrites::OverwriteBehavior::AllowOverwrite,
134+
)
135+
.write(|f| f.write_all(config.to_xml().as_bytes()))
136+
.with_context(|| format!("failed to write to `{}`", path))?;
137+
130138
Ok(config)
131139
}
132140
}
@@ -180,12 +188,20 @@ impl KeeperSettings {
180188
raft_config,
181189
);
182190

183-
// Writing to a temporary file and then renaming it will ensure we
184-
// don't end up with a partially written file after a crash
185-
let mut f = NamedUtf8TempFile::new()?;
186-
f.write_all(config.to_xml().as_bytes())?;
187-
f.flush()?;
188-
rename(f.path(), self.config_dir.join("keeper-config.xml"))?;
191+
match create_dir(self.config_dir.clone()) {
192+
Ok(_) => (),
193+
Err(e) if e.kind() == ErrorKind::AlreadyExists => (),
194+
Err(e) => return Err(e.into()),
195+
};
196+
197+
let path = self.config_dir.join("keeper_config.xml");
198+
AtomicFile::new(
199+
path.clone(),
200+
atomicwrites::OverwriteBehavior::AllowOverwrite,
201+
)
202+
.write(|f| f.write_all(config.to_xml().as_bytes()))
203+
.with_context(|| format!("failed to write to `{}`", path))?;
204+
189205
Ok(config)
190206
}
191207
}
@@ -244,9 +260,9 @@ mod tests {
244260

245261
let expected_file = Utf8PathBuf::from_str("./testutils")
246262
.unwrap()
247-
.join("keeper-config.xml");
263+
.join("keeper_config.xml");
248264
let generated_file =
249-
Utf8PathBuf::from(config_dir.path()).join("keeper-config.xml");
265+
Utf8PathBuf::from(config_dir.path()).join("keeper_config.xml");
250266
let generated_content = std::fs::read_to_string(generated_file)
251267
.expect("Failed to read from generated ClickHouse keeper file");
252268

‎clickhouse-admin/types/testutils/keeper-config.xml renamed to ‎clickhouse-admin/types/testutils/keeper_config.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
</logger>
1111

1212
<listen_host>ff::8</listen_host>
13+
<path>./</path>
1314
<keeper_server>
1415
<enable_reconfiguration>false</enable_reconfiguration>
1516
<tcp_port>9181</tcp_port>

‎clickhouse-admin/types/testutils/replica-server-config.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
</default>
4343
</quotas>
4444

45+
<tmp_path>./data/tmp</tmp_path>
4546
<user_files_path>./data/user_files</user_files_path>
4647
<default_profile>default</default_profile>
4748
<format_schema_path>./data/format_schemas</format_schema_path>

‎openapi/clickhouse-admin.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@
162162
}
163163
]
164164
},
165+
"datastore_path": {
166+
"description": "Directory for all files generated by ClickHouse itself",
167+
"type": "string",
168+
"format": "Utf8PathBuf"
169+
},
165170
"listen_host": {
166171
"description": "Address the keeper is listening on",
167172
"type": "string",
@@ -210,6 +215,7 @@
210215
},
211216
"required": [
212217
"coordination_settings",
218+
"datastore_path",
213219
"listen_host",
214220
"log_storage_path",
215221
"logger",

‎package-manifest.toml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,6 @@ source.paths = [
200200
{ from = "out/clickhouse", to = "/opt/oxide/clickhouse_server" },
201201
{ from = "smf/clickhouse_server/manifest.xml", to = "/var/svc/manifest/site/clickhouse_server/manifest.xml" },
202202
{ from = "smf/clickhouse_server/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse_server.sh" },
203-
{ from = "smf/clickhouse_server/config_replica.xml", to = "/opt/oxide/clickhouse_server/config.d/config_replica.xml" },
204203
]
205204
output.type = "zone"
206205
output.intermediate_only = true
@@ -230,7 +229,6 @@ source.paths = [
230229
{ from = "out/clickhouse", to = "/opt/oxide/clickhouse_keeper" },
231230
{ from = "smf/clickhouse_keeper/manifest.xml", to = "/var/svc/manifest/site/clickhouse_keeper/manifest.xml" },
232231
{ from = "smf/clickhouse_keeper/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse_keeper.sh" },
233-
{ from = "smf/clickhouse_keeper/keeper_config.xml", to = "/opt/oxide/clickhouse_keeper/keeper_config.xml" },
234232
]
235233
output.type = "zone"
236234
output.intermediate_only = true

‎smf/clickhouse_keeper/keeper_config.xml

Lines changed: 0 additions & 43 deletions
This file was deleted.

‎smf/clickhouse_keeper/manifest.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@
1616
<service_fmri value='svc:/oxide/zone-network-setup:default' />
1717
</dependency>
1818

19+
<dependency name='clickhouse-admin' grouping='require_all' restart_on='none'
20+
type='service'>
21+
<service_fmri value='svc:/oxide/clickhouse-admin:default' />
22+
</dependency>
23+
1924
<exec_method type='method' name='start'
2025
exec='/opt/oxide/lib/svc/manifest/clickhouse_keeper.sh'
2126
timeout_seconds='0' />

‎smf/clickhouse_keeper/method_script.sh

Lines changed: 31 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -70,40 +70,37 @@ else
7070
exit "$SMF_EXIT_ERR_CONFIG"
7171
fi
7272

73-
# Setting environment variables this way is best practice, but has the downside
74-
# of obscuring the field values to anyone ssh=ing into the zone. To mitigate
75-
# this, we will be saving them to ${DATASTORE}/config_env_vars
76-
export CH_LOG="${DATASTORE}/clickhouse-keeper.log"
77-
export CH_ERROR_LOG="${DATASTORE}/clickhouse-keeper.err.log"
78-
export CH_LISTEN_ADDR=${LISTEN_ADDR}
79-
export CH_DATASTORE=${DATASTORE}
80-
export CH_LISTEN_PORT=${LISTEN_PORT}
81-
export CH_KEEPER_ID_CURRENT=${KEEPER_ID_CURRENT}
82-
export CH_LOG_STORAGE_PATH="${DATASTORE}/log"
83-
export CH_SNAPSHOT_STORAGE_PATH="${DATASTORE}/snapshots"
84-
export CH_KEEPER_ID_01=${KEEPER_ID_01}
85-
export CH_KEEPER_ID_02=${KEEPER_ID_02}
86-
export CH_KEEPER_ID_03=${KEEPER_ID_03}
87-
export CH_KEEPER_HOST_01=${KEEPER_HOST_01}
88-
export CH_KEEPER_HOST_02=${KEEPER_HOST_02}
89-
export CH_KEEPER_HOST_03=${KEEPER_HOST_03}
90-
91-
content="CH_LOG="${CH_LOG}"\n\
92-
CH_ERROR_LOG="${CH_ERROR_LOG}"\n\
93-
CH_LISTEN_ADDR="${CH_LISTEN_ADDR}"\n\
94-
CH_DATASTORE="${CH_DATASTORE}"\n\
95-
CH_LISTEN_PORT="${CH_LISTEN_PORT}"\n\
96-
CH_KEEPER_ID_CURRENT="${CH_KEEPER_ID_CURRENT}"\n\
97-
CH_LOG_STORAGE_PATH="${CH_LOG_STORAGE_PATH}"\n\
98-
CH_SNAPSHOT_STORAGE_PATH="${CH_SNAPSHOT_STORAGE_PATH}"\n\
99-
CH_KEEPER_ID_01="${CH_KEEPER_ID_01}"\n\
100-
CH_KEEPER_ID_02="${CH_KEEPER_ID_02}"\n\
101-
CH_KEEPER_ID_03="${CH_KEEPER_ID_03}"\n\
102-
CH_KEEPER_HOST_01="${CH_KEEPER_HOST_01}"\n\
103-
CH_KEEPER_HOST_02="${CH_KEEPER_HOST_02}"\n\
104-
CH_KEEPER_HOST_03="${CH_KEEPER_HOST_03}""
105-
106-
echo $content >> "${DATASTORE}/config_env_vars"
73+
curl -X put http://[${LISTEN_ADDR}]:8888/keeper/config \
74+
-H "Content-Type: application/json" \
75+
-d '{
76+
"generation": 0,
77+
"settings": {
78+
"id": '${KEEPER_ID_CURRENT}',
79+
"raft_servers": [
80+
{
81+
"id": '${KEEPER_ID_01}',
82+
"host": {
83+
"domain_name": "'${KEEPER_HOST_01}'"
84+
}
85+
},
86+
{
87+
"id": '${KEEPER_ID_02}',
88+
"host": {
89+
"domain_name": "'${KEEPER_HOST_02}'"
90+
}
91+
},
92+
{
93+
"id": '${KEEPER_ID_03}',
94+
"host": {
95+
"domain_name": "'${KEEPER_HOST_03}'"
96+
}
97+
}
98+
],
99+
"config_dir": "/opt/oxide/clickhouse_keeper",
100+
"datastore_path": "'${DATASTORE}'",
101+
"listen_addr": "'${LISTEN_ADDR}'"
102+
}
103+
}'
107104

108105
# The clickhouse binary must be run from within the directory that contains it.
109106
# Otherwise, it does not automatically detect the configuration files, nor does

0 commit comments

Comments
 (0)