Merge pull request #3368 from drmingdrmer/master

[metasrv] refactor: simplify metasrv cluster starting up in tests.

Author: drmingdrmer

common/meta/api/src/meta_api_test_suite.rs

@@ -454,7 +454,7 @@ impl MetaApiTestSuite {
 // leader-follower tests
 // This is meant for testing distributed MetaApi impl, to ensure a read-after-write consistency.
 impl MetaApiTestSuite {
-    pub async fn database_create_get_drop_leader_follower<MT: MetaApi>(
+    pub async fn database_get_leader_follower<MT: MetaApi>(
         &self,
         leader: &MT,
         follower: &MT,
@@ -495,8 +495,6 @@ impl MetaApiTestSuite {
             assert_eq!("Code: 3, displayText = nonexistent.", format!("{}", err));
         }
 
-        // TODO(xp): test drop is replicated to follower
-
         Ok(())
     }
 

metasrv/tests/it/api/http/cluster_state_test.rs

@@ -36,7 +36,7 @@ use poem::Route;
 use pretty_assertions::assert_eq;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 use crate::tests::tls_constants::TEST_CA_CERT;
 use crate::tests::tls_constants::TEST_CN_NAME;
 use crate::tests::tls_constants::TEST_SERVER_CERT;
@@ -47,8 +47,8 @@ async fn test_cluster_nodes() -> common_exception::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
+    let tc0 = MetaSrvTestContext::new(0);
+    let mut tc1 = MetaSrvTestContext::new(1);
 
     tc1.config.raft_config.single = false;
     tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];
@@ -82,8 +82,8 @@ async fn test_cluster_state() -> common_exception::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
+    let tc0 = MetaSrvTestContext::new(0);
+    let mut tc1 = MetaSrvTestContext::new(1);
 
     tc1.config.raft_config.single = false;
     tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];
@@ -124,8 +124,8 @@ async fn test_http_service_cluster_state() -> common_exception::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
+    let tc0 = MetaSrvTestContext::new(0);
+    let mut tc1 = MetaSrvTestContext::new(1);
 
     tc1.config.raft_config.single = false;
     tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];

metasrv/tests/it/api/http_service.rs

@@ -23,7 +23,7 @@ use databend_meta::configs::Config;
 use databend_meta::meta_service::MetaNode;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 use crate::tests::tls_constants::TEST_CA_CERT;
 use crate::tests::tls_constants::TEST_CN_NAME;
 use crate::tests::tls_constants::TEST_SERVER_CERT;
@@ -41,7 +41,7 @@ async fn test_http_service_tls_server() -> Result<()> {
     conf.admin_tls_server_key = TEST_SERVER_KEY.to_owned();
     conf.admin_tls_server_cert = TEST_SERVER_CERT.to_owned();
     conf.admin_api_address = addr_str.to_owned();
-    let tc = new_metasrv_test_context(0);
+    let tc = MetaSrvTestContext::new(0);
     let meta_node = MetaNode::start(&tc.config.raft_config).await?;
 
     let mut srv = HttpService::create(conf, meta_node);

metasrv/tests/it/flight/metasrv_flight_api.rs

@@ -28,7 +28,7 @@ use pretty_assertions::assert_eq;
 use tokio::time::Duration;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 use crate::tests::start_metasrv_with_context;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -136,8 +136,8 @@ async fn test_join() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let mut tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
+    let mut tc0 = MetaSrvTestContext::new(0);
+    let mut tc1 = MetaSrvTestContext::new(1);
 
     tc1.config.raft_config.single = false;
     tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];

metasrv/tests/it/flight/metasrv_flight_meta_api_leader_follower.rs

@@ -14,34 +14,22 @@
 
 use common_base::tokio;
 use common_meta_api::MetaApiTestSuite;
-use common_meta_flight::MetaFlightClient;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
-use crate::tests::start_metasrv_with_context;
+use crate::tests::service::start_metasrv_cluster;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
 async fn test_meta_api_database_create_get_drop() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let mut tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
+    let tcs = start_metasrv_cluster(&[0, 1]).await?;
 
-    tc1.config.raft_config.single = false;
-    tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];
-
-    start_metasrv_with_context(&mut tc0).await?;
-    start_metasrv_with_context(&mut tc1).await?;
-
-    let addr0 = tc0.config.flight_api_address.clone();
-    let addr1 = tc1.config.flight_api_address.clone();
-
-    let client0 = MetaFlightClient::try_create(addr0.as_str(), "root", "xxx").await?;
-    let client1 = MetaFlightClient::try_create(addr1.as_str(), "root", "xxx").await?;
+    let client0 = tcs[0].flight_client().await?;
+    let client1 = tcs[1].flight_client().await?;
 
     MetaApiTestSuite {}
-        .database_create_get_drop_leader_follower(&client0, &client1)
+        .database_get_leader_follower(&client0, &client1)
         .await
 }
 
@@ -50,20 +38,10 @@ async fn test_meta_api_list_table() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let mut tc0 = new_metasrv_test_context(0);
-    let mut tc1 = new_metasrv_test_context(1);
-
-    tc1.config.raft_config.single = false;
-    tc1.config.raft_config.join = vec![tc0.config.raft_config.raft_api_addr()];
-
-    start_metasrv_with_context(&mut tc0).await?;
-    start_metasrv_with_context(&mut tc1).await?;
-
-    let addr0 = tc0.config.flight_api_address.clone();
-    let addr1 = tc1.config.flight_api_address.clone();
+    let tcs = start_metasrv_cluster(&[0, 1]).await?;
 
-    let client0 = MetaFlightClient::try_create(addr0.as_str(), "root", "xxx").await?;
-    let client1 = MetaFlightClient::try_create(addr1.as_str(), "root", "xxx").await?;
+    let client0 = tcs[0].flight_client().await?;
+    let client1 = tcs[1].flight_client().await?;
 
     MetaApiTestSuite {}
         .list_table_leader_follower(&client0, &client1)

metasrv/tests/it/flight/metasrv_flight_tls.rs

@@ -20,7 +20,7 @@ use common_meta_flight::MetaFlightClient;
 use pretty_assertions::assert_eq;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 use crate::tests::start_metasrv_with_context;
 use crate::tests::tls_constants::TEST_CA_CERT;
 use crate::tests::tls_constants::TEST_CN_NAME;
@@ -32,7 +32,7 @@ async fn test_tls_server() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let mut tc = new_metasrv_test_context(0);
+    let mut tc = MetaSrvTestContext::new(0);
 
     tc.config.flight_tls_server_key = TEST_SERVER_KEY.to_owned();
     tc.config.flight_tls_server_cert = TEST_SERVER_CERT.to_owned();
@@ -63,7 +63,7 @@ async fn test_tls_server_config_failure() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let mut tc = new_metasrv_test_context(0);
+    let mut tc = MetaSrvTestContext::new(0);
 
     tc.config.flight_tls_server_key = "../tests/data/certs/not_exist.key".to_owned();
     tc.config.flight_tls_server_cert = "../tests/data/certs/not_exist.pem".to_owned();

metasrv/tests/it/meta_service/meta_node.rs

@@ -43,7 +43,6 @@ use pretty_assertions::assert_eq;
 
 use crate::init_meta_ut;
 use crate::tests::assert_metasrv_connection;
-use crate::tests::service::new_metasrv_test_context;
 use crate::tests::service::MetaSrvTestContext;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 5)]
@@ -54,7 +53,7 @@ async fn test_meta_node_boot() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc = new_metasrv_test_context(0);
+    let tc = MetaSrvTestContext::new(0);
     let addr = tc.config.raft_config.raft_api_addr();
 
     let mn = MetaNode::boot(&tc.config.raft_config).await?;
@@ -279,7 +278,7 @@ async fn test_meta_node_snapshot_replication() -> anyhow::Result<()> {
     // Create a snapshot every 10 logs
     let snap_logs = 10;
 
-    let mut tc = new_metasrv_test_context(0);
+    let mut tc = MetaSrvTestContext::new(0);
     tc.config.raft_config.snapshot_logs_since_last = snap_logs;
     tc.config.raft_config.install_snapshot_timeout = 10_1000; // milli seconds. In a CI multi-threads test delays async task badly.
     let addr = tc.config.raft_config.raft_api_addr();
@@ -408,7 +407,7 @@ async fn test_meta_node_join() -> anyhow::Result<()> {
     tracing::info!("--- bring up non-voter 2");
 
     let node_id = 2;
-    let tc2 = new_metasrv_test_context(node_id);
+    let tc2 = MetaSrvTestContext::new(node_id);
     let mn2 = MetaNode::open_create_boot(&tc2.config.raft_config, None, Some(()), None).await?;
 
     tracing::info!("--- join non-voter 2 to cluster by leader");
@@ -434,7 +433,7 @@ async fn test_meta_node_join() -> anyhow::Result<()> {
     tracing::info!("--- bring up non-voter 3");
 
     let node_id = 3;
-    let tc3 = new_metasrv_test_context(node_id);
+    let tc3 = MetaSrvTestContext::new(node_id);
     let mn3 = MetaNode::open_create_boot(&tc3.config.raft_config, None, Some(()), None).await?;
 
     tracing::info!("--- join node-3 by sending rpc `join` to a non-leader");
@@ -505,7 +504,7 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> {
     tracing::info!("--- bring up non-voter 1");
 
     let node_id = 1;
-    let tc1 = new_metasrv_test_context(node_id);
+    let tc1 = MetaSrvTestContext::new(node_id);
     let mn1 = MetaNode::open_create_boot(&tc1.config.raft_config, None, Some(()), None).await?;
 
     tracing::info!("--- join non-voter 1 to cluster");
@@ -530,7 +529,7 @@ async fn test_meta_node_join_rejoin() -> anyhow::Result<()> {
     tracing::info!("--- bring up non-voter 3");
 
     let node_id = 2;
-    let tc2 = new_metasrv_test_context(node_id);
+    let tc2 = MetaSrvTestContext::new(node_id);
     let mn2 = MetaNode::open_create_boot(&tc2.config.raft_config, None, Some(()), None).await?;
 
     tracing::info!("--- join node-2 by sending rpc `join` to a non-leader");
@@ -779,7 +778,7 @@ async fn start_meta_node_leader() -> anyhow::Result<(NodeId, MetaSrvTestContext)
     // asserts states are consistent
 
     let nid = 0;
-    let mut tc = new_metasrv_test_context(nid);
+    let mut tc = MetaSrvTestContext::new(nid);
     let addr = tc.config.raft_config.raft_api_addr();
 
     // boot up a single-node cluster
@@ -805,7 +804,7 @@ async fn start_meta_node_non_voter(
     leader: Arc<MetaNode>,
     id: NodeId,
 ) -> anyhow::Result<(NodeId, MetaSrvTestContext)> {
-    let mut tc = new_metasrv_test_context(id);
+    let mut tc = MetaSrvTestContext::new(id);
     let addr = tc.config.raft_config.raft_api_addr();
 
     let raft_config = tc.config.raft_config.clone();

metasrv/tests/it/meta_service/meta_service_impl.rs

@@ -30,14 +30,14 @@ use pretty_assertions::assert_eq;
 
 use crate::init_meta_ut;
 use crate::tests::assert_metasrv_connection;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
 async fn test_metasrv_upsert_kv() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc = new_metasrv_test_context(0);
+    let tc = MetaSrvTestContext::new(0);
     let addr = tc.config.raft_config.raft_api_addr();
 
     let _mn = MetaNode::boot(&tc.config.raft_config).await?;
@@ -82,7 +82,7 @@ async fn test_metasrv_incr_seq() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc = new_metasrv_test_context(0);
+    let tc = MetaSrvTestContext::new(0);
     let addr = tc.config.raft_config.raft_api_addr();
 
     let _mn = MetaNode::boot(&tc.config.raft_config).await?;
@@ -122,8 +122,8 @@ async fn test_metasrv_cluster_write_on_non_leader() -> anyhow::Result<()> {
     let (_log_guards, ut_span) = init_meta_ut!();
     let _ent = ut_span.enter();
 
-    let tc0 = new_metasrv_test_context(0);
-    let tc1 = new_metasrv_test_context(1);
+    let tc0 = MetaSrvTestContext::new(0);
+    let tc1 = MetaSrvTestContext::new(1);
 
     let addr0 = tc0.config.raft_config.raft_api_addr();
     let addr1 = tc1.config.raft_config.raft_api_addr();

metasrv/tests/it/store.rs

@@ -32,7 +32,7 @@ use databend_meta::Opened;
 use maplit::btreeset;
 
 use crate::init_meta_ut;
-use crate::tests::service::new_metasrv_test_context;
+use crate::tests::service::MetaSrvTestContext;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
 async fn test_metasrv_restart() -> anyhow::Result<()> {
@@ -48,7 +48,7 @@ async fn test_metasrv_restart() -> anyhow::Result<()> {
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     tracing::info!("--- new metasrv");
     {
@@ -93,7 +93,7 @@ async fn test_metasrv_get_membership_from_log() -> anyhow::Result<()> {
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     tracing::info!("--- new metasrv");
     let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
@@ -193,7 +193,7 @@ async fn test_metasrv_do_log_compaction_empty() -> anyhow::Result<()> {
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
 
@@ -241,7 +241,7 @@ async fn test_metasrv_do_log_compaction_1_snap_ptr_1_log() -> anyhow::Result<()>
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
 
@@ -315,7 +315,7 @@ async fn test_metasrv_do_log_compaction_all_logs_with_memberchange() -> anyhow::
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
 
@@ -368,7 +368,7 @@ async fn test_metasrv_do_log_compaction_current_snapshot() -> anyhow::Result<()>
     let _ent = ut_span.enter();
 
     let id = 3;
-    let tc = new_metasrv_test_context(id);
+    let tc = MetaSrvTestContext::new(id);
 
     let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
 
@@ -424,7 +424,7 @@ async fn test_metasrv_install_snapshot() -> anyhow::Result<()> {
     let id = 3;
     let snap;
     {
-        let tc = new_metasrv_test_context(id);
+        let tc = MetaSrvTestContext::new(id);
 
         let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;
 
@@ -441,7 +441,7 @@ async fn test_metasrv_install_snapshot() -> anyhow::Result<()> {
 
     tracing::info!("--- reopen a new metasrv to install snapshot");
     {
-        let tc = new_metasrv_test_context(id);
+        let tc = MetaSrvTestContext::new(id);
 
         let ms = MetaRaftStore::open_create(&tc.config.raft_config, None, Some(())).await?;