🔪 flaky and Zombienet tests (#8600)

Commenting out all flaky tests and tracking them here
#48

Changes:
- Disable flaky Rust tests by adding a new disabled feature. `#[ignore]`
attribute is not possible since CI runs with `--ignored`
- Disable all Zombienet tests

- [ ] Waiting for CI what other tests fail.

---------

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Author: ggwpez

.github/workflows/zombienet_cumulus.yml

@@ -1,12 +1,13 @@
 name: Zombienet Cumulus
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-  merge_group:
+  workflow_dispatch: # Disabled for being flaky
+  #push:
+  #  branches:
+  #    - master
+  #pull_request:
+  #  types: [opened, synchronize, reopened, ready_for_review]
+  #merge_group:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

.github/workflows/zombienet_parachain-template.yml

@@ -1,12 +1,13 @@
 name: Zombienet Parachain Templates
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-  merge_group:
+  workflow_dispatch: # Disabled for being flaky
+  #push:
+  #  branches:
+  #    - master
+  #pull_request:
+  #  types: [opened, synchronize, reopened, ready_for_review]
+  #merge_group:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

.github/workflows/zombienet_polkadot.yml

@@ -1,12 +1,13 @@
 name: Zombienet Polkadot
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-  merge_group:
+  workflow_dispatch: # Disabled for being flaky
+  #push:
+  #  branches:
+  #    - master
+  #pull_request:
+  #  types: [opened, synchronize, reopened, ready_for_review]
+  #merge_group:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

.github/workflows/zombienet_substrate.yml

@@ -1,12 +1,13 @@
 name: Zombienet Substrate
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-  merge_group:
+  workflow_dispatch: # Disabled for being flaky
+  #push:
+  #  branches:
+  #    - master
+  #pull_request:
+  #  types: [opened, synchronize, reopened, ready_for_review]
+  #merge_group:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

.github/zombienet-env

@@ -8,4 +8,4 @@ RUN_IN_CI=1
 KUBERNETES_CPU_REQUEST=512m
 KUBERNETES_MEMORY_REQUEST=1Gi
 TEMP_IMAGES_BASE=europe-docker.pkg.dev/parity-ci-2024/temp-images
-FLAKY_TESTS="zombienet-polkadot-coretime-revenue, zombienet-polkadot-smoke-0003-deregister-register-validator"
+FLAKY_TESTS="zombienet-polkadot-coretime-revenue, zombienet-polkadot-smoke-0003-deregister-register-validator, zombienet-polkadot-elastic-scaling-slot-based-12cores, zombienet-polkadot-elastic-scaling-doesnt-break-parachains, zombienet-polkadot-functional-duplicate-collations, zombienet-polkadot-functional-0002-parachains-disputes, zombienet-polkadot-functional-async-backing-6-seconds-rate, zombienet-polkadot-elastic-scaling-slot-based-3cores, zombienet-polkadot-malus-0001-dispute-valid, zombienet-substrate-0002-validators-warp-sync"

Cargo.toml

@@ -587,6 +587,7 @@ unexpected_cfgs = { level = "warn", check-cfg = [
 	'cfg(build_profile, values("debug", "release"))',
 	'cfg(enable_alloc_error_handler)',
 	'cfg(fuzzing)',
+	'cfg(ignore_flaky_test)',
 	'cfg(substrate_runtime)',
 ] }
 

substrate/client/consensus/babe/src/tests.rs

@@ -423,6 +423,7 @@ async fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + '
 	.await;
 }
 
+#[cfg(ignore_flaky_test)] // https://github.com/paritytech/polkadot-sdk/issues/48
 #[tokio::test]
 async fn authoring_blocks() {
 	run_one_test(|_, _| ()).await;
@@ -441,6 +442,7 @@ async fn rejects_missing_inherent_digest() {
 	.await;
 }
 
+#[cfg(ignore_flaky_test)] // https://github.com/paritytech/polkadot-sdk/issues/48
 #[tokio::test]
 #[should_panic(expected = "has a bad seal")]
 async fn rejects_missing_seals() {

substrate/client/network/src/discovery.rs

@@ -1303,26 +1303,26 @@ fn kademlia_protocol_name<Hash: AsRef<[u8]>>(
 
 #[cfg(test)]
 mod tests {
-	use super::{
-		kademlia_protocol_name, legacy_kademlia_protocol_name, DiscoveryConfig, DiscoveryOut,
-	};
+	use super::{kademlia_protocol_name, legacy_kademlia_protocol_name, DiscoveryConfig};
 	use crate::config::ProtocolId;
-	use futures::prelude::*;
-	use libp2p::{
-		core::{
-			transport::{MemoryTransport, Transport},
-			upgrade,
-		},
-		identity::Keypair,
-		noise,
-		swarm::{Swarm, SwarmEvent},
-		yamux, Multiaddr,
-	};
+	use libp2p::{identity::Keypair, Multiaddr};
 	use sp_core::hash::H256;
-	use std::{collections::HashSet, task::Poll, time::Duration};
 
+	#[cfg(ignore_flaky_test)] // https://github.com/paritytech/polkadot-sdk/issues/48
 	#[tokio::test]
 	async fn discovery_working() {
+		use super::DiscoveryOut;
+		use futures::prelude::*;
+		use libp2p::{
+			core::{
+				transport::{MemoryTransport, Transport},
+				upgrade,
+			},
+			noise,
+			swarm::{Swarm, SwarmEvent},
+			yamux,
+		};
+		use std::{collections::HashSet, task::Poll, time::Duration};
 		let mut first_swarm_peer_id_and_addr = None;
 
 		let genesis_hash = H256::from_low_u64_be(1);

substrate/client/network/test/src/sync.rs

@@ -826,6 +826,7 @@ async fn sync_to_tip_requires_that_sync_protocol_is_informed_about_best_block()
 
 /// Ensures that if we as a syncing node sync to the tip while we are connected to another peer
 /// that is currently also doing a major sync.
+#[cfg(ignore_flaky_test)] // https://github.com/paritytech/polkadot-sdk/issues/48
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn sync_to_tip_when_we_sync_together_with_multiple_peers() {
 	sp_tracing::try_init_simple();