Start prometheus endpoint in minimal-node

Use prebuilt try-runtime binary in CI (paritytech#1898)

`cargo install` takes a long time in CI. We want to run it relatively
frequently without chewing through so much compute (see
paritytech/ci_cd#771) so I added automatic
binary releases to the try-runtime-cli repo.

A small added benefit is we can use it in our existing
`on-runtime-upgrade` checks, which should cut their execution time by
about half.

Cumulus: Allow aura to use initialized collation request receiver (paritytech#1911)

When launching our [small
network](https://github.com/paritytech/polkadot-sdk/blob/master/cumulus/zombienet/examples/small_network.toml)
for testing the node was crashing here shortly after launch:

https://github.com/paritytech/polkadot-sdk/blob/5cdd819ed295645958afd9d937d989978fd0c84e/polkadot/node/collation-generation/src/lib.rs#L140

After changes in paritytech#1788 for the asset hub collator we are waiting for
blocks of the shell runtime to pass before we initialize aura. However,
this means that we attempted to initialize the collation related relay
chain subsystems twice, leading to the error.

I modified Aura to let it optionally take an already initialized stream
of collation requests.

Remove comments

Author: skunert

.gitlab/pipeline/check.yml

@@ -112,12 +112,16 @@ test-rust-feature-propagation:
   script:
     - |
       export RUST_LOG=remote-ext=debug,runtime=debug
-      echo "---------- Installing try-runtime-cli ----------"
-      time cargo install --locked --git https://github.com/paritytech/try-runtime-cli --tag v0.3.0
+
+      echo "---------- Downloading try-runtime CLI ----------"
+      curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.3.3/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+      chmod +x ./try-runtime
+
       echo "---------- Building ${PACKAGE} runtime ----------"
       time cargo build --release --locked -p "$PACKAGE" --features try-runtime
+
       echo "---------- Executing `on-runtime-upgrade` for ${NETWORK} ----------"
-      time try-runtime \
+      time ./try-runtime \
           --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
           on-runtime-upgrade --checks=pre-and-post ${EXTRA_ARGS} live --uri ${URI}
 

Cargo.lock

@@ -23,7 +23,9 @@
 //! For more information about AuRa, the Substrate crate should be checked.
 
 use codec::{Codec, Decode};
-use cumulus_client_collator::service::ServiceInterface as CollatorServiceInterface;
+use cumulus_client_collator::{
+	relay_chain_driven::CollationRequest, service::ServiceInterface as CollatorServiceInterface,
+};
 use cumulus_client_consensus_common::ParachainBlockImportMarker;
 use cumulus_client_consensus_proposer::ProposerInterface;
 use cumulus_primitives_core::{relay_chain::BlockId as RBlockId, CollectCollationInfo};
@@ -33,7 +35,7 @@ use polkadot_node_primitives::CollationResult;
 use polkadot_overseer::Handle as OverseerHandle;
 use polkadot_primitives::{CollatorPair, Id as ParaId};
 
-use futures::prelude::*;
+use futures::{channel::mpsc::Receiver, prelude::*};
 use sc_client_api::{backend::AuxStore, BlockBackend, BlockOf};
 use sc_consensus::BlockImport;
 use sp_api::ProvideRuntimeApi;
@@ -81,6 +83,10 @@ pub struct Params<BI, CIDP, Client, RClient, SO, Proposer, CS> {
 	pub collator_service: CS,
 	/// The amount of time to spend authoring each block.
 	pub authoring_duration: Duration,
+	/// Receiver for collation requests. If `None`, Aura consensus will establish a new receiver.
+	/// Should be used when a chain migrates from a different consensus algorithm and was already
+	/// processing collation requests before initializing Aura.
+	pub collation_request_receiver: Option<Receiver<CollationRequest>>,
 }
 
 /// Run bare Aura consensus as a relay-chain-driven collator.
@@ -110,12 +116,16 @@ where
 	P::Signature: TryFrom<Vec<u8>> + Member + Codec,
 {
 	async move {
-		let mut collation_requests = cumulus_client_collator::relay_chain_driven::init(
-			params.collator_key,
-			params.para_id,
-			params.overseer_handle,
-		)
-		.await;
+		let mut collation_requests = match params.collation_request_receiver {
+			Some(receiver) => receiver,
+			None =>
+				cumulus_client_collator::relay_chain_driven::init(
+					params.collator_key,
+					params.para_id,
+					params.overseer_handle,
+				)
+				.await,
+		};
 
 		let mut collator = {
 			let params = collator_util::Params {

cumulus/client/consensus/aura/src/collators/basic.rs

@@ -23,6 +23,7 @@ sc-authority-discovery = { path = "../../../substrate/client/authority-discovery
 sc-network = { path = "../../../substrate/client/network" }
 sc-network-common = { path = "../../../substrate/client/network/common" }
 sc-service = { path = "../../../substrate/client/service" }
+substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" }
 sc-tracing = { path = "../../../substrate/client/tracing" }
 sc-utils = { path = "../../../substrate/client/utils" }
 sp-api = { path = "../../../substrate/primitives/api" }

cumulus/client/relay-chain-minimal-node/Cargo.toml

@@ -44,7 +44,6 @@ use sc_authority_discovery::Service as AuthorityDiscoveryService;
 use sc_network::NetworkStateInfo;
 use sc_service::TaskManager;
 use sc_utils::mpsc::tracing_unbounded;
-use sp_runtime::traits::Block as BlockT;
 
 use cumulus_primitives_core::relay_chain::{Block, Hash as PHash};
 use cumulus_relay_chain_interface::RelayChainError;
@@ -209,8 +208,6 @@ pub struct NewMinimalNode {
 	pub task_manager: TaskManager,
 	/// Overseer handle to interact with subsystems
 	pub overseer_handle: Handle,
-	/// Network service
-	pub network: Arc<sc_network::NetworkService<Block, <Block as BlockT>::Hash>>,
 }
 
 /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding

cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs

@@ -32,10 +32,10 @@ use polkadot_primitives::CollatorPair;
 
 use sc_authority_discovery::Service as AuthorityDiscoveryService;
 use sc_network::{config::FullNetworkConfiguration, Event, NetworkEventStream, NetworkService};
-use sc_service::{Configuration, TaskManager};
+use sc_service::{config::PrometheusConfig, Configuration, TaskManager};
 use sp_runtime::{app_crypto::Pair, traits::Block as BlockT};
 
-use futures::StreamExt;
+use futures::{FutureExt, StreamExt};
 use std::sync::Arc;
 
 mod blockchain_rpc_client;
@@ -69,7 +69,7 @@ fn build_authority_discovery_service<Block: BlockT>(
 			..Default::default()
 		},
 		client,
-		network.clone(),
+		network,
 		Box::pin(dht_event_stream),
 		authority_discovery_role,
 		prometheus_registry,
@@ -160,12 +160,16 @@ async fn new_minimal_relay_chain(
 	let role = config.role.clone();
 	let mut net_config = sc_network::config::FullNetworkConfiguration::new(&config.network);
 
-	let task_manager = {
-		let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry);
-		TaskManager::new(config.tokio_handle.clone(), registry)?
-	};
+	let prometheus_registry = config.prometheus_registry();
+	let task_manager = TaskManager::new(config.tokio_handle.clone(), prometheus_registry)?;
 
-	let prometheus_registry = config.prometheus_registry().cloned();
+	if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() {
+		task_manager.spawn_handle().spawn(
+			"prometheus-endpoint",
+			None,
+			substrate_prometheus_endpoint::init_prometheus(port, registry).map(drop),
+		);
+	}
 
 	let genesis_hash = relay_chain_rpc_client
 		.block_get_hash(Some(0))
@@ -203,18 +207,18 @@ async fn new_minimal_relay_chain(
 		relay_chain_rpc_client.clone(),
 		&config,
 		network.clone(),
-		prometheus_registry.clone(),
+		prometheus_registry.cloned(),
 	);
 
 	let overseer_args = CollatorOverseerGenArgs {
 		runtime_client: relay_chain_rpc_client.clone(),
-		network_service: network.clone(),
+		network_service: network,
 		sync_oracle,
 		authority_discovery_service,
 		collation_req_receiver_v1,
 		collation_req_receiver_v2,
 		available_data_req_receiver,
-		registry: prometheus_registry.as_ref(),
+		registry: prometheus_registry,
 		spawner: task_manager.spawn_handle(),
 		collator_pair,
 		req_protocol_names: request_protocol_names,
@@ -226,7 +230,7 @@ async fn new_minimal_relay_chain(
 
 	network_starter.start_network();
 
-	Ok(NewMinimalNode { task_manager, overseer_handle, network })
+	Ok(NewMinimalNode { task_manager, overseer_handle })
 }
 
 fn build_request_response_protocol_receivers(

cumulus/client/relay-chain-minimal-node/src/lib.rs

@@ -411,6 +411,7 @@ fn start_consensus(
 		collator_service,
 		// Very limited proposal time.
 		authoring_duration: Duration::from_millis(500),
+		collation_request_receiver: None,
 	};
 
 	let fut =

cumulus/parachain-template/node/src/service.rs

@@ -988,6 +988,7 @@ pub async fn start_rococo_parachain_node(
 				collator_service,
 				// Very limited proposal time.
 				authoring_duration: Duration::from_millis(500),
+				collation_request_receiver: None,
 			};
 
 			let fut = basic_aura::run::<
@@ -1380,6 +1381,7 @@ where
 				collator_service,
 				// Very limited proposal time.
 				authoring_duration: Duration::from_millis(500),
+				collation_request_receiver: None,
 			};
 
 			let fut =
@@ -1520,6 +1522,7 @@ where
 					collator_service,
 					// Very limited proposal time.
 					authoring_duration: Duration::from_millis(500),
+					collation_request_receiver: Some(request_stream),
 				};
 
 				basic_aura::run::<Block, <AuraId as AppCrypto>::Pair, _, _, _, _, _, _, _>(params)
@@ -1925,6 +1928,7 @@ pub async fn start_contracts_rococo_node(
 				collator_service,
 				// Very limited proposal time.
 				authoring_duration: Duration::from_millis(500),
+				collation_request_receiver: None,
 			};
 
 			let fut = basic_aura::run::<

cumulus/polkadot-parachain/src/service.rs

@@ -59,6 +59,7 @@ pub use self::{
 	},
 	client::{ClientConfig, LocalCallExecutor},
 	error::Error,
+	metrics::MetricsService,
 };
 
 pub use sc_chain_spec::{