network delay

docker/Dockerfile.network_tests

@@ -0,0 +1,5 @@
+FROM aleph-node:latest
+
+RUN apt update && \
+        apt install curl iproute2 iputils-ping net-tools netwox tcpdump gdb gdbserver -y && \
+        apt clean

docker/docker-compose.network_tests.yml

@@ -0,0 +1,50 @@
+services:
+  Node0:
+    image: aleph-node:network_tests
+    networks:
+      - main
+    environment:
+      - PURGE_BEFORE_START=false
+    cap_add:
+      - NET_ADMIN
+      - SYS_PTRACE
+
+  Node1:
+    image: aleph-node:network_tests
+    networks:
+      - main
+    environment:
+      - PURGE_BEFORE_START=false
+    cap_add:
+      - NET_ADMIN
+      - SYS_PTRACE
+
+  Node2:
+    image: aleph-node:network_tests
+    networks:
+      - main
+    environment:
+      - PURGE_BEFORE_START=false
+    cap_add:
+      - NET_ADMIN
+      - SYS_PTRACE
+
+  Node3:
+    image: aleph-node:network_tests
+    networks:
+      - main
+    environment:
+      - PURGE_BEFORE_START=false
+    cap_add:
+      - NET_ADMIN
+      - SYS_PTRACE
+
+  Node4:
+    image: aleph-node:network_tests
+    networks:
+      - main
+    environment:
+      - PURGE_BEFORE_START=false
+    cap_add:
+      - NET_ADMIN
+      - SYS_PTRACE

scripts/catchup_version_upgrade_test.sh

@@ -14,42 +14,12 @@ ALL_NODES_PORTS=${ALL_NODES_PORTS:-"9933:9934:9935:9936:9937"}
 WAIT_BLOCKS=${WAIT_BLOCKS:-30}
 EXT_STATUS=${EXT_STATUS:-"in-block"}
 
-function log() {
-    echo $1 1>&2
-}
-
-function into_array() {
-    result=()
-    local tmp=$IFS
-    IFS=:
-    for e in $1; do
-        result+=($e)
-    done
-    IFS=$tmp
-}
+source ./scripts/common.sh
 
 function initialize {
     wait_for_finalized_block $1 $2 $3
 }
 
-function wait_for_finalized_block() {
-    local block_to_be_finalized=$1
-    local node=$2
-    local port=$3
-
-    while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do
-        sleep 3
-    done
-}
-
-function get_best_finalized {
-    local validator=$1
-    local rpc_port=$2
-
-    local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/')
-    printf "%d" $best_finalized
-}
-
 function set_upgrade_session {
     local session=$1
     local version=$2
@@ -110,45 +80,6 @@ function disconnect_nodes {
     done
 }
 
-function wait_for_block {
-    local block=$1
-    local validator=$2
-    local rpc_port=$3
-
-    local last_block=""
-    while [[ -z "$last_block" ]]; do
-        last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \
-                            -H "Content-Type: application/json" \
-                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result')
-    done
-}
-
-function get_last_block {
-    local validator=$1
-    local rpc_port=$2
-
-    local last_block_number=$(docker run --rm --network container:$validator appropriate/curl:latest \
-                                     -H "Content-Type: application/json" \
-                                     -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number')
-    printf "%d" $last_block_number
-}
-
-function check_finalization {
-    local block_to_check=$1
-    local -n nodes=$2
-    local -n ports=$3
-
-    log "checking finalization for block $block_to_check"
-
-    for i in "${!nodes[@]}"; do
-        local node=${nodes[$i]}
-        local rpc_port=${ports[$i]}
-
-        log "checking finalization at node $node"
-        wait_for_finalized_block $block_to_check $node $rpc_port
-    done
-}
-
 into_array $NODES
 NODES=(${result[@]})
 

scripts/common.sh

@@ -0,0 +1,112 @@
+#!/bin/env bash
+
+function log() {
+    echo $1 1>&2
+}
+
+function into_array() {
+    result=()
+    local tmp=$IFS
+    IFS=:
+    for e in $1; do
+        result+=($e)
+    done
+    IFS=$tmp
+}
+
+function check_finalization() {
+    local block_to_check=$1
+    local -n nodes=$2
+    local -n ports=$3
+
+    log "checking finalization for block $block_to_check"
+
+    for i in "${!nodes[@]}"; do
+        local node=${nodes[$i]}
+        local rpc_port=${ports[$i]}
+
+        log "checking finalization at node $node"
+        wait_for_finalized_block $block_to_check $node $rpc_port
+    done
+}
+
+function check_relative_finalization_at_node() {
+    local node=$1
+    local rpc_port=$2
+    local awaited_blocks=$3
+
+    local last_block=$(get_last_block $node $rpc_port)
+    local awaited_finalized=$(($last_block+$awaited_blocks))
+
+    log "Last block seen at node $node was $last_block, awaiting block $awaited_finalized to be finalized"
+
+    wait_for_finalized_block $awaited_finalized $node $rpc_port
+}
+
+function check_relative_finalization() {
+    local awaited_blocks=$1
+    local -n nodes=$2
+    local -n ports=$3
+
+    log "checking finalization for $awaited_blocks block(s) in the future"
+
+    for i in "${!nodes[@]}"; do
+        local node=${nodes[$i]}
+        local rpc_port=${ports[$i]}
+
+        log "checking finalization at node $node (${node}:$rpc_port)"
+        check_relative_finalization_at_node $node $rpc_port $awaited_blocks
+    done
+}
+
+function get_best_finalized() {
+    local validator=$1
+    local rpc_port=$2
+
+    local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/')
+    printf "%d" $best_finalized
+}
+
+function wait_for_finalized_block() {
+    local block_to_be_finalized=$1
+    local node=$2
+    local port=$3
+
+    while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do
+        sleep 3
+    done
+}
+
+function wait_for_block() {
+    local block=$1
+    local validator=$2
+    local rpc_port=$3
+
+    local last_block=""
+    while [[ -z "$last_block" ]]; do
+        last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \
+                            -H "Content-Type: application/json" \
+                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result')
+    done
+}
+
+function retrieve_last_block() {
+    local validator=$1
+    local rpc_port=$2
+
+    docker run --rm --network container:$validator appropriate/curl:latest \
+           -H "Content-Type: application/json" \
+           -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number'
+}
+
+function get_last_block() {
+    local validator=$1
+    local rpc_port=$2
+
+    local last_block=0
+    while [[ -z "$last_block" ]]; do
+        last_block=$(retrieve_last_block $validator $rpc_port)
+        sleep 1
+    done
+    printf "%d" $last_block
+}

scripts/run_consensus_network_delay.sh

@@ -0,0 +1,110 @@
+#!/bin/env bash
+
+set -euo pipefail
+
+source ./scripts/common.sh
+
+function usage(){
+    cat << EOF
+Usage:
+  $0
+     --network-delays "500:300"
+        list of delays for each node in ms; default="500:500:500:500:500"
+    --no-build-image
+        skip docker image build
+    --nodes "Node0:9933:Node1:9934"
+        list of pairs node:rpc_port; default="Node0:9933:Node1:9934:Node2:9935:Node3:9936:Node4:9937"
+    --check-block number
+        check finalization for a given block number, 0 means no-check; default=42
+EOF
+    exit 0
+}
+
+function build_test_image() {
+    docker build -t aleph-node:network_tests -f docker/Dockerfile.network_tests .
+}
+
+function set_network_delay() {
+    local node=$1
+    local delay=$2
+
+    log "setting network delay for node $node"
+    docker exec $node tc qdisc add dev eth1 root netem delay ${delay}ms
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --network-delays)
+            NETWORK_DELAYS="$2"
+            shift;shift
+            ;;
+        --no-build-image)
+            BUILD_IMAGE=false
+            shift
+            ;;
+        --nodes)
+            NODES="$2"
+            shift;shift
+            ;;
+        --check-block)
+            CHECK_BLOCK_FINALIZATION="$2"
+            shift;shift
+            ;;
+        --help)
+            usage
+            shift
+            ;;
+        *)
+            error "Unrecognized argument $1!"
+            ;;
+    esac
+done
+
+NETWORK_DELAYS=${NETWORK_DELAYS:-"500:500:500:500:500"}
+BUILD_IMAGE=${BUILD_IMAGE:-true}
+NODE_PAIRS=${NODES:-"Node0:9933:Node1:9934:Node2:9935:Node3:9936:Node4:9937"}
+NODES_PORTS=${NODES_PORTS:-"9933:9934:9935:9936:9937"}
+CHECK_BLOCK_FINALIZATION=${CHECK_BLOCK_FINALIZATION:-44}
+
+into_array $NETWORK_DELAYS
+NETWORK_DELAYS=(${result[@]})
+
+into_array $NODE_PAIRS
+NODE_PAIRS=(${result[@]})
+NODES=()
+NODES_PORTS=()
+for ((i=0; i<${#NODE_PAIRS[@]}; i+=2)); do
+    node=${NODE_PAIRS[$i]}
+    port=${NODE_PAIRS[(($i + 1))]}
+
+    NODES+=($node)
+    NODES_PORTS+=($port)
+done
+
+
+if [[ "$BUILD_IMAGE" = true ]]; then
+    log "building custom docker image for network tests"
+    build_test_image
+fi
+
+log "starting network"
+OVERRIDE_DOCKER_COMPOSE=./docker/docker-compose.network_tests.yml DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2
+log "network started"
+
+for i in "${!NODES[@]}"; do
+    node=${NODES[$i]}
+    delay=${NETWORK_DELAYS[$i]}
+    log "setting network delay for node $node to ${delay}ms"
+
+    set_network_delay $node $delay
+done
+
+if [[ $CHECK_BLOCK_FINALIZATION -gt 0 ]]; then
+    log "checking finalization"
+    check_relative_finalization $CHECK_BLOCK_FINALIZATION NODES NODES_PORTS
+    log "finalization checked"
+fi
+
+log "done"
+
+exit 0