chore(tests/load): C-chain load testing

.github/workflows/ci.yml

@@ -179,6 +179,15 @@ jobs:
       - uses: ./.github/actions/setup-go-for-project
       - shell: bash
         run: ./scripts/run_task.sh check-generate-canoto
+  check_contract_bindings:
+    name: Up-to-date contract bindings
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-go-for-project
+      - uses: ./.github/actions/install-nix
+      - shell: nix develop --command bash -x {0}
+        run: task check-generate-load-contract-bindings
   go_mod_tidy:
     name: Up-to-date go.mod and go.sum
     runs-on: ubuntu-latest
@@ -227,3 +236,20 @@ jobs:
       - name: Run e2e tests
         shell: bash
         run: nix develop --command ./scripts/run_task.sh test-bootstrap-monitor-e2e
+  load_tests:
+    name: load tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup-go-for-project
+      - uses: ./.github/actions/install-nix
+      - uses: ./.github/actions/run-monitored-tmpnet-cmd
+        with:
+          run: ./scripts/run_task.sh test-load
+          artifact_prefix: load
+          prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }}
+          prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
+          loki_username: ${{ secrets.LOKI_ID || '' }}
+          loki_password: ${{ secrets.LOKI_PASSWORD || '' }}

Taskfile.yml

@@ -61,6 +61,12 @@ tasks:
       - task: generate-canoto
       - task: check-clean-branch
 
+  check-generate-load-contract-bindings:
+    desc: Checks that generated load contract bindings are up-to-date (requires a clean git working tree)
+    cmds:
+      - task: generate-load-contract-bindings
+      - task: check-clean-branch
+
   check-generate-mocks:
     desc: Checks that generated mocks are up-to-date (requires a clean git working tree)
     cmds:
@@ -89,6 +95,12 @@ tasks:
     desc: Generates canoto
     cmd: go generate -run "github.com/StephenButtolph/canoto/canoto" ./...
 
+  generate-load-contract-bindings:
+    desc: Generates load contract bindings
+    cmds:
+      - cmd: grep -lr -E '^// Code generated - DO NOT EDIT\.$' tests/load/c | xargs -r rm
+      - cmd: go generate ./tests/load/c/...
+
   generate-protobuf:
     desc: Generates protobuf
     cmd: ./scripts/protobuf_codegen.sh
@@ -214,6 +226,13 @@ tasks:
       FUZZTIME: '{{.FUZZTIME| default "900"}}'
     cmd: ./scripts/build_fuzz.sh {{.FUZZTIME}} ./x/merkledb
 
+  test-load:
+    desc: Runs load tests
+    cmds:
+      - task: generate-load-contract-bindings
+      - task: build
+      - cmd: go run ./tests/load/c/main --avalanchego-path=./build/avalanchego {{.CLI_ARGS}}
+
   test-unit:
     desc: Runs unit tests
     # Invoking with bash ensures compatibility with CI execution on Windows

flake.nix

@@ -57,6 +57,9 @@
             buf
             protoc-gen-go
             protoc-gen-go-grpc
+
+            # Solidity compiler
+            solc
           ] ++ lib.optionals stdenv.isDarwin [
             # macOS-specific frameworks
             darwin.apple_sdk.frameworks.Security

go.sum

@@ -258,7 +258,10 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
 github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=

scripts/build_test.sh

@@ -7,7 +7,7 @@ AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 # Load the constants
 source "$AVALANCHE_PATH"/scripts/constants.sh
 
-EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/upgrade | grep -v tests/fixture/bootstrapmonitor/e2e"
+EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/load/c | grep -v tests/upgrade | grep -v tests/fixture/bootstrapmonitor/e2e"
 
 if [[ "$(go env GOOS)" == "windows" ]]; then
   # Test discovery for the antithesis test setups is broken due to

scripts/lint.sh

@@ -41,7 +41,7 @@ function test_golangci_lint {
 _addlicense_flags=${ADDLICENSE_FLAGS:-"--verify --debug"}
 function test_license_header {
   local files=()
-  while IFS= read -r line; do files+=("$line"); done < <(find . -type f -name '*.go' ! -name '*.pb.go' ! -name 'mock_*.go' ! -name 'mocks_*.go' ! -path './**/*mock/*.go' ! -name '*.canoto.go')
+  while IFS= read -r line; do files+=("$line"); done < <(find . -type f -name '*.go' ! -name '*.pb.go' ! -name 'mock_*.go' ! -name 'mocks_*.go' ! -path './**/*mock/*.go' ! -name '*.canoto.go' ! -name '*.bindings.go')
 
   # shellcheck disable=SC2086
   go run github.com/palantir/[email protected] \

tests/fixture/tmpnet/monitor_processes.go

@@ -253,6 +253,12 @@ func getWorkingDir(cmdName string) (string, error) {
 	return filepath.Join(tmpnetDir, cmdName), nil
 }
 
+// GetPrometheusServiceDiscoveryDir returns the path for prometheus file-based
+// service discovery configuration.
+func GetPrometheusServiceDiscoveryDir() (string, error) {
+	return getServiceDiscoveryDir(prometheusCmd)
+}
+
 func getServiceDiscoveryDir(cmdName string) (string, error) {
 	tmpnetDir, err := getTmpnetPath()
 	if err != nil {

tests/fixture/tmpnet/network.go

@@ -57,6 +57,9 @@ const (
 
 	// eth address: 0x8db97C7cEcE249c2b98bDC0226Cc4C2A57BF52FC
 	HardHatKeyStr = "56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027"
+
+	// grafanaURI is remote Grafana URI
+	grafanaURI = "grafana-poc.avax-dev.network"
 )
 
 var (
@@ -1103,7 +1106,8 @@ func MetricsLinkForNetwork(networkUUID string, startTime string, endTime string)
 		endTime = "now"
 	}
 	return fmt.Sprintf(
-		"https://grafana-poc.avax-dev.network/d/kBQpRdWnk/avalanche-main-dashboard?&var-filter=network_uuid%%7C%%3D%%7C%s&var-filter=is_ephemeral_node%%7C%%3D%%7Cfalse&from=%s&to=%s",
+		"https://%s/d/kBQpRdWnk/avalanche-main-dashboard?&var-filter=network_uuid%%7C%%3D%%7C%s&var-filter=is_ephemeral_node%%7C%%3D%%7Cfalse&from=%s&to=%s",
+		grafanaURI,
 		networkUUID,
 		startTime,
 		endTime,

tests/fixture/tmpnet/utils.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"syscall"
 	"time"
@@ -91,6 +92,31 @@ func FilterAvailableNodes(nodes []*Node) []*Node {
 	return filteredNodes
 }
 
+// GetNodeWebsocketURIs returns a list of websocket URIs for the given nodes and
+// blockchain ID, in the form "ws://<node-uri>/ext/bc/<blockchain-id>/ws".
+// Ephemeral and stopped nodes are ignored.
+func GetNodeWebsocketURIs(
+	ctx context.Context,
+	nodes []*Node,
+	blockchainID string,
+	deferCleanupFunc func(func()),
+) ([]string, error) {
+	nodeURIs, err := GetNodeURIs(ctx, nodes, deferCleanupFunc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get node URIs: %w", err)
+	}
+	wsURIs := make([]string, len(nodeURIs))
+	for i := range nodeURIs {
+		uri, err := url.Parse(nodeURIs[i].URI)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse node URI: %w", err)
+		}
+		uri.Scheme = "ws" // use websocket to be able to stream events
+		wsURIs[i] = fmt.Sprintf("%s/ext/bc/%s/ws", uri, blockchainID)
+	}
+	return wsURIs, nil
+}
+
 // Marshal to json with default prefix and indent.
 func DefaultJSONMarshal(v interface{}) ([]byte, error) {
 	return json.MarshalIndent(v, "", "  ")

tests/load/c/README.md

@@ -0,0 +1,69 @@
+# Load testing
+
+The C-chain load test entrypoint is in [main.go](main/main.go).
+
+It runs with 5 nodes and 5 "agents".
+
+Each "agent" runs a transaction issuer and a transaction listener asynchronously,
+and is assigned uniformly to the nodes available, via websocket connections.
+
+The load test picks at weighted random a transaction type to generate and issue, defined in [issuer.go](issuer.go).
+
+For some transaction types, [this contract](contracts/EVMLoadSimulator.sol) is deployed and transactions call functions of this contract. This contract has different functions, each testing a particular performance aspect of the EVM, for example memory writes.
+
+From the load test perspective, only the TPS (transactions per second) is logged out. Metrics available are:
+
+- total transactions issued `txs_issued`
+- total transactions confirmed `txs_confirmed`
+- total transactions failed `txs_failed`
+- transaction latency histogram `tx_latency`
+
+There are more interesting metrics available from the tmpnet nodes being load tested.
+
+Finally, to run the load test, run:
+
+```bash
+# Install nix (skip this step if nix is already installed)
+./scripts/run_task.sh install-nix
+# Start the dev shell
+nix develop
+# Start the load test
+task test-load
+```
+
+## Visualize metrics in Grafana
+
+### Private remote instances
+
+If you have the credentials (internal to Ava Labs) for the CI monitoring stack, you can visualize the metrics following these steps:
+
+1. Start a dev shell to ensure `prometheus` and `promtail` binaries are available to the test runner so it can use them to collect metrics and logs:
+
+    ```bash
+    nix develop
+    ```
+
+2. Set your monitoring credentials using the credentials you can find in your password manager
+
+    ```bash
+    export PROMETHEUS_USERNAME=<username>
+    export PROMETHEUS_PASSWORD=<password>
+    export LOKI_USERNAME=<username>
+    export LOKI_PASSWORD=<password>
+    ```
+
+3. Run the load test:
+
+    ```bash
+    task test-load
+    ```
+
+4. Prior to the test beginning, you will see the following log:
+
+    ```log
+    INFO metrics and logs available via grafana (collectors must be running)     {"uri": "https://grafana-poc.avax-dev.network/d/eabddd1d-0a06-4ba1-8e68-a44504e37535/C-Chain%20Load?from=1747817500582&to=1747817952631&var-filter=network_uuid%7C%3D%7C4f419e3a-dba5-4ccd-b2fd-bda15f9826ff"}
+    ```
+
+5. Open the URL in your browser, and log in with the Grafana credentials which you can find in your password manager.
+
+For reference, see [the tmpnet monitoring section](../../fixture/tmpnet/README.md#monitoring)

tests/load/c/contracts/Dummy.sol

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.21;
+
+/// @notice Dummy is a contract used only for simulating the load
+/// of contract creation operations in the EVM. It has state variables,
+/// a constructor and simple functions to have a real-like deployed size.
+contract Dummy {
+    uint256 public value;
+    address public owner;
+    mapping(uint256 => uint256) public data;
+
+    event ValueUpdated(uint256 newValue);
+    event DataWritten(uint256 key, uint256 value);
+
+    constructor() {
+        value = 42;
+        owner = msg.sender;
+    }
+
+    function updateValue(uint256 newValue) external {
+        value = newValue;
+    }
+
+    function writeData(uint256 key, uint256 val) external {
+        data[key] = val;
+        emit DataWritten(key, val);
+    }
+
+    function readData(uint256 key) external view returns (uint256) {
+        return data[key];
+    }
+}