diff --git a/.env b/.env index 16b6b2d16..91d193035 100644 --- a/.env +++ b/.env @@ -209,3 +209,10 @@ STACKS_NODE_TYPE=L1 # Folder with events to be imported by the event-replay. STACKS_EVENTS_DIR=./events + +# If enabled this service will connect to the specified SNP redis server and consume events from the SNP stream. +# This is an alternative to consuming events directly from a stacks-node. +# SNP_EVENT_STREAMING=true +# SNP_REDIS_URL=redis://127.0.0.1:6379 +# Only specify `SNP_REDIS_STREAM_KEY_PREFIX` if `REDIS_STREAM_KEY_PREFIX` is configured on the SNP server. +# SNP_REDIS_STREAM_KEY_PREFIX= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3943be1d3..5dc97a45d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,6 +131,45 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} if: always() + test-snp: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + # https://github.com/actions/cache/blob/main/examples.md#node---npm + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + + - name: Cache node modules + uses: actions/cache@v4 + id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Install deps + run: npm ci --audit=false + + - name: Install client deps + working-directory: client + run: npm ci --audit=false + - name: Run tests + run: npm run test:snp -- --coverage + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + if: always() + test-2_5: strategy: fail-fast: false diff --git a/.nvmrc b/.nvmrc index 9a2a0e219..53d1c14db 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20 +v22 diff --git a/.vscode/launch.json b/.vscode/launch.json index 72a01bd2f..bb0996689 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -54,7 +54,7 @@ "skipFiles": [ "/**" ], - "runtimeVersion": "20", + "runtimeVersion": "22", "runtimeArgs": ["-r", "ts-node/register/transpile-only", "-r", "tsconfig-paths/register"], "args": ["${workspaceFolder}/src/index.ts"], "outputCapture": "std", @@ -446,6 +446,23 @@ "STACKS_CHAIN_ID": "0x80000000" } }, + { + "type": "node", + "request": "launch", + "name": "Jest: SNP", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "args": [ + "--testTimeout=3600000", + "--runInBand", + "--no-cache", + "--config", + "${workspaceRoot}/tests/jest.config.snp.js" + ], + "outputCapture": "std", + "console": "integratedTerminal", + "nodeVersionHint": 22, + "runtimeVersion": "22" + }, { "type": "node", "request": "launch", diff --git a/Dockerfile b/Dockerfile index 0e37872fd..14e6980be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-bookworm-slim +FROM node:22-bookworm-slim WORKDIR /app COPY . . diff --git a/docker/rosetta.Dockerfile b/docker/rosetta.Dockerfile index e48c16695..a66bb2ed9 100644 --- a/docker/rosetta.Dockerfile +++ b/docker/rosetta.Dockerfile @@ -12,7 +12,7 @@ ARG ARCHIVE_VERSION=latest ####################################################################### ## Build the stacks-blockchain-api -FROM node:20-bookworm-slim as stacks-blockchain-api-build +FROM node:22-bookworm-slim as stacks-blockchain-api-build ARG STACKS_API_VERSION ENV STACKS_API_REPO=hirosystems/stacks-blockchain-api ENV STACKS_API_VERSION=${STACKS_API_VERSION} diff --git a/docker/standalone-regtest.Dockerfile b/docker/standalone-regtest.Dockerfile index 4b641d1b3..1645a8975 100644 --- a/docker/standalone-regtest.Dockerfile +++ b/docker/standalone-regtest.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM node:20-bookworm-slim as api-builder +FROM node:22-bookworm-slim as api-builder ARG API_GIT_COMMIT ARG STACKS_API_VERSION diff --git a/migrations/1748256987656_snp-state.js b/migrations/1748256987656_snp-state.js new file mode 100644 index 000000000..9d115ba5d --- /dev/null +++ b/migrations/1748256987656_snp-state.js @@ -0,0 +1,25 @@ +// @ts-check +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.up = pgm => { + pgm.createTable('snp_state', { + id: { + type: 'boolean', + primaryKey: true, + default: true, + }, + last_redis_msg_id: { + type: 'text', + notNull: true, + default: '0', + }, + }); + // Ensure only a single row can exist + pgm.addConstraint('snp_state', 'snp_state_one_row', 'CHECK(id)'); + // Create the single row + pgm.sql('INSERT INTO snp_state VALUES(DEFAULT)'); +}; + +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.down = pgm => { + pgm.dropTable('snp_state'); +}; diff --git a/package-lock.json b/package-lock.json index da6fd9b78..2a94be02b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "@fastify/multipart": "8.3.1", "@fastify/swagger": "8.15.0", "@fastify/type-provider-typebox": "4.0.0", - "@hirosystems/api-toolkit": "1.6.2", + "@hirosystems/api-toolkit": "1.9.0", + "@hirosystems/salt-n-pepper-client": "1.1.1", "@scure/base": "1.1.1", "@sinclair/typebox": "0.32.35", "@stacks/common": "6.10.0", @@ -29,14 +30,14 @@ "ajv": "8.17.1", "ajv-formats": "3.0.1", "bignumber.js": "9.0.2", - "bitcoinjs-lib": "^6.1.1", + "bitcoinjs-lib": "6.1.1", "c32check": "1.1.3", "coinselect": "3.1.12", "cors": "2.8.5", "cross-env": "7.0.3", "dotenv": "8.6.0", "dotenv-flow": "3.2.0", - "duckdb": "^1.2.0", + "duckdb": "1.2.0", "ecpair": "2.1.0", "elliptic": "6.6.1", "escape-goat": "3.0.0", @@ -83,12 +84,13 @@ "@redocly/cli": "1.34.1", "@stacks/eslint-config": "1.2.0", "@types/cors": "2.8.12", + "@types/dockerode": "3.3.39", "@types/dotenv-flow": "3.2.0", "@types/elliptic": "6.4.14", "@types/express": "4.17.13", "@types/is-ci": "3.0.0", "@types/jest": "29.5.6", - "@types/node": "20.11.4", + "@types/node": "22.15.21", "@types/node-fetch": "2.5.12", "@types/pg": "7.14.11", "@types/pg-copy-streams": "1.2.1", @@ -101,6 +103,7 @@ "@typescript-eslint/parser": "5.51.0", "concurrently": "7.3.0", "docker-compose": "0.24.8", + "dockerode": "4.0.6", "eslint": "8.29.0", "eslint-plugin-prettier": "4.2.1", "eslint-plugin-tsdoc": "0.2.17", @@ -123,7 +126,7 @@ "why-is-node-running": "2.2.0" }, "engines": { - "node": ">=20" + "node": ">=22" }, "optionalDependencies": { "bufferutil": "4.0.5", @@ -741,6 +744,13 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -1314,15 +1324,50 @@ "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==" }, + "node_modules/@grpc/grpc-js": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", + "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.7.13", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@hirosystems/api-toolkit": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/@hirosystems/api-toolkit/-/api-toolkit-1.6.2.tgz", - "integrity": "sha512-soL9IoRVADCm06x1sK5JHccvdAIhs4wk385lzlH/nIH3dIM7qJTQwC4gv5dCoSB4O9o0hK98sgB4xebjikPTGg==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@hirosystems/api-toolkit/-/api-toolkit-1.9.0.tgz", + "integrity": "sha512-9SFStFtDohud0U0J8HPhUp5Br0La4BggqOPijUJgOSjmAE3mWK7SNOAcybbe7G3YP9bpf4QMWaxcwH5mAKjHpQ==", + "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", "@fastify/swagger": "^8.3.1", "@fastify/type-provider-typebox": "^3.2.0", "@sinclair/typebox": "^0.28.20", + "@types/node": "^22.14.1", "fastify": "^4.3.0", "fastify-metrics": "^10.2.0", "node-pg-migrate": "^6.2.2", @@ -1333,7 +1378,7 @@ "api-toolkit-git-info": "bin/api-toolkit-git-info.js" }, "engines": { - "node": ">=18" + "node": ">=22" } }, "node_modules/@hirosystems/api-toolkit/node_modules/@fastify/cors": { @@ -1502,6 +1547,16 @@ "node": ">=12" } }, + "node_modules/@hirosystems/salt-n-pepper-client": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@hirosystems/salt-n-pepper-client/-/salt-n-pepper-client-1.1.1.tgz", + "integrity": "sha512-xQqHAX0xIrpGFD9Hc4sqbSZ862vE1yDUT67RFv6ok7hd+OFJUNfCrfr+nMQJZK8qSt3jlmZJt/5R38+D6pQvvw==", + "license": "GPL-3.0-only", + "dependencies": { + "@hirosystems/api-toolkit": "^1.7.2", + "redis": "^4.7.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -2430,6 +2485,17 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@jsdevtools/ono": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", @@ -3237,6 +3303,65 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", + "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@redocly/ajv": { "version": "8.11.2", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", @@ -4222,6 +4347,29 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.39", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.39.tgz", + "integrity": "sha512-uMPmxehH6ofeYjaslASPtjvyH8FRJdM9fZ+hjhGzL4Jq3bGjr9D7TKmp9soSwgFncNk0HOwmyBxjqOb3ikjjsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/dotenv-flow": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@types/dotenv-flow/-/dotenv-flow-3.2.0.tgz", @@ -4347,11 +4495,12 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.11.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.4.tgz", - "integrity": "sha512-6I0fMH8Aoy2lOejL3s4LhyIYX34DPwY8bl5xlNjBvUEk8OHrcuzsFt+Ied4LvJihbtXPM+8zUqdydfIti86v9g==", + "version": "22.15.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.21.tgz", + "integrity": "sha512-EV/37Td6c+MgKAbkcLG6vqZ2zEYHD7bvSrzqqs2RIhbA6w3x+Dqz8MZM3sP6kGTeLrdoOgKZe+Xja7tUB2DNkQ==", + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/node-fetch": { @@ -4378,6 +4527,12 @@ "node": ">= 6" } }, + "node_modules/@types/node/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -4494,6 +4649,26 @@ "@types/node": "*" } }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.103", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.103.tgz", + "integrity": "sha512-hHTHp+sEz6SxFsp+SA+Tqrua3AbmlAw+Y//aEwdHrdZkYVRWdvWD3y5uPZ0flYOkgskaFWqZ/YGFm3FaFQ0pRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -5807,6 +5982,18 @@ "bs58": "^5.0.0" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/bn.js": { "version": "4.12.0", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", @@ -6013,6 +6200,16 @@ "node": ">=6.14.2" } }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -6377,6 +6574,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -6775,6 +6981,21 @@ "typescript": ">=4" } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-hash": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", @@ -7302,6 +7523,55 @@ "node": ">= 6.0.0" } }, + "node_modules/docker-modem": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", + "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", + "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.6", + "protobufjs": "^7.3.2", + "tar-fs": "~2.1.2", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -9060,6 +9330,13 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -9131,6 +9408,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -12918,6 +13204,13 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, "node_modules/mnemonist": { "version": "0.39.6", "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.6.tgz", @@ -12994,6 +13287,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/nan": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", + "integrity": "sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -13466,31 +13767,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, - "peer": true, - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, - "node_modules/npmlog": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", - "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", - "deprecated": "This package is no longer supported.", - "dependencies": { - "are-we-there-yet": "^2.0.0", - "console-control-strings": "^1.1.0", - "gauge": "^3.0.0", - "set-blocking": "^2.0.0" - } - }, "node_modules/oas-kit-common": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/oas-kit-common/-/oas-kit-common-1.0.8.tgz", @@ -15175,6 +15451,23 @@ "node": ">= 12.13.0" } }, + "node_modules/redis": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz", + "integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==", + "license": "MIT", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.1", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/redoc": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/redoc/-/redoc-2.4.0.tgz", @@ -16232,6 +16525,13 @@ "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==", "dev": true }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true, + "license": "ISC" + }, "node_modules/split2": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", @@ -16245,6 +16545,24 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" }, + "node_modules/ssh2": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", + "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.20.0" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -16744,6 +17062,43 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", diff --git a/package.json b/package.json index 73e122c3a..0dab23af5 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:rpc": "npm run test -- --config ./tests/jest.config.rpc.js", "test:event-replay": "npm run test -- --config ./tests/jest.config.event-replay.js", "test:btc-faucet": "npm run test -- --config ./tests/jest.config.btc-faucet.js", + "test:snp": "npm run test -- --config ./tests/jest.config.snp.js", "test:integration": "concurrently \"docker compose -f docker/docker-compose.dev.postgres.yml up --force-recreate -V\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.js --no-cache --runInBand; npm run devenv:stop:pg\"", "test:integration:subnets": "concurrently --hide \"devenv:deploy:subnets\" \"npm:devenv:deploy:subnets\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.subnets.js --no-cache --runInBand; npm run devenv:stop:subnets\"", "test:integration:2.5": "concurrently --hide \"devenv:deploy-krypton\" \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.2.5.js --no-cache --runInBand; npm run devenv:stop-krypton\"", @@ -73,7 +74,7 @@ "homepage": "https://github.com/hirosystems/stacks-blockchain-api#readme", "prettier": "@stacks/prettier-config", "engines": { - "node": ">=20" + "node": ">=22" }, "engineStrict": true, "commitlint": { @@ -94,7 +95,8 @@ "@fastify/multipart": "8.3.1", "@fastify/swagger": "8.15.0", "@fastify/type-provider-typebox": "4.0.0", - "@hirosystems/api-toolkit": "1.6.2", + "@hirosystems/api-toolkit": "1.9.0", + "@hirosystems/salt-n-pepper-client": "1.1.1", "@scure/base": "1.1.1", "@sinclair/typebox": "0.32.35", "@stacks/common": "6.10.0", @@ -161,12 +163,13 @@ "@redocly/cli": "1.34.1", "@stacks/eslint-config": "1.2.0", "@types/cors": "2.8.12", + "@types/dockerode": "3.3.39", "@types/dotenv-flow": "3.2.0", "@types/elliptic": "6.4.14", "@types/express": "4.17.13", "@types/is-ci": "3.0.0", "@types/jest": "29.5.6", - "@types/node": "20.11.4", + "@types/node": "22.15.21", "@types/node-fetch": "2.5.12", "@types/pg": "7.14.11", "@types/pg-copy-streams": "1.2.1", @@ -179,6 +182,7 @@ "@typescript-eslint/parser": "5.51.0", "concurrently": "7.3.0", "docker-compose": "0.24.8", + "dockerode": "4.0.6", "eslint": "8.29.0", "eslint-plugin-prettier": "4.2.1", "eslint-plugin-tsdoc": "0.2.17", diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index 41c0ef87d..19c8b291c 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -3644,6 +3644,17 @@ export class PgWriteStore extends PgStore { } } + async getLastIngestedSnpRedisMsgId(): Promise { + const [{ last_redis_msg_id }] = await this.sql< + { last_redis_msg_id: string }[] + >`SELECT last_redis_msg_id FROM snp_state`; + return last_redis_msg_id; + } + + async updateLastIngestedSnpRedisMsgId(sql: PgSqlClient, msgId: string): Promise { + await sql`UPDATE snp_state SET last_redis_msg_id = ${msgId}`; + } + async close(args?: { timeout?: number }): Promise { if (this._debounceMempoolStat.debounce) { clearTimeout(this._debounceMempoolStat.debounce); diff --git a/src/event-stream/event-server.ts b/src/event-stream/event-server.ts index 11a3fbd6a..0495b7234 100644 --- a/src/event-stream/event-server.ts +++ b/src/event-stream/event-server.ts @@ -1,6 +1,6 @@ import { inspect } from 'util'; import * as net from 'net'; -import Fastify, { FastifyRequest, FastifyServerOptions } from 'fastify'; +import Fastify, { FastifyInstance, FastifyRequest, FastifyServerOptions } from 'fastify'; import PQueue from 'p-queue'; import * as prom from 'prom-client'; import { @@ -751,8 +751,10 @@ function createMessageProcessorQueue(db: PgWriteStore): EventMessageHandler { return handler; } -export type EventStreamServer = net.Server & { +export type EventStreamServer = { + server: net.Server; serverAddress: net.AddressInfo; + fastifyInstance: FastifyInstance; closeAsync: () => Promise; }; @@ -983,10 +985,12 @@ export async function startEventServer(opts: { logger.info('Closing event observer server...'); await app.close(); }; - const eventStreamServer: EventStreamServer = Object.assign(app.server, { + const eventStreamServer: EventStreamServer = { + server: app.server, serverAddress: app.addresses()[0], + fastifyInstance: app, closeAsync: closeFn, - }); + }; return eventStreamServer; } diff --git a/src/event-stream/snp-event-stream.ts b/src/event-stream/snp-event-stream.ts new file mode 100644 index 000000000..16ede151b --- /dev/null +++ b/src/event-stream/snp-event-stream.ts @@ -0,0 +1,75 @@ +import { SERVER_VERSION } from '@hirosystems/api-toolkit'; +import { logger as defaultLogger } from '@hirosystems/api-toolkit'; +import { StacksEventStream, StacksEventStreamType } from '@hirosystems/salt-n-pepper-client'; +import { EventEmitter } from 'node:events'; +import { EventStreamServer } from './event-server'; +import { PgWriteStore } from '../datastore/pg-write-store'; + +export class SnpEventStreamHandler { + db: PgWriteStore; + eventServer: EventStreamServer; + logger = defaultLogger.child({ name: 'SnpEventStreamHandler' }); + snpClientStream: StacksEventStream; + redisUrl: string; + redisStreamPrefix: string | undefined; + + readonly events = new EventEmitter<{ + processedMessage: [{ msgId: string }]; + }>(); + + constructor(opts: { db: PgWriteStore; eventServer: EventStreamServer; lastMessageId: string }) { + this.db = opts.db; + this.eventServer = opts.eventServer; + + this.redisUrl = process.env.SNP_REDIS_URL as string; + if (!this.redisUrl) { + throw new Error('SNP_REDIS_URL environment variable is not set'); + } + + this.redisStreamPrefix = process.env.SNP_REDIS_STREAM_KEY_PREFIX; + + this.logger.info(`SNP streaming enabled, lastMsgId: ${opts.lastMessageId}`); + + const appName = `stacks-blockchain-api ${SERVER_VERSION.tag} (${SERVER_VERSION.branch}:${SERVER_VERSION.commit})`; + + this.snpClientStream = new StacksEventStream({ + redisUrl: this.redisUrl, + redisStreamPrefix: this.redisStreamPrefix, + eventStreamType: StacksEventStreamType.all, + lastMessageId: opts.lastMessageId, + appName, + }); + } + + async start() { + this.logger.info(`Connecting to SNP event stream at ${this.redisUrl} ...`); + await this.snpClientStream.connect({ waitForReady: true }); + this.snpClientStream.start(async (messageId, timestamp, path, body) => { + return this.handleMsg(messageId, timestamp, path, body); + }); + } + + async handleMsg(messageId: string, timestamp: string, path: string, body: any) { + this.logger.debug(`Received SNP stream event ${path}, msgId: ${messageId}`); + + const response = await this.eventServer.fastifyInstance.inject({ + method: 'POST', + url: path, + payload: body, + }); + + if (response.statusCode < 200 || response.statusCode > 299) { + const errorMessage = `Failed to process SNP message ${messageId} at path ${path}, status: ${response.statusCode}, body: ${response.body}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + await this.db.updateLastIngestedSnpRedisMsgId(this.db.sql, messageId); + + this.events.emit('processedMessage', { msgId: messageId }); + } + + async stop(): Promise { + await this.snpClientStream.stop(); + } +} diff --git a/src/helpers.ts b/src/helpers.ts index ae6fb5d94..71c1e5054 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -14,6 +14,7 @@ import { logger } from './logger'; import { has0xPrefix, isDevEnv, numberToHex } from '@hirosystems/api-toolkit'; import { StacksNetwork, StacksTestnet } from '@stacks/network'; import { getStacksTestnetNetwork } from './api/routes/debug'; +import { EventEmitter, addAbortListener } from 'node:events'; export const apiDocumentationUrl = process.env.API_DOCS_URL; diff --git a/src/index.ts b/src/index.ts index a1c0b1ee0..914ec0d00 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,7 @@ import { timeout, } from '@hirosystems/api-toolkit'; import Fastify from 'fastify'; +import { SnpEventStreamHandler } from './event-stream/snp-event-stream'; enum StacksApiMode { /** @@ -161,6 +162,21 @@ async function init(): Promise { monitorCoreRpcConnection().catch(error => { logger.error(error, 'Error monitoring RPC connection'); }); + + const snpEnabled = parseBoolean(process.env['SNP_EVENT_STREAMING']); + if (snpEnabled) { + const lastRedisMsgId = await dbWriteStore.getLastIngestedSnpRedisMsgId(); + const snpStream = new SnpEventStreamHandler({ + lastMessageId: lastRedisMsgId, + db: dbWriteStore, + eventServer, + }); + registerShutdownConfig({ + name: 'SNP client stream', + handler: () => snpStream.stop(), + forceKillable: false, + }); + } } if ( diff --git a/tests/jest.config.snp.js b/tests/jest.config.snp.js new file mode 100644 index 000000000..94479e10b --- /dev/null +++ b/tests/jest.config.snp.js @@ -0,0 +1,16 @@ +/** @type {import('jest').Config} */ +const config = { + preset: 'ts-jest', + testEnvironment: 'node', + rootDir: `${require('path').dirname(__dirname)}`, + testMatch: ['/tests/snp/**/*.test.ts'], + collectCoverageFrom: ['/src/**/*.ts'], + coverageDirectory: '/coverage', + globalSetup: '/tests/snp/jest-global-setup.ts', + globalTeardown: '/tests/snp/jest-global-teardown.ts', + testTimeout: 60_000, + verbose: true, + bail: true, +}; + +module.exports = config; diff --git a/tests/rosetta-cli-construction/validate-construction.test.ts b/tests/rosetta-cli-construction/validate-construction.test.ts index 109ef8fbb..d0b8cb55f 100644 --- a/tests/rosetta-cli-construction/validate-construction.test.ts +++ b/tests/rosetta-cli-construction/validate-construction.test.ts @@ -1,5 +1,5 @@ import { ApiServer, startApiServer } from '../../src/api/init'; -import { startEventServer } from '../../src/event-stream/event-server'; +import { EventStreamServer, startEventServer } from '../../src/event-stream/event-server'; import { Server } from 'net'; import { AnchorMode, @@ -30,7 +30,7 @@ const stacksNetwork = GetStacksTestnetNetwork(); describe('Rosetta API', () => { let db: PgWriteStore; - let eventServer: Server; + let eventServer: EventStreamServer; let api: ApiServer; let rosettaOutput: any; let nonceJar: NonceJar; @@ -128,7 +128,7 @@ describe('Rosetta API', () => { }); console.log('compose down result:', composeDownResult); - await new Promise(resolve => eventServer.close(() => resolve(true))); + await new Promise(resolve => eventServer.server.close(() => resolve(true))); await api.terminate(); await db?.close(); await migrate('down'); diff --git a/tests/rosetta-cli-data/validate-rosetta.test.ts b/tests/rosetta-cli-data/validate-rosetta.test.ts index 683fb858c..7bcf5b330 100644 --- a/tests/rosetta-cli-data/validate-rosetta.test.ts +++ b/tests/rosetta-cli-data/validate-rosetta.test.ts @@ -238,7 +238,7 @@ describe('Rosetta API', () => { }); console.log('compose down result:', composeDownResult); - await new Promise(resolve => eventServer.close(() => resolve())); + await new Promise(resolve => eventServer.server.close(() => resolve())); await api.terminate(); await db?.close(); await migrate('down'); diff --git a/tests/snp/dumps/epoch-3-transition.tsv.gz b/tests/snp/dumps/epoch-3-transition.tsv.gz new file mode 100644 index 000000000..aea57bfed Binary files /dev/null and b/tests/snp/dumps/epoch-3-transition.tsv.gz differ diff --git a/tests/snp/jest-global-setup.ts b/tests/snp/jest-global-setup.ts new file mode 100644 index 000000000..bd94c537f --- /dev/null +++ b/tests/snp/jest-global-setup.ts @@ -0,0 +1,251 @@ +import * as net from 'node:net'; +import * as Docker from 'dockerode'; +import { connectPostgres, timeout } from '@hirosystems/api-toolkit'; +import { createClient } from 'redis'; +import { loadDotEnv } from '../../src/helpers'; +import { fetch } from 'undici'; + +const testContainerLabel = 'stacks-blockchain-api-tests'; + +function isDockerImagePulled(docker: Docker, imgName: string) { + return docker + .getImage(imgName) + .inspect() + .then( + () => true, + () => false + ); +} + +async function pullDockerImage(docker: Docker, imgName: string) { + await new Promise((resolve, reject) => { + docker.pull(imgName, {}, (err, stream) => { + if (err || !stream) return reject(err as Error); + docker.modem.followProgress(stream, err => (err ? reject(err) : resolve()), console.log); + }); + }); +} + +async function pruneContainers(docker: Docker, label: string) { + const containers = await docker.listContainers({ all: true, filters: { label: [label] } }); + for (const container of containers) { + const c = docker.getContainer(container.Id); + await c.remove({ v: true, force: true }); + } + await docker.pruneContainers({ filters: { label: [label] } }); + return containers.length; +} + +async function startContainer(args: { + docker: Docker; + image: string; + ports: { container: number; host: number }[]; + env: string[]; +}) { + const { docker, image, ports, env } = args; + try { + const imgPulled = await isDockerImagePulled(docker, image); + if (!imgPulled) { + console.log(`Pulling ${image} image...`); + await pullDockerImage(docker, image); + } + console.log(`Creating ${image} container...`); + const exposedPorts = ports.reduce( + (acc, port) => ({ ...acc, [`${port.container}/tcp`]: {} }), + {} + ); + const portBindings: Record = {}; + ports.forEach(port => { + portBindings[`${port.container}/tcp`] = [{ HostPort: port.host.toString() }]; + }); + const container = await docker.createContainer({ + Labels: { [testContainerLabel]: 'true' }, + Image: image, + ExposedPorts: exposedPorts, + HostConfig: { + PortBindings: portBindings, + ExtraHosts: ['host.docker.internal:host-gateway'], + }, + Env: env, + }); + + console.log(`Starting ${image} container...`); + await container.start(); + + console.log(`${image} container started on ports ${JSON.stringify(ports)}`); + + const containers: { id: string; image: string }[] = + (globalThis as any).__TEST_DOCKER_CONTAINERS ?? []; + containers.push({ id: container.id, image }); + Object.assign(globalThis, { __TEST_DOCKER_CONTAINERS: containers }); + + return { image, containerId: container.id }; + } catch (error) { + console.error('Error starting PostgreSQL container:', error); + throw error; + } +} + +async function findFreePorts(count: number) { + const servers = await Promise.all( + Array.from({ length: count }, () => { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.listen(0, () => resolve(server)).on('error', reject); + }); + }) + ); + const ports = await Promise.all( + servers.map(server => { + const { port } = server.address() as net.AddressInfo; + return new Promise(resolve => server.close(() => resolve(port))); + }) + ); + return ports; +} + +// Helper function to wait for PostgreSQL to be ready +async function waitForPostgres(): Promise { + const sql = await connectPostgres({ + usageName: testContainerLabel, + connectionArgs: { + host: process.env.PG_HOST, + port: parseInt(process.env.PG_PORT as string), + user: process.env.PG_USER, + password: process.env.PG_PASSWORD, + database: process.env.PG_DATABASE, + }, + }); + await sql`SELECT 1`; + await sql.end(); + console.log('Postgres is ready'); +} + +async function waitForRedis(): Promise { + const redisClient = createClient({ + url: process.env['SNP_REDIS_URL'], + name: 'stacks-blockchain-api-server-tests', + }); + redisClient.on('error', (err: Error) => console.error(`Redis not ready: ${err}`)); + redisClient.once('ready', () => console.log('Connected to Redis successfully!')); + while (true) { + try { + await redisClient.connect(); + break; + } catch (error) { + console.error(`Failed to connect to Redis:`, error); + await timeout(100); + } + } + await redisClient.disconnect(); +} + +async function waitForSNP(): Promise { + const snpUrl = process.env['SNP_OBSERVER_URL']; + if (!snpUrl) { + throw new Error('SNP_OBSERVER_URL is not set'); + } + while (true) { + try { + const response = await fetch(snpUrl + '/status'); + if (response.ok) { + console.log('SNP is ready'); + break; + } else { + console.error(`SNP not ready at ${snpUrl}: ${response.statusText}`); + } + } catch (error) { + console.error(`SNP not ready at ${snpUrl}: ${error}`); + } + await timeout(100); + } +} + +// Jest global setup +// ts-unused-exports:disable-next-line +export default async function setup(): Promise { + loadDotEnv(); + + // use a random PGSCHEMA for each test to avoid conflicts + process.env.PG_SCHEMA = `test_${crypto.randomUUID()}`; + + const docker = new Docker(); + const prunedCount = await pruneContainers(docker, testContainerLabel); + if (prunedCount > 0) { + console.log(`Pruned ${prunedCount} existing test docker containers`); + } + + const [pgHostPort, redisHostPort, snpHostPort] = await findFreePorts(3); + + const startPg = async () => { + const pgPort = 5432; + const pgContainer = await startContainer({ + docker, + image: 'postgres:17', + ports: [{ container: pgPort, host: pgHostPort }], + env: [ + `POSTGRES_USER=${process.env.PG_USER}`, + `POSTGRES_PASSWORD=${process.env.PG_PASSWORD}`, + `POSTGRES_DB=${process.env.PG_DATABASE}`, + `PGPORT=${pgPort}`, + ], + }); + process.env.PG_PORT = pgHostPort.toString(); + process.env['_PG_DOCKER_CONTAINER_ID'] = pgContainer.containerId; + // Wait for the database to be ready + await waitForPostgres(); + }; + + const startRedis = async () => { + const redisPort = 6379; + const redisContainer = await startContainer({ + docker, + image: 'redis:7', + ports: [{ container: redisPort, host: redisHostPort }], + env: [], + }); + process.env['SNP_REDIS_URL'] = `redis://127.0.0.1:${redisHostPort}`; + process.env['_REDIS_DOCKER_CONTAINER_ID'] = redisContainer.containerId; + // wait for redis to be ready + await waitForRedis(); + }; + + const startServices = await Promise.allSettled([startPg(), startRedis()]); + for (const service of startServices) { + if (service.status === 'rejected') { + throw service.reason; + } + } + + const startSNP = async () => { + const snpObserverPort = 3022; + process.env.SNP_REDIS_STREAM_KEY_PREFIX = `test_${crypto.randomUUID()}`; + console.log(`Using REDIS_STREAM_KEY_PREFIX: ${process.env.SNP_REDIS_STREAM_KEY_PREFIX}`); + const snpContainer = await startContainer({ + docker, + image: 'hirosystems/salt-n-pepper:1.1.1', + ports: [{ container: snpObserverPort, host: snpHostPort }], + env: [ + `OBSERVER_HOST=0.0.0.0`, + `OBSERVER_PORT=${snpObserverPort}`, + `REDIS_URL=redis://host.docker.internal:${redisHostPort}`, + `REDIS_STREAM_KEY_PREFIX=${process.env.SNP_REDIS_STREAM_KEY_PREFIX}`, + `PGHOST=host.docker.internal`, + `PGPORT=${process.env.PG_PORT}`, + `PGUSER=${process.env.PG_USER}`, + `PGPASSWORD=${process.env.PG_PASSWORD}`, + `PGDATABASE=${process.env.PG_DATABASE}`, + `PGSCHEMA=test_snp_${crypto.randomUUID()}`, + ], + }); + process.env['SNP_OBSERVER_URL'] = `http://127.0.0.1:${snpHostPort}`; + process.env['_SNP_DOCKER_CONTAINER_ID'] = snpContainer.containerId; + // Wait for snp to be ready + await waitForSNP(); + }; + await startSNP(); + + // Setup misc required env vars + process.env.STACKS_NODE_RPC_HOST = ''; + process.env.STACKS_NODE_RPC_PORT = '1'; +} diff --git a/tests/snp/jest-global-teardown.ts b/tests/snp/jest-global-teardown.ts new file mode 100644 index 000000000..668e40644 --- /dev/null +++ b/tests/snp/jest-global-teardown.ts @@ -0,0 +1,15 @@ +import * as Docker from 'dockerode'; + +// Jest global teardown to stop and remove the container +// ts-unused-exports:disable-next-line +export default async function teardown(): Promise { + const containers: { id: string; image: string }[] = + (globalThis as any).__TEST_DOCKER_CONTAINERS ?? []; + for (const { id, image } of containers) { + console.log(`Stopping and removing container ${image} - ${id}...`); + const docker = new Docker(); + const container = docker.getContainer(id); + await container.remove({ v: true, force: true }); + console.log(`Test docker container ${image} ${id} stopped and removed`); + } +} diff --git a/tests/snp/snp-ingestion.test.ts b/tests/snp/snp-ingestion.test.ts new file mode 100644 index 000000000..522f4d53a --- /dev/null +++ b/tests/snp/snp-ingestion.test.ts @@ -0,0 +1,130 @@ +import * as readline from 'node:readline/promises'; +import * as fs from 'node:fs'; +import * as zlib from 'node:zlib'; +import * as assert from 'node:assert/strict'; + +import { ChainID } from '@stacks/transactions'; +import { ApiServer, startApiServer } from '../../src/api/init'; +import { EventStreamServer, startEventServer } from '../../src/event-stream/event-server'; +import { PgWriteStore } from '../../src/datastore/pg-write-store'; +import { onceWhen, PgSqlClient } from '@hirosystems/api-toolkit'; +import { migrate } from '../utils/test-helpers'; +import { SnpEventStreamHandler } from '../../src/event-stream/snp-event-stream'; +import { fetch } from 'undici'; +import * as supertest from 'supertest'; + +describe('SNP integration tests', () => { + let snpObserverUrl: string; + let db: PgWriteStore; + let client: PgSqlClient; + let eventServer: EventStreamServer; + let apiServer: ApiServer; + + const sampleEventsLastMsgId = '238-0'; + const sampleEventsLastBlockHeight = 50; + const sampleEventsLastBlockHash = + '0x5705546ec6741f77957bb3e73bf795dcf120c0a869c1d408396e7e30a3b2f94f'; + + beforeAll(async () => { + snpObserverUrl = process.env['SNP_OBSERVER_URL'] as string; + + await migrate('up'); + + db = await PgWriteStore.connect({ + usageName: 'tests', + withNotifier: false, + skipMigrations: true, + }); + client = db.sql; + + eventServer = await startEventServer({ + datastore: db, + chainId: ChainID.Mainnet, + serverHost: '127.0.0.1', + serverPort: 0, + }); + apiServer = await startApiServer({ + datastore: db, + chainId: ChainID.Mainnet, + }); + }); + + afterAll(async () => { + await apiServer.terminate(); + await eventServer.closeAsync(); + await db?.close(); + await migrate('down'); + }); + + test('populate SNP server data', async () => { + const payloadDumpFile = './tests/snp/dumps/epoch-3-transition.tsv.gz'; + // const payloadDumpFile = './tests/snp/dumps/stackerdb-sample-events.tsv.gz'; + const rl = readline.createInterface({ + input: fs.createReadStream(payloadDumpFile).pipe(zlib.createGunzip()), + crlfDelay: Infinity, + }); + for await (const line of rl) { + const [_id, timestamp, path, payload] = line.split('\t'); + // use fetch to POST the payload to the SNP event observer server + try { + const res = await fetch(snpObserverUrl + path, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Original-Timestamp': timestamp }, + body: payload, + }); + if (res.status !== 200) { + throw new Error(`Failed to POST event: ${path} - ${payload.slice(0, 100)}`); + } + } catch (error) { + console.error(`Error posting event: ${error}`, error); + throw error; + } + } + rl.close(); + }); + + test('ingest SNP events', async () => { + const lastMsgId = await db.getLastIngestedSnpRedisMsgId(); + expect(lastMsgId).toBe('0'); + const snpClient = new SnpEventStreamHandler({ + db, + eventServer, + lastMessageId: lastMsgId, + }); + + await snpClient.start(); + + // wait for last msgID to be processed + const [{ msgId: lastMsgProcessed }] = await onceWhen( + snpClient.events, + 'processedMessage', + ({ msgId }) => { + return msgId === sampleEventsLastMsgId; + } + ); + + expect(lastMsgProcessed).toBe(sampleEventsLastMsgId); + + await snpClient.stop(); + }); + + test('validate all events ingested', async () => { + const finalPostgresMsgId = await db.getLastIngestedSnpRedisMsgId(); + expect(finalPostgresMsgId).toBe(sampleEventsLastMsgId); + }); + + test('validate blocks ingested', async () => { + const chainTip = await db.getCurrentBlockHeight(); + assert(chainTip.found); + expect(chainTip.result).toBe(sampleEventsLastBlockHeight); + }); + + test('test block API fetch', async () => { + const response = await supertest(apiServer.server) + .get(`/extended/v1/block/by_height/${sampleEventsLastBlockHeight}`) + .expect(200); + expect(response.body).toMatchObject({ + hash: sampleEventsLastBlockHash, + }); + }); +}); diff --git a/tests/utils/test-helpers.ts b/tests/utils/test-helpers.ts index eb80f7da5..0245f8920 100644 --- a/tests/utils/test-helpers.ts +++ b/tests/utils/test-helpers.ts @@ -45,14 +45,23 @@ import { CoreRpcPoxInfo, StacksCoreRpcClient } from '../../src/core-rpc/client'; import { DbBlock, DbTx, DbTxStatus } from '../../src/datastore/common'; import { PgWriteStore } from '../../src/datastore/pg-write-store'; import { BitcoinAddressFormat, ECPair, getBitcoinAddressFromKey } from '../../src/ec-helpers'; -import { coerceToBuffer, runMigrations, timeout } from '@hirosystems/api-toolkit'; +import { coerceToBuffer, connectPostgres, runMigrations, timeout } from '@hirosystems/api-toolkit'; import { MIGRATIONS_DIR } from '../../src/datastore/pg-store'; import { getConnectionArgs } from '../../src/datastore/connection'; import { AddressStxBalance } from '../../src/api/schemas/entities/addresses'; import { ServerStatusResponse } from '../../src/api/schemas/responses/responses'; export async function migrate(direction: 'up' | 'down') { - await runMigrations(MIGRATIONS_DIR, direction, getConnectionArgs()); + const connArgs = getConnectionArgs(); + if (typeof connArgs !== 'string' && connArgs.schema) { + const sql = await connectPostgres({ + usageName: 'tests-migrations-setup', + connectionArgs: connArgs, + }); + await sql`CREATE SCHEMA IF NOT EXISTS ${sql(connArgs.schema)}`; + await sql.end(); + } + await runMigrations(MIGRATIONS_DIR, direction, connArgs); } export interface TestEnvContext {