Update dockerfile dependencies #20439
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flow build and test | |
| on: | |
| pull_request: | |
| branches: [main] | |
| # Ignore changes to files that don't require builds or validations. | |
| # If more complex build avoidance is required, or for other events than pull_request, push, and pull_request_target, | |
| # replace this with a gate step in the job. | |
| # TODO: Re-enable build avoidance once we figure out how not to block the merge on the flow checks. | |
| #paths-ignore: &ignore_paths | |
| # - '.claude/**' | |
| # - 'README.md' | |
| push: | |
| branches: [main] | |
| #paths-ignore: *ignore_paths | |
| pull_request_target: | |
| branches: [main] | |
| #paths-ignore: *ignore_paths | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| flow_test: | |
| name: flow_test (${{ matrix.runner }}, ${{ matrix.db-version.pg }}, ${{ matrix.db-version.mysql }}, ${{ matrix.db-version.mongo }}, ${{ matrix.db-version.ch }}) | |
| if: | | |
| (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || | |
| (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) || | |
| github.event_name == 'push' | |
| environment: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && 'external-contributor' || null }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [ubuntu-latest-16-cores] | |
| db-version: [ | |
| {pg: 16, mysql: 'mysql-gtid', mongo: '6.0', ch: 'lts'}, | |
| {pg: 17, mysql: 'mysql-pos', mongo: '7.0', ch: 'stable'}, | |
| {pg: 18, mysql: 'maria-pos', mongo: '8.0', ch: 'latest'}, | |
| {pg: 18, mysql: 'maria-gtid', mongo: '8.0', ch: 'stable'}, | |
| ] | |
| # Per-version container settings consumed by the "MySQL" step, keyed by the matrix | |
| # db-version.mysql value. Grouped under "mysql" since these are MySQL/MariaDB families; | |
| # only the databases handled by the MySQL step are listed here. Wrapped in a single-item | |
| # list because matrix values must be arrays; it stays a single shared value (no extra jobs). | |
| version-configs: | |
| - mysql: | |
| mysql-gtid: | |
| img: 'mysql:9.5' | |
| env: ['MYSQL_ROOT_PASSWORD=cipass'] | |
| parameters: [] | |
| mysql-pos: | |
| img: 'mysql:5.7' | |
| env: ['MYSQL_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log_bin=mysql-bin', '--server-id=1', '--bind-address=::'] | |
| maria-pos: | |
| img: 'mariadb:lts-ubi9@sha256:55a81b2d791d2ff8ad33fef413d9e45e0ac57a951127e0cfc69a8e59f922ba6e' | |
| env: ['MARIADB_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log-bin=maria'] | |
| maria-gtid: | |
| img: 'mariadb:lts-ubi9@sha256:55a81b2d791d2ff8ad33fef413d9e45e0ac57a951127e0cfc69a8e59f922ba6e' | |
| env: ['MARIADB_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log-bin=maria', '--gtid-strict-mode=ON'] | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| services: | |
| catalog: | |
| image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine | |
| ports: | |
| - 5432:5432 | |
| env: | |
| PGUSER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_INITDB_ARGS: --locale=C.UTF-8 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| catalog2: | |
| image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine | |
| ports: | |
| - 5437:5432 | |
| env: | |
| PGUSER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_INITDB_ARGS: --locale=C.UTF-8 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redpanda: | |
| image: redpandadata/redpanda@sha256:394cee6e2076a23f9b7b01f5d68228d52429fe683fd087de11c8a68ff5287037 | |
| ports: | |
| - 9092:9092 | |
| - 9644:9644 | |
| elasticsearch: | |
| image: elasticsearch:9.4.2@sha256:715c5d552328d83dcee6082bd5125327332a649f8e0aedb5fe0553d7be75bf21 | |
| ports: | |
| - 9200:9200 | |
| env: | |
| discovery.type: single-node | |
| xpack.security.enabled: false | |
| xpack.security.enrollment.enabled: false | |
| otelcol: | |
| image: otel/opentelemetry-collector-contrib:0.154.0@sha256:b3079f45e19bdb7326bf49cdddce6cf60dfd865138db39f2733ea48ab17bc4cb | |
| ports: | |
| - 4317:4317 | |
| toxiproxy: | |
| image: ghcr.io/shopify/toxiproxy:2.12.0@sha256:9378ed52a28bc50edc1350f936f518f31fa95f0d15917d6eb40b8e376d1a214e | |
| ports: | |
| - 18474:8474 | |
| - 9902:9902 | |
| - 9904:9904 | |
| - 9903:9903 | |
| - 10001:10001 | |
| - 12001:12001 | |
| - 12002:12002 | |
| - 12003:12003 | |
| - 12004:12004 | |
| - 12005:12005 | |
| - 14001:14001 | |
| - 14002:14002 | |
| - 14003:14003 | |
| openssh: | |
| image: linuxserver/openssh-server:latest@sha256:835c1f321ff16b10e1e3d82ec3808a5253f02f73a98d7dca5b0786184f44e69a | |
| ports: | |
| - 2222:2222 | |
| env: | |
| PUID: 1000 | |
| PGID: 1000 | |
| TZ: Etc/UTC | |
| USER_NAME: testuser | |
| USER_PASSWORD: testpass | |
| PASSWORD_ACCESS: true | |
| DOCKER_MODS: linuxserver/mods:openssh-server-ssh-tunnel | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.ref }} | |
| - name: generate or hydrate protos | |
| uses: ./.github/actions/genprotos | |
| - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 | |
| with: | |
| go-version: '1.26.4' | |
| cache-dependency-path: flow/go.sum | |
| - name: install lib-geos and pg_dump | |
| run: | | |
| # No need to update man pages on package install | |
| sudo apt-get remove --purge man-db | |
| # Add PGDG apt repo for latest PostgreSQL client packages | |
| sudo install -d /usr/share/postgresql-common/pgdg | |
| sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ | |
| --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | |
| echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ | |
| https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | |
| | sudo tee /etc/apt/sources.list.d/pgdg.list | |
| sudo apt-get update | |
| sudo apt-get install -y libgeos-dev | |
| # pg_dump must be >= the server major version; install v18 so it | |
| # can dump PG 16, 17, and 18 (backward compatible). | |
| sudo apt-get install -y postgresql-client-18 | |
| echo /usr/lib/postgresql/18/bin >> $GITHUB_PATH | |
| - name: install retry tool | |
| run: | | |
| sudo apt-get install -y retry | |
| - run: go mod download | |
| working-directory: ./flow | |
| - name: setup gcp service account | |
| id: gcp-service-account | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "bq_service_account.json" | |
| json: ${{ secrets.GCP_GH_CI_PKEY }} | |
| - name: setup snowflake credentials | |
| id: sf-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "snowflake_creds.json" | |
| json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }} | |
| - name: setup GCS credentials | |
| id: gcs-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "gcs_creds.json" | |
| json: ${{ secrets.GCS_CREDS }} | |
| - name: setup Eventhubs credentials | |
| id: eventhubs-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "eh_creds.json" | |
| json: ${{ secrets.EH_CREDS }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 | |
| id: setup-aws | |
| with: | |
| audience: sts.amazonaws.com | |
| aws-region: us-west-2 | |
| role-to-assume: ${{ secrets.FLOW_TESTS_AWS_ROLE_ARN }} | |
| mask-aws-account-id: true | |
| output-credentials: true | |
| - name: MySQL | |
| env: | |
| DB_IMG: ${{ matrix.version-configs.mysql[matrix.db-version.mysql].img }} | |
| DB_ENV: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].env, ' -e ') }} | |
| DB_PARAMS: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].parameters, ' ') }} | |
| run: | | |
| docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 \ | |
| -e $DB_ENV $DB_IMG $DB_PARAMS | |
| - name: Mongo | |
| run: | | |
| echo "starting mongoDB..." | |
| docker run -d --rm --name mongo -p 27017:27017 mongo:${{ matrix.db-version.mongo }} \ | |
| bash -c 'openssl rand -base64 756 > /data/mongo.key && chmod 400 /data/mongo.key && mongod --replSet rs0 --oplogMinRetentionHours 24 --bind_ip_all --keyFile /data/mongo.key' | |
| until docker exec mongo mongosh --eval 'db.runCommand({ ping: 1 })' &> /dev/null; do | |
| echo "waiting for MongoDB to be ready..." | |
| sleep 2 | |
| done | |
| echo "initialize replica set" | |
| docker exec mongo mongosh --eval 'rs.initiate({ | |
| _id: "rs0", | |
| members: [{ _id: 0, host: "localhost:27017" }] | |
| })' | |
| echo "create admin user for writing data to mongo" | |
| docker exec mongo mongosh --eval ' | |
| db = db.getSiblingDB("admin"); | |
| db.createUser({ | |
| user: "admin", | |
| pwd: "admin", | |
| roles: ["root"] | |
| })' | |
| echo "create non-admin user for reading data from changestream" | |
| docker exec mongo mongosh -u admin -p admin --eval ' | |
| db = db.getSiblingDB("admin"); | |
| db.createUser({ | |
| user: "csuser", | |
| pwd: "cspass", | |
| roles: ["readAnyDatabase", "clusterMonitor"] | |
| })' | |
| - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: cache-minio | |
| with: | |
| path: ./minio | |
| key: ${{ runner.os }}-minio | |
| - name: Install MinIO Server | |
| if: steps.cache-minio.outputs.cache-hit != 'true' | |
| run: | | |
| curl -O https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio | |
| - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: cache-minio-client | |
| with: | |
| path: ./mc | |
| key: ${{ runner.os }}-minio-client | |
| - name: Install MinIO Client | |
| if: steps.cache-minio-client.outputs.cache-hit != 'true' | |
| run: | | |
| curl -O https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc | |
| - name: MinIO | |
| run: > | |
| mkdir -p certs minio-data && | |
| openssl genrsa -out certs/cert.key 2048 && | |
| openssl req -new -key certs/cert.key -out certs/cert.csr -subj /CN=minio.local && | |
| openssl x509 -req -days 3650 -in certs/cert.csr -signkey certs/cert.key -out certs/cert.crt && | |
| chown -R 1001 certs && | |
| ./minio server ./minio-data --certs-dir ./certs --address :9999 & | |
| sleep 2 && | |
| ./mc alias set myminiopeerdb http://localhost:9999 minio miniosecret && | |
| ./mc mb myminiopeerdb/peerdb | |
| env: | |
| MINIO_ROOT_USER: minio | |
| MINIO_ROOT_PASSWORD: miniosecret | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Generate ClickHouse TLS certificates | |
| run: | | |
| mkdir -p ch-certs | |
| # CA | |
| openssl genrsa -out ch-certs/ca.key 2048 | |
| openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA" | |
| # Server cert (CN=localhost, SAN for TLS 1.3) | |
| openssl genrsa -out ch-certs/server.key 2048 | |
| openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" | |
| openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall | |
| # Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention | |
| openssl genrsa -out ch-certs/tls.key 2048 | |
| openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client" | |
| openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt | |
| - name: create postgres extensions, increase logical replication limits, and setup catalog database | |
| run: > | |
| docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git && | |
| docker exec "${{ job.services.catalog.id }}" git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git /tmp/pgvector && | |
| docker exec "${{ job.services.catalog.id }}" sh -c 'cd /tmp/pgvector && make with_llvm=no && make with_llvm=no install' && | |
| docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;CREATE EXTENSION vector;" | |
| -c "ALTER SYSTEM SET wal_level=logical;" | |
| -c "ALTER SYSTEM SET max_replication_slots=192;" | |
| -c "ALTER SYSTEM SET max_wal_senders=256;" | |
| -c "ALTER SYSTEM SET max_connections=2048;" && | |
| (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) && | |
| docker restart "${{ job.services.catalog.id }}" | |
| env: | |
| PGPASSWORD: postgres | |
| - name: prepare secondary postgres for cross-cluster schema-dump tests | |
| run: > | |
| docker exec "${{ job.services.catalog2.id }}" psql -U postgres | |
| -c "ALTER SYSTEM SET wal_level=logical;" | |
| -c "ALTER SYSTEM SET max_replication_slots=192;" | |
| -c "ALTER SYSTEM SET max_wal_senders=256;" | |
| -c "ALTER SYSTEM SET max_connections=2048;" && | |
| docker restart "${{ job.services.catalog2.id }}" | |
| env: | |
| PGPASSWORD: postgres | |
| - name: set ClickHouse version | |
| id: ch-version | |
| run: | | |
| if [ "${{ matrix.db-version.ch }}" = "lts" ]; then | |
| echo "ch_version=v25.8.11.66-lts" >> $GITHUB_OUTPUT | |
| elif [ "${{ matrix.db-version.ch }}" = "stable" ]; then | |
| echo "ch_version=v25.12.4.35-stable" >> $GITHUB_OUTPUT | |
| elif [ "${{ matrix.db-version.ch }}" = "latest" ]; then | |
| # note: latest tag does not always reflect the latest version (could be an update on an lts), | |
| # but that is okay as we are only using it to invalidate the cache. | |
| echo "ch_version=$(curl -s https://api.github.com/repos/ClickHouse/ClickHouse/releases/latest | jq -r .tag_name)" >> $GITHUB_OUTPUT | |
| fi | |
| - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: cache-clickhouse | |
| with: | |
| path: ./clickhouse | |
| key: ${{ runner.os }}-clickhouse-${{ steps.ch-version.outputs.ch_version }} | |
| - name: Install ClickHouse | |
| if: steps.cache-clickhouse.outputs.cache-hit != 'true' | |
| run: | | |
| if [[ "${{ matrix.db-version.ch }}" = 'latest' ]]; then | |
| curl https://clickhouse.com | sh | |
| else | |
| VERSION="${{ steps.ch-version.outputs.ch_version }}" | |
| VERSION_NUM=$(echo ${VERSION#v} | cut -d'-' -f1) | |
| curl -sL https://github.com/ClickHouse/ClickHouse/releases/download/$VERSION/clickhouse-common-static-$VERSION_NUM-amd64.tgz | tar -xzf - | |
| mv "clickhouse-common-static-$VERSION_NUM/usr/bin/clickhouse" ./clickhouse | |
| rm -rf "clickhouse-common-static-$VERSION_NUM" | |
| chmod +x clickhouse | |
| fi | |
| - name: Run ClickHouse | |
| run: | | |
| ./clickhouse --version | |
| cat > config1.xml <<EOF | |
| <clickhouse> | |
| <profiles><default></default></profiles> | |
| <users> | |
| <default> | |
| <password></password> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| <named_collection_control>1</named_collection_control> | |
| </default> | |
| <peerdb_tls> | |
| <ssl_certificates> | |
| <common_name>peerdb-client</common_name> | |
| </ssl_certificates> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| </peerdb_tls> | |
| </users> | |
| <logger><level>none</level></logger> | |
| <path>var/lib/clickhouse</path> | |
| <tmp_path>var/lib/clickhouse/tmp</tmp_path> | |
| <user_files_path>var/lib/clickhouse/user_files</user_files_path> | |
| <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path> | |
| <tcp_port>9000</tcp_port> | |
| <tcp_port_secure>9440</tcp_port_secure> | |
| <http_port remove="1"/> | |
| <postgresql_port remove="1"/> | |
| <mysql_port remove="1"/> | |
| <openSSL> | |
| <server> | |
| <certificateFile>../ch-certs/server.crt</certificateFile> | |
| <privateKeyFile>../ch-certs/server.key</privateKeyFile> | |
| <caConfig>../ch-certs/ca.crt</caConfig> | |
| <verificationMode>relaxed</verificationMode> | |
| <cacheSessions>true</cacheSessions> | |
| <disableProtocols>sslv2,sslv3</disableProtocols> | |
| <preferServerCiphers>true</preferServerCiphers> | |
| </server> | |
| </openSSL> | |
| <macros> | |
| <shard>1</shard> | |
| <replica>1</replica> | |
| </macros> | |
| <zookeeper> | |
| <node> | |
| <host>localhost</host> | |
| <port>2181</port> | |
| </node> | |
| </zookeeper> | |
| <distributed_ddl> | |
| <path>/clickhouse/task_queue/ddl</path> | |
| </distributed_ddl> | |
| <remote_servers> | |
| <cicluster> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9000</port> | |
| </replica> | |
| </shard> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9001</port> | |
| </replica> | |
| </shard> | |
| </cicluster> | |
| </remote_servers> | |
| </clickhouse> | |
| EOF | |
| cat > config2.xml <<EOF | |
| <clickhouse> | |
| <profiles><default></default></profiles> | |
| <users> | |
| <default> | |
| <password></password> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| <named_collection_control>1</named_collection_control> | |
| </default> | |
| <peerdb_tls> | |
| <ssl_certificates> | |
| <common_name>peerdb-client</common_name> | |
| </ssl_certificates> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| </peerdb_tls> | |
| </users> | |
| <logger><level>none</level></logger> | |
| <path>var/lib/clickhouse</path> | |
| <tmp_path>var/lib/clickhouse/tmp</tmp_path> | |
| <user_files_path>var/lib/clickhouse/user_files</user_files_path> | |
| <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path> | |
| <tcp_port>9001</tcp_port> | |
| <tcp_port_secure>9441</tcp_port_secure> | |
| <http_port remove="1"/> | |
| <postgresql_port remove="1"/> | |
| <mysql_port remove="1"/> | |
| <openSSL> | |
| <server> | |
| <certificateFile>../ch-certs/server.crt</certificateFile> | |
| <privateKeyFile>../ch-certs/server.key</privateKeyFile> | |
| <caConfig>../ch-certs/ca.crt</caConfig> | |
| <verificationMode>relaxed</verificationMode> | |
| <cacheSessions>true</cacheSessions> | |
| <disableProtocols>sslv2,sslv3</disableProtocols> | |
| <preferServerCiphers>true</preferServerCiphers> | |
| </server> | |
| </openSSL> | |
| <macros> | |
| <shard>2</shard> | |
| <replica>1</replica> | |
| </macros> | |
| <zookeeper> | |
| <node> | |
| <host>localhost</host> | |
| <port>2181</port> | |
| </node> | |
| </zookeeper> | |
| <distributed_ddl> | |
| <path>/clickhouse/task_queue/ddl</path> | |
| </distributed_ddl> | |
| <remote_servers> | |
| <cicluster> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9000</port> | |
| </replica> | |
| </shard> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9001</port> | |
| </replica> | |
| </shard> | |
| </cicluster> | |
| </remote_servers> | |
| </clickhouse> | |
| EOF | |
| cat > config-keeper.xml <<EOF | |
| <clickhouse> | |
| <keeper_server> | |
| <tcp_port>2181</tcp_port> | |
| <server_id>1</server_id> | |
| <log_storage_path>var/lib/clickhouse/coordination/log</log_storage_path> | |
| <snapshot_storage_path>var/lib/clickhouse/coordination/snapshots</snapshot_storage_path> | |
| <raft_configuration> | |
| <server> | |
| <id>1</id> | |
| <hostname>localhost</hostname> | |
| <port>9234</port> | |
| </server> | |
| </raft_configuration> | |
| </keeper_server> | |
| </clickhouse> | |
| EOF | |
| mkdir chkeep ch1 ch2 | |
| (cd chkeep && ../clickhouse keeper -C ../config-keeper.xml) & | |
| while true; do | |
| if echo "ruok" | nc -w 3 127.0.0.1 2181 2>/dev/null | grep -q "imok"; then | |
| break | |
| fi | |
| echo "Waiting for keeper..." | |
| sleep 1 | |
| done | |
| sleep 5 | |
| (cd ch1 && ../clickhouse server -C ../config1.xml) & | |
| (cd ch2 && ../clickhouse server -C ../config2.xml) & | |
| - name: Install Temporal CLI | |
| uses: temporalio/setup-temporal@1059a504f87e7fa2f385e3fa40d1aa7e62f1c6ca # v0 | |
| - name: Setup AWS CA Certs | |
| env: | |
| URL: https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem | |
| run: | | |
| curl -fsSL -o aws-global-bundle.pem "$URL" | |
| sudo csplit -b '%02d.crt' -s -z -f /usr/local/share/ca-certificates/aws-global-split-- aws-global-bundle.pem '/-----BEGIN CERTIFICATE-----/' '{*}' | |
| sudo update-ca-certificates | |
| - name: Install gotestsum | |
| run: | | |
| go install gotest.tools/gotestsum@latest | |
| - name: run tests | |
| run: | | |
| mkdir coverage | |
| mkdir -p ../logs | |
| temporal server start-dev --namespace default --headless > ../logs/temporal.log 2>&1 & | |
| go build -cover -ldflags="-s -w" -o peer-flow | |
| temporal operator search-attribute create --name MirrorName --type Text --namespace default | |
| ./peer-flow worker > ../logs/peer-flow-worker.log 2>&1 & | |
| ./peer-flow snapshot-worker > ../logs/peer-flow-snapshot-worker.log 2>&1 & | |
| ./peer-flow api --port 8112 --gateway-port 8113 > ../logs/peer-flow-api.log 2>&1 & | |
| gotestsum --format standard-quiet --no-color --junitfile ../logs/test-results.xml -- -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 1200s -args -test.gocoverdir="$PWD/coverage" | |
| killall peer-flow | |
| sleep 1 | |
| go tool covdata textfmt -i=coverage -o ../coverage.out | |
| working-directory: ./flow | |
| env: | |
| GOCOVERDIR: coverage | |
| AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| AWS_ACCESS_KEY_ID: minio | |
| AWS_SECRET_ACCESS_KEY: miniosecret | |
| AWS_REGION: us-east-1 | |
| AWS_ENDPOINT_URL_S3_TLS: https://localhost:9998 | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1 | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1 | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb | |
| TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json | |
| TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json | |
| TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json | |
| TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json | |
| TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| PEERDB_CATALOG_HOST: localhost | |
| PEERDB_CATALOG_PORT: 5432 | |
| PEERDB_CATALOG_USER: postgres | |
| PEERDB_CATALOG_PASSWORD: postgres | |
| PEERDB_CATALOG_DATABASE: postgres | |
| PG_HOST: localhost | |
| PG_PORT: 5432 | |
| PG_USER: postgres | |
| PG_PASSWORD: postgres | |
| PG_DATABASE: postgres | |
| PG2_HOST: localhost | |
| PG2_PORT: 5437 | |
| PG2_USER: postgres | |
| PG2_PASSWORD: postgres | |
| PG2_DATABASE: postgres | |
| PEERDB_SWITCHBOARD_ENABLED: "true" | |
| PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true" | |
| ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200 | |
| CI_PG_VERSION: ${{ matrix.db-version.pg }} | |
| CI_MYSQL_VERSION: ${{ matrix.db-version.mysql }} | |
| CI_MONGO_ADMIN_URI: mongodb://localhost:27017 | |
| CI_MONGO_ADMIN_USERNAME: "admin" | |
| CI_MONGO_ADMIN_PASSWORD: "admin" | |
| CI_MONGO_URI: mongodb://localhost:27017 | |
| CI_MONGO_USERNAME: "csuser" | |
| CI_MONGO_PASSWORD: "cspass" | |
| SSH_POSTGRES_HOST: catalog | |
| ENABLE_OTEL_METRICS: ${{ (matrix.db-version.pg == '17' || matrix.db-version.mysql == 'mysql-pos') && 'true' || 'false' }} | |
| OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://localhost:4317 | |
| OTEL_EXPORTER_OTLP_METRICS_PROTOCOL: grpc | |
| PEERDB_OTEL_METRICS_NAMESPACE: 'peerdb_ci_tests.' | |
| PEERDB_OTEL_TEMPORAL_METRICS_EXPORT_LIST: '__ALL__' | |
| PEERDB_OTEL_METRICS_PANIC_ON_EXPORT_FAILURE: 'true' | |
| # Below are used to test RDS IAM Auth for Postgres and MySQL | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }} | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }} | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY }} | |
| FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES }} | |
| FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL }} | |
| FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE }} | |
| FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE }} | |
| # For ClickHouse S3 IAM Role based tests | |
| FLOW_TESTS_AWS_S3_BUCKET_NAME: ${{ secrets.FLOW_TESTS_AWS_S3_BUCKET_NAME }} | |
| FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }} | |
| FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }} | |
| FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }} | |
| # ClickHouse TLS/mTLS test certificates | |
| PEERDB_CLICKHOUSE_TLS_PORT: "9440" | |
| PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs | |
| - name: Upload peer-flow logs and test results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }} | |
| path: logs/ | |
| retention-days: 30 | |
| - name: Upload test results to Codecov | |
| if: success() || failure() | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 | |
| with: | |
| report_type: test_results | |
| files: logs/test-results.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Ingest tests results for analysis | |
| if: success() || failure() | |
| uses: ./.github/actions/ingest-test-results | |
| with: | |
| combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}" | |
| o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }} | |
| o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }} | |
| o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }} |