flow.yml

name: Flow build and test

on:
  pull_request:
    branches: [main]
    # Ignore changes to files that don't require builds or validations.
    # If more complex build avoidance is required, or for other events than pull_request, push, and pull_request_target,
    # replace this with a gate step in the job.
    # TODO: Re-enable build avoidance once we figure out how not to block the merge on the flow checks.
    #paths-ignore: &ignore_paths
    #    - '.claude/**'
    #    - 'README.md'
  push:
    branches: [main]
    #paths-ignore: *ignore_paths
  pull_request_target:
    branches: [main]
    #paths-ignore: *ignore_paths

permissions:
  id-token: write
  contents: read

jobs:
  flow_test:
    name: flow_test (${{ matrix.runner }}, ${{ matrix.db-version.pg }}, ${{ matrix.db-version.mysql }}, ${{ matrix.db-version.mongo }}, ${{ matrix.db-version.ch }})
    if: |
      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
      (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) ||
      github.event_name == 'push'
    environment: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && 'external-contributor' || null }}
    strategy:
      fail-fast: false
      matrix:
        runner: [ubuntu-latest-16-cores]
        db-version: [
          {pg: 16, mysql: 'mysql-gtid', mariadb: 'maria-11', mongo: '6.0', ch: 'lts'},
          {pg: 17, mysql: 'mysql-pos', mariadb: 'maria-12', mongo: '7.0', ch: 'stable'},
          {pg: 18, mysql: 'mysql-gtid', mariadb: 'maria-13', mongo: '8.0', ch: 'latest'},
        ]
        # Per-version container settings consumed by the MySQL and MariaDB steps.
        # Wrapped in a single-item list because matrix values must be arrays; it stays
        # a single shared value (no extra jobs).
        version-configs:
          - mysql:
              mysql-gtid:
                img: 'mysql:9.5'
                env: ['MYSQL_ROOT_PASSWORD=cipass']
                parameters: []
              mysql-pos:
                img: 'mysql:5.7'
                env: ['MYSQL_ROOT_PASSWORD=cipass']
                parameters: ['--log_bin=mysql-bin', '--server-id=1', '--bind-address=::']
            mariadb:
              maria-11:
                img: 'mariadb:lts-ubi9'
                env: ['MARIADB_ROOT_PASSWORD=cipass']
                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
              maria-12:
                img: 'mariadb:12-ubi'
                env: ['MARIADB_ROOT_PASSWORD=cipass']
                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
              maria-13:
                img: 'mariadb:13.0-ubi-rc'
                env: ['MARIADB_ROOT_PASSWORD=cipass']
                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    services:
      catalog:
        image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine
        ports:
          - 5432:5432
        env:
          PGUSER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
          POSTGRES_INITDB_ARGS: --locale=C.UTF-8
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      catalog2:
        image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine
        ports:
          - 5437:5432
        env:
          PGUSER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
          POSTGRES_INITDB_ARGS: --locale=C.UTF-8
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redpanda:
        image: redpandadata/redpanda@sha256:3e72ea35731a893bdd3b6d283eb554416377e23766434fc85dd7941540673639
        ports:
          - 9092:9092
          - 9644:9644
      elasticsearch:
        image: elasticsearch:9.4.2@sha256:e182d810a437e30ad7d0221fc153910a06d9be5b05e57e59e7983467b89e41e2
        ports:
          - 9200:9200
        env:
          discovery.type: single-node
          xpack.security.enabled: false
          xpack.security.enrollment.enabled: false
      otelcol:
        image: otel/opentelemetry-collector-contrib:0.154.0@sha256:b3079f45e19bdb7326bf49cdddce6cf60dfd865138db39f2733ea48ab17bc4cb
        ports:
          - 4317:4317
      toxiproxy:
        image: ghcr.io/shopify/toxiproxy:2.12.0@sha256:9378ed52a28bc50edc1350f936f518f31fa95f0d15917d6eb40b8e376d1a214e
        ports:
          - 18474:8474
          - 9902:9902
          - 9904:9904
          - 9903:9903
          - 10001:10001
          - 12001:12001
          - 12002:12002
          - 12003:12003
          - 12004:12004
          - 12005:12005
          - 14001:14001
          - 14002:14002
          - 14003:14003
      openssh:
        image: linuxserver/openssh-server:latest@sha256:67d4c3a1402179a6579aa217a38b52ced557eb8a0c17a8e32fe986a4549fdee4
        ports:
          - 2222:2222
        env:
          PUID: 1000
          PGID: 1000
          TZ: Etc/UTC
          USER_NAME: testuser
          USER_PASSWORD: testpass
          PASSWORD_ACCESS: true
          DOCKER_MODS: linuxserver/mods:openssh-server-ssh-tunnel

    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.ref }}
      - name: generate or hydrate protos
        uses: ./.github/actions/genprotos

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
        with:
          go-version: '1.26.4'
          cache-dependency-path: flow/go.sum

      - name: install lib-geos and pg_dump
        run: |
          # No need to update man pages on package install
          sudo apt-get remove --purge man-db
          # Add PGDG apt repo for latest PostgreSQL client packages
          sudo install -d /usr/share/postgresql-common/pgdg
          sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
            --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
          echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \
            https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
            | sudo tee /etc/apt/sources.list.d/pgdg.list
          sudo apt-get update
          sudo apt-get install -y libgeos-dev
          # pg_dump must be >= the server major version; install v18 so it
          # can dump PG 16, 17, and 18 (backward compatible).
          sudo apt-get install -y postgresql-client-18
          echo /usr/lib/postgresql/18/bin >> $GITHUB_PATH

      - name: install retry tool
        run: |
          sudo apt-get install -y retry

      - run: go mod download
        working-directory: ./flow

      - name: setup gcp service account
        id: gcp-service-account
        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
        with:
          name: "bq_service_account.json"
          json: ${{ secrets.GCP_GH_CI_PKEY }}

      - name: setup snowflake credentials
        id: sf-credentials
        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
        with:
          name: "snowflake_creds.json"
          json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}

      - name: setup GCS credentials
        id: gcs-credentials
        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
        with:
          name: "gcs_creds.json"
          json: ${{ secrets.GCS_CREDS }}

      - name: setup Eventhubs credentials
        id: eventhubs-credentials
        uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3
        with:
          name: "eh_creds.json"
          json: ${{ secrets.EH_CREDS }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6
        id: setup-aws
        with:
          audience: sts.amazonaws.com
          aws-region: us-west-2
          role-to-assume: ${{ secrets.FLOW_TESTS_AWS_ROLE_ARN }}
          mask-aws-account-id: true
          output-credentials: true

      - name: MySQL
        env:
          DB_IMG: ${{ matrix.version-configs.mysql[matrix.db-version.mysql].img }}
          DB_ENV: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].env, ' -e ') }}
          DB_PARAMS: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].parameters, ' ') }}
        run: |
          docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 \
            -e $DB_ENV $DB_IMG $DB_PARAMS

      - name: MariaDB
        env:
          DB_IMG: ${{ matrix.version-configs.mariadb[matrix.db-version.mariadb].img }}
          DB_ENV: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].env, ' -e ') }}
          DB_PARAMS: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].parameters, ' ') }}
        run: |
          docker run -d --rm --name mariadb --network ${{ job.container.network }} -p 3316:3306 \
            -e $DB_ENV $DB_IMG $DB_PARAMS

      - name: Mongo
        run: |
          echo "starting mongoDB..."
          docker run -d --rm --name mongo -p 27017:27017 mongo:${{ matrix.db-version.mongo }} \
            bash -c 'openssl rand -base64 756 > /data/mongo.key && chmod 400 /data/mongo.key && mongod --replSet rs0  --oplogMinRetentionHours 24 --bind_ip_all --keyFile /data/mongo.key'

          until docker exec mongo mongosh --eval 'db.runCommand({ ping: 1 })' &> /dev/null; do
            echo "waiting for MongoDB to be ready..."
            sleep 2
          done

          echo "initialize replica set"
          docker exec mongo mongosh --eval 'rs.initiate({
            _id: "rs0",
            members: [{ _id: 0, host: "localhost:27017" }]
          })'

          echo "create admin user for writing data to mongo"
          docker exec mongo mongosh --eval '
            db = db.getSiblingDB("admin");
            db.createUser({
              user: "admin",
              pwd: "admin",
              roles: ["root"]
            })'

          echo "create non-admin user for reading data from changestream"
          docker exec mongo mongosh -u admin -p admin --eval '
            db = db.getSiblingDB("admin");
            db.createUser({
              user: "csuser",
              pwd: "cspass",
              roles: ["readAnyDatabase", "clusterMonitor"]
            })'

      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
        id: cache-minio
        with:
          path: ./minio
          key: ${{ runner.os }}-minio

      - name: Install MinIO Server
        if: steps.cache-minio.outputs.cache-hit != 'true'
        run: |
          curl -O https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio

      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
        id: cache-minio-client
        with:
          path: ./mc
          key: ${{ runner.os }}-minio-client

      - name: Install MinIO Client
        if: steps.cache-minio-client.outputs.cache-hit != 'true'
        run: |
          curl -O https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc

      - name: MinIO
        run: >
          mkdir -p certs minio-data &&
          openssl genrsa -out certs/cert.key 2048 &&
          openssl req -new -key certs/cert.key -out certs/cert.csr -subj /CN=minio.local &&
          openssl x509 -req -days 3650 -in certs/cert.csr -signkey certs/cert.key -out certs/cert.crt &&
          chown -R 1001 certs &&
          ./minio server ./minio-data --certs-dir ./certs --address :9999 &
          sleep 2 &&
          ./mc alias set myminiopeerdb http://localhost:9999 minio miniosecret &&
          ./mc mb myminiopeerdb/peerdb
        env:
          MINIO_ROOT_USER: minio
          MINIO_ROOT_PASSWORD: miniosecret
          AWS_EC2_METADATA_DISABLED: true

      - name: Generate ClickHouse TLS certificates
        run: |
          mkdir -p ch-certs
          # CA
          openssl genrsa -out ch-certs/ca.key 2048
          openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA"
          # Server cert (CN=localhost, SAN for TLS 1.3)
          openssl genrsa -out ch-certs/server.key 2048
          openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
          openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall
          # Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention
          openssl genrsa -out ch-certs/tls.key 2048
          openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client"
          openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt

      - name: create postgres extensions, increase logical replication limits, and setup catalog database
        run: >
          docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git &&
          docker exec "${{ job.services.catalog.id }}" git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git /tmp/pgvector &&
          docker exec "${{ job.services.catalog.id }}" sh -c 'cd /tmp/pgvector && make with_llvm=no && make with_llvm=no install' &&
          docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;CREATE EXTENSION vector;"
          -c "ALTER SYSTEM SET wal_level=logical;"
          -c "ALTER SYSTEM SET max_replication_slots=192;"
          -c "ALTER SYSTEM SET max_wal_senders=256;"
          -c "ALTER SYSTEM SET max_connections=2048;" &&
          (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) &&
          docker restart "${{ job.services.catalog.id }}"
        env:
          PGPASSWORD: postgres

      - name: prepare secondary postgres for cross-cluster schema-dump tests
        run: >
          docker exec "${{ job.services.catalog2.id }}" psql -U postgres
          -c "ALTER SYSTEM SET wal_level=logical;"
          -c "ALTER SYSTEM SET max_replication_slots=192;"
          -c "ALTER SYSTEM SET max_wal_senders=256;"
          -c "ALTER SYSTEM SET max_connections=2048;" &&
          docker restart "${{ job.services.catalog2.id }}"
        env:
          PGPASSWORD: postgres

      - name: set ClickHouse version
        id: ch-version
        run: |
          if [ "${{ matrix.db-version.ch }}" = "lts" ]; then
            echo "ch_version=v25.8.11.66-lts" >> $GITHUB_OUTPUT
          elif [ "${{ matrix.db-version.ch }}" = "stable" ]; then
            echo "ch_version=v25.12.4.35-stable" >> $GITHUB_OUTPUT
          elif [ "${{ matrix.db-version.ch }}" = "latest" ]; then
            # note: latest tag does not always reflect the latest version (could be an update on an lts),
            # but that is okay as we are only using it to invalidate the cache.
            echo "ch_version=$(curl -s https://api.github.com/repos/ClickHouse/ClickHouse/releases/latest | jq -r .tag_name)" >> $GITHUB_OUTPUT
          fi

      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
        id: cache-clickhouse
        with:
          path: ./clickhouse
          key: ${{ runner.os }}-clickhouse-${{ steps.ch-version.outputs.ch_version }}

      - name: Install ClickHouse
        if: steps.cache-clickhouse.outputs.cache-hit != 'true'
        run: |
          if [[ "${{ matrix.db-version.ch }}" = 'latest' ]]; then
            curl https://clickhouse.com | sh
          else
            VERSION="${{ steps.ch-version.outputs.ch_version }}"
            VERSION_NUM=$(echo ${VERSION#v} | cut -d'-' -f1)
            curl -sL https://github.com/ClickHouse/ClickHouse/releases/download/$VERSION/clickhouse-common-static-$VERSION_NUM-amd64.tgz | tar -xzf -
            mv "clickhouse-common-static-$VERSION_NUM/usr/bin/clickhouse" ./clickhouse
            rm -rf "clickhouse-common-static-$VERSION_NUM"
            chmod +x clickhouse
          fi

      - name: Run ClickHouse
        run: |
          ./clickhouse --version
          cat > config1.xml <<EOF
          <clickhouse>
            <profiles><default></default></profiles>
            <users>
              <default>
                <password></password>
                <networks>
                  <ip>::/0</ip>
                </networks>
                <profile>default</profile>
                <quota>default</quota>
                <access_management>1</access_management>
                <named_collection_control>1</named_collection_control>
              </default>
              <peerdb_tls>
                <ssl_certificates>
                  <common_name>peerdb-client</common_name>
                </ssl_certificates>
                <networks>
                  <ip>::/0</ip>
                </networks>
                <profile>default</profile>
                <quota>default</quota>
                <access_management>1</access_management>
              </peerdb_tls>
            </users>
            <logger><level>none</level></logger>
            <path>var/lib/clickhouse</path>
            <tmp_path>var/lib/clickhouse/tmp</tmp_path>
            <user_files_path>var/lib/clickhouse/user_files</user_files_path>
            <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
            <tcp_port>9000</tcp_port>
            <tcp_port_secure>9440</tcp_port_secure>
            <http_port remove="1"/>
            <postgresql_port remove="1"/>
            <mysql_port remove="1"/>
            <openSSL>
              <server>
                <certificateFile>../ch-certs/server.crt</certificateFile>
                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
                <caConfig>../ch-certs/ca.crt</caConfig>
                <verificationMode>relaxed</verificationMode>
                <cacheSessions>true</cacheSessions>
                <disableProtocols>sslv2,sslv3</disableProtocols>
                <preferServerCiphers>true</preferServerCiphers>
              </server>
            </openSSL>
            <macros>
              <shard>1</shard>
              <replica>1</replica>
            </macros>
            <zookeeper>
              <node>
                <host>localhost</host>
                <port>2181</port>
              </node>
            </zookeeper>
            <distributed_ddl>
              <path>/clickhouse/task_queue/ddl</path>
            </distributed_ddl>
            <remote_servers>
              <cicluster>
                <shard>
                  <replica>
                    <host>localhost</host>
                    <port>9000</port>
                  </replica>
                </shard>
                <shard>
                  <replica>
                    <host>localhost</host>
                    <port>9001</port>
                  </replica>
                </shard>
              </cicluster>
            </remote_servers>
          </clickhouse>
          EOF
          cat > config2.xml <<EOF
          <clickhouse>
            <profiles><default></default></profiles>
            <users>
              <default>
                <password></password>
                <networks>
                  <ip>::/0</ip>
                </networks>
                <profile>default</profile>
                <quota>default</quota>
                <access_management>1</access_management>
                <named_collection_control>1</named_collection_control>
              </default>
              <peerdb_tls>
                <ssl_certificates>
                  <common_name>peerdb-client</common_name>
                </ssl_certificates>
                <networks>
                  <ip>::/0</ip>
                </networks>
                <profile>default</profile>
                <quota>default</quota>
                <access_management>1</access_management>
              </peerdb_tls>
            </users>
            <logger><level>none</level></logger>
            <path>var/lib/clickhouse</path>
            <tmp_path>var/lib/clickhouse/tmp</tmp_path>
            <user_files_path>var/lib/clickhouse/user_files</user_files_path>
            <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
            <tcp_port>9001</tcp_port>
            <tcp_port_secure>9441</tcp_port_secure>
            <http_port remove="1"/>
            <postgresql_port remove="1"/>
            <mysql_port remove="1"/>
            <openSSL>
              <server>
                <certificateFile>../ch-certs/server.crt</certificateFile>
                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
                <caConfig>../ch-certs/ca.crt</caConfig>
                <verificationMode>relaxed</verificationMode>
                <cacheSessions>true</cacheSessions>
                <disableProtocols>sslv2,sslv3</disableProtocols>
                <preferServerCiphers>true</preferServerCiphers>
              </server>
            </openSSL>
            <macros>
              <shard>2</shard>
              <replica>1</replica>
            </macros>
            <zookeeper>
              <node>
                <host>localhost</host>
                <port>2181</port>
              </node>
            </zookeeper>
            <distributed_ddl>
              <path>/clickhouse/task_queue/ddl</path>
            </distributed_ddl>
            <remote_servers>
              <cicluster>
                <shard>
                  <replica>
                    <host>localhost</host>
                    <port>9000</port>
                  </replica>
                </shard>
                <shard>
                  <replica>
                    <host>localhost</host>
                    <port>9001</port>
                  </replica>
                </shard>
              </cicluster>
            </remote_servers>
          </clickhouse>
          EOF
          cat > config-keeper.xml <<EOF
          <clickhouse>
            <keeper_server>
                <tcp_port>2181</tcp_port>
                <server_id>1</server_id>
                <log_storage_path>var/lib/clickhouse/coordination/log</log_storage_path>
                <snapshot_storage_path>var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
                <raft_configuration>
                  <server>
                    <id>1</id>
                    <hostname>localhost</hostname>
                    <port>9234</port>
                  </server>
                </raft_configuration>
            </keeper_server>
          </clickhouse>
          EOF
          mkdir chkeep ch1 ch2
          (cd chkeep && ../clickhouse keeper -C ../config-keeper.xml) &
          while true; do
            if echo "ruok" | nc -w 3 127.0.0.1 2181 2>/dev/null | grep -q "imok"; then
                break
            fi
            echo "Waiting for keeper..."
            sleep 1
          done
          sleep 5
          (cd ch1 && ../clickhouse server -C ../config1.xml) &
          (cd ch2 && ../clickhouse server -C ../config2.xml) &

      - name: Install Temporal CLI
        uses: temporalio/setup-temporal@1059a504f87e7fa2f385e3fa40d1aa7e62f1c6ca # v0

      - name: Setup AWS CA Certs
        env:
          URL: https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem
        run: |
          curl -fsSL -o aws-global-bundle.pem "$URL"
          sudo csplit -b '%02d.crt' -s -z -f /usr/local/share/ca-certificates/aws-global-split-- aws-global-bundle.pem '/-----BEGIN CERTIFICATE-----/' '{*}'
          sudo update-ca-certificates
      - name: Install gotestsum
        run: |
          go install gotest.tools/gotestsum@latest
      - name: run tests
        run: |
          mkdir coverage
          mkdir -p ../logs
          temporal server start-dev --namespace default --headless > ../logs/temporal.log 2>&1 &
          go build -cover -ldflags="-s -w" -o peer-flow
          temporal operator search-attribute create --name MirrorName --type Text --namespace default
          ./peer-flow worker > ../logs/peer-flow-worker.log 2>&1 &
          ./peer-flow snapshot-worker > ../logs/peer-flow-snapshot-worker.log 2>&1 &
          ./peer-flow api --port 8112 --gateway-port 8113 > ../logs/peer-flow-api.log 2>&1 &
          gotestsum --format standard-quiet --no-color --junitfile ../logs/test-results.xml -- -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 1200s -args -test.gocoverdir="$PWD/coverage"
          killall peer-flow
          sleep 1
          go tool covdata textfmt -i=coverage -o ../coverage.out
        working-directory: ./flow
        env:
          GOCOVERDIR: coverage
          AWS_ENDPOINT_URL_S3: http://localhost:9999
          AWS_ACCESS_KEY_ID: minio
          AWS_SECRET_ACCESS_KEY: miniosecret
          AWS_REGION: us-east-1
          AWS_ENDPOINT_URL_S3_TLS: https://localhost:9998
          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
          PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
          PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb
          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1
          PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
          PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb
          TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
          TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
          TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
          TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
          TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          PEERDB_CATALOG_HOST: localhost
          PEERDB_CATALOG_PORT: 5432
          PEERDB_CATALOG_USER: postgres
          PEERDB_CATALOG_PASSWORD: postgres
          PEERDB_CATALOG_DATABASE: postgres
          PG_HOST: localhost
          PG_PORT: 5432
          PG_USER: postgres
          PG_PASSWORD: postgres
          PG_DATABASE: postgres
          PG2_HOST: localhost
          PG2_PORT: 5437
          PG2_USER: postgres
          PG2_PASSWORD: postgres
          PG2_DATABASE: postgres
          PEERDB_SWITCHBOARD_ENABLED: "true"
          PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
          ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200
          CI_PG_VERSION: ${{ matrix.db-version.pg }}
          CI_MYSQL_HOST: localhost
          CI_MYSQL_PORT: 3306
          CI_MYSQL_ROOT_PASSWORD: cipass
          CI_MYSQL_VERSION: ${{ matrix.db-version.mysql }}
          CI_SSH_MYSQL_HOST: mysql
          CI_MARIADB_HOST: localhost
          CI_MARIADB_PORT: 3316
          CI_MARIADB_ROOT_PASSWORD: cipass
          CI_MARIADB_VERSION: ${{ matrix.db-version.mariadb }}
          CI_MONGO_ADMIN_URI: mongodb://localhost:27017
          CI_MONGO_ADMIN_USERNAME: "admin"
          CI_MONGO_ADMIN_PASSWORD: "admin"
          CI_MONGO_URI: mongodb://localhost:27017
          CI_MONGO_USERNAME: "csuser"
          CI_MONGO_PASSWORD: "cspass"
          SSH_POSTGRES_HOST: catalog
          ENABLE_OTEL_METRICS: ${{ (matrix.db-version.pg == '17' || matrix.db-version.mysql == 'mysql-pos') && 'true' || 'false' }}
          OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://localhost:4317
          OTEL_EXPORTER_OTLP_METRICS_PROTOCOL: grpc
          PEERDB_OTEL_METRICS_NAMESPACE: 'peerdb_ci_tests.'
          PEERDB_OTEL_TEMPORAL_METRICS_EXPORT_LIST: '__ALL__'
          PEERDB_OTEL_METRICS_PANIC_ON_EXPORT_FAILURE: 'true'
          # Below are used to test RDS IAM Auth for Postgres and MySQL
          FLOW_TESTS_RDS_IAM_AUTH_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }}
          FLOW_TESTS_RDS_IAM_AUTH_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }}
          FLOW_TESTS_RDS_IAM_AUTH_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }}
          FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES }}
          FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY }}
          FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL }}
          FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY }}
          FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES }}
          FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL }}
          FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE }}
          FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE }}
          # For ClickHouse S3 IAM Role based tests
          FLOW_TESTS_AWS_S3_BUCKET_NAME: ${{ secrets.FLOW_TESTS_AWS_S3_BUCKET_NAME }}
          FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }}
          FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }}
          FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }}
          # ClickHouse TLS/mTLS test certificates
          PEERDB_CLICKHOUSE_TLS_PORT: "9440"
          PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs

      - name: Upload peer-flow logs and test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}
          path: logs/
          retention-days: 30

      - name: Upload test results to Codecov
        if: success() || failure()
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
        with:
          report_type: test_results
          files: logs/test-results.xml
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Ingest tests results for analysis
        if: success() || failure()
        uses: ./.github/actions/ingest-test-results
        with:
          combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}"
          o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}
          o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}
          o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}