feat(tiered): Support Resumable Uploads #3075
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release/** | |
| - release-library/** | |
| pull_request: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| # librdkafka is built from source and links against libcurl, whose headers are not | |
| # on the runner by default. Needed by anything that builds the `kafka` feature. | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Install Rust Toolchain | |
| run: | | |
| rustup toolchain install stable --profile minimal --no-self-update | |
| rustup component add clippy rustfmt rust-docs --toolchain stable | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Format | |
| run: cargo fmt --all -- --check | |
| - name: Clippy | |
| run: cargo clippy --workspace --all-targets --all-features --no-deps -- -D warnings | |
| - name: Docs | |
| run: cargo doc --workspace --all-features --no-deps --document-private-items | |
| env: | |
| RUSTDOCFLAGS: -Dwarnings | |
| lint-python: | |
| name: Lint Python | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Install dependencies | |
| run: uv sync --all-packages --all-groups | |
| - name: Format | |
| run: uv run ruff format | |
| - name: Lint | |
| run: uv run ruff check | |
| - name: Run mypy | |
| run: uv run mypy . | |
| test-all: | |
| name: Test (all features) | |
| runs-on: ubuntu-latest | |
| steps: | |
| # librdkafka is built from source and links against libcurl, whose headers are not | |
| # on the runner by default. Needed by anything that builds the `kafka` feature. | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Start Bigtable Emulator | |
| run: | | |
| docker run -d \ | |
| --name bigtable-emulator \ | |
| -p 8086:8086 \ | |
| -e BIGTABLE_EMULATOR_HOST=localhost:8086 \ | |
| -v $PWD/devservices/run-bigtable.sh:/run-bigtable.sh \ | |
| google/cloud-sdk \ | |
| /run-bigtable.sh | |
| - name: Build GCS Emulator | |
| run: | | |
| docker build -t gcs-emulator-local https://github.com/getsentry/google-cloud-storage-testbench.git#main | |
| - name: Start GCS Emulator | |
| run: | | |
| docker run -d \ | |
| --name gcs-emulator \ | |
| -p 8087:9000 \ | |
| -e GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME=test-bucket \ | |
| gcs-emulator-local | |
| - name: Start SeaweedFS | |
| run: | | |
| docker run -d \ | |
| --name seaweedfs \ | |
| -p 8089:8333 \ | |
| -e S3_BUCKET=test-bucket \ | |
| chrislusf/seaweedfs:4.47 | |
| - name: Install Rust Toolchain | |
| run: | | |
| rustup toolchain install stable --profile minimal --no-self-update | |
| rustup component add llvm-tools-preview | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| with: | |
| key: ${{ github.job }} | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Run Rust doctests | |
| run: cargo test --workspace --all-features --doc --locked | |
| - name: Run tests with coverage | |
| run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info --locked | |
| - name: Upload Rust coverage | |
| uses: codecov/codecov-action@v5 | |
| continue-on-error: true | |
| with: | |
| files: lcov.info | |
| flags: rust | |
| use_oidc: true | |
| test-python: | |
| name: Test Python | |
| runs-on: ubuntu-latest | |
| steps: | |
| # librdkafka is built from source and links against libcurl, whose headers are not | |
| # on the runner by default. Needed by anything that builds the `kafka` feature. | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Install Rust Toolchain | |
| run: | | |
| rustup toolchain install stable --profile minimal --no-self-update | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| - name: Build server binary | |
| run: cargo build --locked | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Install dependencies | |
| run: uv sync --all-packages --all-groups | |
| - name: Run tests with coverage | |
| run: uv run pytest clients/python --cov=clients/python/src --cov-report=xml:python-coverage.xml | |
| - name: Upload Python coverage | |
| uses: codecov/codecov-action@v5 | |
| continue-on-error: true | |
| with: | |
| files: python-coverage.xml | |
| flags: python | |
| use_oidc: true | |
| docs: | |
| name: Build and Publish Documentation | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pages: write | |
| id-token: write | |
| steps: | |
| # librdkafka is built from source and links against libcurl, whose headers are not | |
| # on the runner by default. Needed by anything that builds the `kafka` feature. | |
| - name: Install libcurl-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Install Rust Toolchain | |
| run: rustup toolchain install stable --profile minimal --component rust-docs --no-self-update | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| with: | |
| key: ${{ github.job }} | |
| - name: Build Rust Docs | |
| run: | | |
| cargo doc --workspace --all-features --no-deps | |
| echo '<meta http-equiv="refresh" content="0; url=objectstore/" />Redirecting to <a href="objectstore/">objectstore</a>' > target/doc/index.html | |
| env: | |
| RUSTDOCFLAGS: -Dwarnings | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Install Python dependencies | |
| run: uv sync --all-packages --all-groups | |
| - name: Build Python Docs | |
| run: | | |
| mkdir -p docs/_build | |
| uv run sphinx-apidoc -o clients/python/docs/ clients/python/src/ --force --module-first --no-toc | |
| uv run sphinx-build -vv -W -b html clients/python/docs/ docs/_build | |
| - name: Setup combined docs directory | |
| run: | | |
| mkdir -p gh-pages/rust | |
| mkdir -p gh-pages/python | |
| cp -r target/doc/* gh-pages/rust/ | |
| cp -r docs/_build/* gh-pages/python/ | |
| cp docs/index.html gh-pages/index.html | |
| - name: Setup Pages | |
| uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 | |
| with: | |
| path: "gh-pages" | |
| - name: Deploy to GitHub Pages | |
| uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 | |
| if: github.ref == 'refs/heads/main' | |
| validate_gocd: | |
| if: ${{ !startsWith(github.ref, 'refs/heads/release/') && !startsWith(github.ref, 'refs/heads/release-library/') }} | |
| name: Validate GoCD Pipelines | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 | |
| id: changes | |
| with: | |
| filters: | | |
| gocd: | |
| - 'gocd/**' | |
| - id: "auth" | |
| if: steps.changes.outputs.gocd == 'true' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: "projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool" | |
| service_account: "gha-gocd-api@sac-prod-sa.iam.gserviceaccount.com" | |
| token_format: "id_token" | |
| id_token_audience: "610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com" | |
| id_token_include_email: true | |
| - uses: getsentry/action-gocd-jsonnet@3aec6fd54ac8d2fecfe700360f5d020e6902ba2d # v1.1.0 | |
| if: steps.changes.outputs.gocd == 'true' | |
| with: | |
| jb-install: true | |
| jsonnet-dir: gocd/templates | |
| generated-dir: gocd/generated-pipelines | |
| - uses: getsentry/action-validate-gocd-pipelines@5662a2b631d4e2aa1bfc21e878f9e131c31c40c1 # v1.0.0 | |
| if: steps.changes.outputs.gocd == 'true' | |
| with: | |
| configrepo: objectstore__main | |
| gocd_access_token: ${{ secrets.GOCD_ACCESS_TOKEN }} | |
| google_oidc_token: ${{ steps.auth.outputs.id_token }} |