feat(tiered): Support Resumable Uploads #2849
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release/** | |
| pull_request: | |
| permissions: | |
| contents: read | |
| jobs: | |
| build: | |
| name: Build ${{ matrix.platform }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-22.04 | |
| platform: amd64 | |
| target: x86_64-unknown-linux-gnu | |
| - os: ubuntu-22.04-arm | |
| platform: arm64 | |
| target: aarch64-unknown-linux-gnu | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| # libcurl4-openssl-dev: librdkafka is built from source and links against libcurl, | |
| # whose headers are not on the runner by default. | |
| sudo apt-get install -y llvm libcurl4-openssl-dev | |
| - name: Install sentry-cli | |
| uses: getsentry/action-setup-cli@70d7e587b84c2e78cf4d37cd33d7b74fb3729c1b # v1 | |
| - name: Install Rust Toolchain | |
| run: | | |
| rustup toolchain install stable --profile minimal --target ${{ matrix.target }} --no-self-update | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Ensure a clean build on release, use caches when building main. | |
| - uses: swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| if: ${{ !startsWith(github.ref, 'refs/heads/release/') }} | |
| with: | |
| key: ${{ github.job }} | |
| - name: Build Binary | |
| run: | | |
| cargo build --release --locked --target=${{ matrix.target }} --bin objectstore --features profiling | |
| cp target/${{ matrix.target }}/release/objectstore ./objectstore | |
| - name: Split Debug Info | |
| run: | | |
| llvm-objcopy --only-keep-debug objectstore objectstore.debug | |
| llvm-objcopy --strip-debug --strip-unneeded objectstore | |
| llvm-objcopy --add-gnu-debuglink objectstore.debug objectstore | |
| sentry-cli difutil bundle-sources objectstore.debug | |
| zip objectstore-debug.zip objectstore.debug | |
| mkdir -p /tmp/artifacts/${{ matrix.platform }} | |
| mv objectstore-debug.zip objectstore.src.zip /tmp/artifacts/${{ matrix.platform }}/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 | |
| - name: Build Image | |
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 | |
| with: | |
| context: . | |
| platforms: linux/${{ matrix.platform }} | |
| tags: ${{ matrix.platform }} | |
| outputs: type=docker,dest=/tmp/artifacts/objectstore-${{ matrix.platform }}.tar | |
| push: false | |
| - name: Test Image | |
| run: | | |
| docker load --input /tmp/artifacts/objectstore-${{ matrix.platform }}.tar | |
| docker run --rm ${{ matrix.platform }} version | grep -q '^objectstore@' | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 | |
| with: | |
| name: objectstore-${{ matrix.platform }} | |
| path: /tmp/artifacts/ | |
| assemble-ghcr: | |
| name: Publish to GHCR | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| # Intentionally never publish on pull requests | |
| if: ${{ github.event_name != 'pull_request' }} | |
| permissions: | |
| packages: write | |
| env: | |
| REGISTRY: ghcr.io/getsentry/objectstore | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< "$GHCR_TOKEN" | |
| env: | |
| GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - &download | |
| name: Download Images | |
| uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| pattern: objectstore-* | |
| path: /tmp | |
| merge-multiple: true | |
| - &assemble | |
| name: Push to GitHub Container Registry | |
| run: | | |
| for PLATFORM in amd64 arm64; do | |
| docker load --input /tmp/objectstore-$PLATFORM.tar | |
| docker tag $PLATFORM "$REGISTRY:$GITHUB_SHA-$PLATFORM" | |
| docker push "$REGISTRY:$GITHUB_SHA-$PLATFORM" | |
| done | |
| # Push nightly on main branch - releases are tagged by craft. | |
| TAGS="$GITHUB_SHA" | |
| if [[ "$GITHUB_REF" != refs/heads/release/* ]]; then | |
| TAGS="$TAGS nightly" | |
| fi | |
| for TAG in $TAGS; do | |
| docker manifest create "$REGISTRY:$TAG" \ | |
| --amend "$REGISTRY:$GITHUB_SHA-amd64" \ | |
| --amend "$REGISTRY:$GITHUB_SHA-arm64" | |
| docker manifest push "$REGISTRY:$TAG" | |
| done | |
| assemble-gcr: | |
| name: Publish to GCR | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| # Intentionally never publish on pull requests | |
| if: ${{ github.event_name != 'pull_request' }} | |
| permissions: | |
| id-token: write | |
| env: | |
| REGISTRY: us-docker.pkg.dev/sentryio/objectstore-mr/image | |
| steps: | |
| - name: Google Auth | |
| id: auth | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool | |
| service_account: gha-objectstore-push@sac-prod-sa.iam.gserviceaccount.com | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db" # v3.0.1 | |
| with: | |
| version: ">= 390.0.0" | |
| - name: Configure docker | |
| run: gcloud auth configure-docker us-docker.pkg.dev | |
| - *download | |
| - *assemble | |
| gocd-artifacts: | |
| name: Upload Build Artifacts for GoCD | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| # Intentionally never publish on pull requests | |
| if: ${{ github.event_name != 'pull_request' }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Google Auth | |
| id: auth | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool | |
| service_account: gha-objectstore-push@sac-prod-sa.iam.gserviceaccount.com | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db" # v3.0.1 | |
| with: | |
| version: ">= 390.0.0" | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| pattern: objectstore-* | |
| path: /tmp | |
| merge-multiple: true | |
| - name: Upload artifacts | |
| env: | |
| # the upload SA has no delete perms; disable parallel composite uploads | |
| # since their chunk cleanup would fail on delete | |
| CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_ENABLED: false | |
| run: | | |
| docker load --input /tmp/objectstore-amd64.tar | |
| docker run --rm amd64 version > /tmp/release-name | |
| gcloud storage cp --no-clobber /tmp/release-name \ | |
| "gs://dicd-team-devinfra-cd--objectstore/deployment-assets/$GITHUB_SHA/" | |
| for PLATFORM in amd64 arm64; do | |
| gcloud storage cp --no-clobber \ | |
| "/tmp/$PLATFORM/objectstore-debug.zip" \ | |
| "/tmp/$PLATFORM/objectstore.src.zip" \ | |
| "gs://dicd-team-devinfra-cd--objectstore/deployment-assets/$GITHUB_SHA/$PLATFORM/" | |
| done |