diff --git a/.claude/commands/release-notes b/.claude/commands/release-notes new file mode 100644 index 0000000000..686ca4213b --- /dev/null +++ b/.claude/commands/release-notes @@ -0,0 +1,638 @@ +--- +description: Generate release notes for Envoy AI Gateway following the established style and structure. +--- + +Generate release notes for Envoy AI Gateway following the established style and structure. + +## Instructions + +You are generating release notes for the Envoy AI Gateway project. Follow this workflow: + +### Step 1: Gather Context + +1. **Identify the previous release tag:** + + ```bash + git describe --tags --abbrev=0 + git tag --sort=-v:refname | head -10 + ``` + +2. **Review commits since the last release:** + + ```bash + git log ..HEAD --oneline --no-merges + git log ..HEAD --pretty=format:"%h %s" --no-merges + ``` + +3. **Get detailed PR information for significant changes:** + + ```bash + git log ..HEAD --grep="Merge pull request" --oneline + ``` + +4. **Review the existing release notes for style reference:** + - Read `site/src/data/releases/v0.4.json` and `site/src/data/releases/v0.3.json` + - Read `site/src/pages/release-notes/v0.4.mdx` and `site/src/pages/release-notes/v0.3.mdx` + +### Step 2: Verify Features Against Code + +For each potential feature identified from commits, verify it exists and understand its implementation by examining the actual code. Do not rely solely on commit messages. + +#### API and CRD Changes + +Review changes to the API definitions: + +```bash +git diff ..HEAD -- api/v1alpha1/ +``` + +Key files to examine: + +- `api/v1alpha1/*.go` - CRD type definitions (look for new fields, structs, or types) +- `api/v1alpha1/zz_generated.deepcopy.go` - Generated code indicates new types +- Look for `// +kubebuilder:` markers indicating validation rules or defaults + +#### Provider and Translator Changes + +Check for new or modified LLM provider support: + +```bash +git diff ..HEAD -- internal/translator/ +``` + +Key patterns to look for: + +- New files in `internal/translator/` indicate new provider support +- Changes to `internal/translator/translator.go` for routing logic +- New schema types or translation functions + +#### External Processor Changes + +Review the core request processing logic: + +```bash +git diff ..HEAD -- internal/extproc/ +``` + +Look for: + +- New endpoint support (e.g., `/v1/images/generations`, `/v1/embeddings`) +- Changes to streaming handling +- Token usage tracking modifications + +#### Controller Changes + +Check for new controller logic or resource handling: + +```bash +git diff ..HEAD -- internal/controller/ +``` + +Look for: + +- New reconcilers for CRDs +- Changes to resource watching or status updates +- Cross-namespace reference handling + +#### Observability Changes + +Review metrics and tracing implementations: + +```bash +git diff ..HEAD -- internal/metrics/ internal/tracing/ +``` + +Look for: + +- New metric definitions +- New span attributes or trace points +- Changes to OpenTelemetry semantic conventions + +#### CLI Changes + +Check for new CLI commands or options: + +```bash +git diff ..HEAD -- cmd/aigw/ +``` + +Look for: + +- New subcommands +- New flags or configuration options +- Auto-configuration changes + +#### Authentication Changes + +Review backend authentication implementations: + +```bash +git diff ..HEAD -- internal/backendauth/ +``` + +Look for: + +- New authentication providers (AWS, Azure, GCP, Anthropic) +- Credential chain support +- Token refresh logic + +#### Dependency Updates + +Verify actual dependency versions from source files: + +```bash +# Go version +grep -E "^go " go.mod + +# Key dependencies +grep -E "envoyproxy/gateway|sigs.k8s.io/gateway-api" go.mod + +# Envoy version +cat .envoy-version +``` + +#### Test Coverage + +Review test files to understand feature scope and edge cases: + +```bash +git diff ..HEAD -- '*_test.go' +git diff ..HEAD -- tests/ +``` + +Test files often reveal: + +- Supported configurations and options +- Edge cases and limitations +- Integration points + +### Step 3: Categorize Changes + +Categorize all changes into these sections: + +#### New Features + +Group features by theme (e.g., "Provider Support", "Observability", "Architecture"). Each feature group contains: + +- A descriptive `title` for the group +- An `items` array with individual features, each having a `title` and `description` + +#### API Changes (`apiChanges`) + +Document changes to CRDs and APIs: + +- New fields or resources +- Modified field types or behaviors +- New validation rules + +#### Deprecations (`deprecations`) + +List deprecated features with: + +- What is being deprecated +- What to use instead +- Timeline for removal (typically N+2 versions) + +#### Bug Fixes (`bugFixes`) + +Document fixes for user-facing bugs + +#### Breaking Changes (`breakingChanges`) + +List changes that require user action: + +- Configuration changes +- Removed features +- Behavioral changes + +#### Dependencies (`dependencies`) + +Update version information for: + +- Go version +- Envoy Gateway version +- Envoy Proxy version +- Gateway API version +- Gateway API Inference Extension version + +### Step 4: Generate the JSON Data File + +Create `site/src/data/releases/vX.Y.json` following this exact schema: + +```json +{ + "series": { + "version": "vX.Y", + "title": "Envoy AI Gateway vX.Y.x", + "subtitle": "Release introducing [key features summary].", + "badge": "Latest", + "badgeType": "milestone" + }, + "releases": [ + { + "version": "vX.Y.0", + "date": "Month DD, YYYY", + "type": "minor", + "tags": [{ "text": "Feature Name", "type": "feature" }], + "overview": "1-2 paragraph summary of the release highlights.", + "features": [ + { + "title": "Feature Group Name", + "items": [ + { + "title": "Feature title (can use tags)", + "description": "Detailed description of the feature and its benefits." + } + ] + } + ], + "apiChanges": [ + { + "title": "Change title", + "description": "Description of the API change." + } + ], + "deprecations": [ + { + "title": "DeprecatedField Pattern", + "description": "Description and migration guidance." + } + ], + "bugFixes": [ + { + "title": "Bug title", + "description": "Description of what was fixed." + } + ], + "breakingChanges": [ + { + "title": "Breaking change title", + "description": "Description and migration steps." + } + ], + "dependencies": [ + { + "title": "Go X.Y.Z", + "description": "Updated to Go X.Y.Z for improved performance and security." + }, + { + "title": "Envoy Gateway vX.Y", + "description": "Built on Envoy Gateway vX.Y for proven data plane capabilities." + }, + { + "title": "Envoy vX.Y", + "description": "Leveraging Envoy Proxy's battle-tested networking capabilities." + }, + { + "title": "Gateway API vX.Y.Z", + "description": "Support for Gateway API vX.Y.Z specifications." + } + ] + } + ], + "navigation": { + "previous": { "version": "vX.Y.x Series", "path": "/release-notes/vX.Y" }, + "next": { "version": "vX.Y.x Series", "path": "/release-notes/vX.Y" } + } +} +``` + +### Step 5: Generate the MDX Page + +Create `site/src/pages/release-notes/vX.Y.mdx` using this template: + +```mdx +--- +title: Envoy AI Gateway vX.Y.x Release Series +description: [Brief description of major features] +toc_min_heading_level: 2 +toc_max_heading_level: 4 +--- + +import Link from "@docusaurus/Link"; +import { + ReleaseSeriesLayout, + ReleaseHeader, + FeatureSectionCard, + PatchRelease, + ItemList, + Dependencies, +} from "../../components/ReleaseNotes"; +import releaseData from "../../data/releases/vX.Y.json"; +import React from "react"; + +export const { series, releases, navigation } = releaseData; +export const mainRelease = releases[0]; +export const patchReleases = releases.slice(1).reverse(); + + + + + +{mainRelease.overview} + + + +## ✨ New Features + +{mainRelease.features.map((featureSection, index) => ( + +{" "} + + +))} + +## 🔗 API Updates + +{mainRelease.apiChanges.length > 0 && ( + +{" "} + + +)} + +{mainRelease.deprecations?.length > 0 && ( + +{" "} + +<> + ### Deprecations + + +)} + +{mainRelease.breakingChanges?.length > 0 && ( + +{" "} + +<> + ## ⚠️ Breaking Changes + + +)} + +## 📖 Upgrade Guidance + +[Include migration steps for users upgrading from the previous version] + +## 📦 Dependencies Versions + + + +## ⏩ Patch Releases + +{patchReleases.map((release, index) => ( + + +))} + +## 🙏 Acknowledgements + +[Acknowledge contributors and organizations] + +## 🔮 What's Next + +[List upcoming features for future releases] + + +``` + +### Step 5b: Generate the plain-markdown copy + +Create `release-notes/vX.Y.Z.md` (at the repo root, sibling to `RELEASES.md`). This is a self-contained markdown rendering of the same content the site shows, intended for copy-paste into the GitHub release body. + +Rules: + +- **No MDX.** No imports, no React components, no `` HTML tags — use markdown backticks instead. +- **Same sections, same order** as the MDX page, so the GitHub release body and the site stay aligned: short overview paragraph → Breaking Changes → New Features (with the same feature-group subheadings) → API Updates → Bug Fixes → Upgrade Guidance (full migration steps and code blocks) → Dependency Versions → Acknowledgements → What's Next. +- **Self-contained.** A reader pasting this into GitHub should see the full release without needing to follow any link, but include a single link near the top to the rendered version on the site (`https://aigateway.envoyproxy.io/release-notes/vX.Y`). +- **Filename includes the patch version** (`v0.6.0.md`, not `v0.6.md`) so future patch releases get their own files. + +### Step 6: Update Navigation + +Update the previous release's JSON to add the `next` navigation: + +```json +"navigation": { + "previous": { ... }, + "next": { "version": "vX.Y.x Series", "path": "/release-notes/vX.Y" } +} +``` + +Update `site/src/data/releases/index.json` with the new `lastUpdated` date. + +### Step 6b: Update the Release Notes Index Page + +The release notes index page (`site/src/pages/release-notes/index.mdx`) has hardcoded imports and release cards that must be updated manually. + +#### 1. Add the import for the new release data + +Add after the existing imports: + +```tsx +import vXYData from "../../data/releases/vX.Y.json"; +``` + +#### 2. Add the new release to the `allReleases` array + +Update the array to include the new release data first: + +```tsx +export const allReleases = [ + ...vXYData.releases, // Add new release first + ...vPreviousData.releases, + // ... other releases +].sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); +``` + +#### 3. Add a `ReleaseCard` for the new release + +Add as the first card in the `releaseNotesGrid` div, with `featured={true}`: + +```tsx + release.version).join(", ")} +/> +``` + +#### 4. Update the previous release card + +Change the previous "Latest" release card: + +- Set `featured={false}` +- Update the `date` range to end at the new release date (e.g., "November 7, 2025 - January 16, 2026") + +### Step 7: Verify Each Item Against Code + +After generating the release notes, go through **each individual item** and verify its accuracy against the actual code: + +#### For Each Feature Item + +1. **Locate the implementation**: Find the specific file(s) that implement the feature +2. **Verify the description is accurate**: Read the code to confirm the feature works as described +3. **Check for limitations not mentioned**: Look for edge cases, unsupported scenarios, or constraints +4. **Verify field names and types**: Ensure any API fields, config options, or CLI flags match exactly + +#### For Each API Change + +1. **Find the struct definition**: Locate the Go struct in `api/v1alpha1/` +2. **Verify field names**: Ensure the documented field name matches the JSON tag +3. **Check field types and validation**: Confirm types and any kubebuilder validation markers +4. **Look for related changes**: Check if the field affects other parts of the API + +#### For Each Breaking Change + +1. **Identify what breaks**: Find the code that changed behavior +2. **Verify migration path**: Ensure the suggested migration actually works +3. **Check for undocumented impacts**: Look for side effects the user needs to know about + +#### Example Verification Process + +``` +Feature: "AWS SDK default credential chain support" + +1. Search: grep -r "credential" internal/backendauth/ +2. Read: internal/backendauth/aws.go - find the credential chain implementation +3. Verify: Does it actually support IRSA, Pod Identity, Instance Profiles as claimed? +4. Check tests: What scenarios are tested? Any limitations? +5. Update description if needed to match actual behavior +``` + +### Step 8: Review for Quality and Clarity + +#### Check for Repetition and Redundancy + +1. **Scan for duplicate concepts**: Look for features described multiple times in different sections +2. **Consolidate related items**: Combine items that describe the same feature from different angles +3. **Remove redundant phrases**: Eliminate repeated words like "support for", "enables", "provides" +4. **Check tags vs features**: Ensure release tags don't just repeat what's in the features list + +#### Common Redundancy Patterns to Fix + +- Same feature in both "features" and "apiChanges" - keep API details in apiChanges only +- Multiple items describing sub-features that should be one item with sub-bullets +- Overlapping descriptions between feature groups + +#### Apply "So What?" Framing + +For each feature, ensure the description answers: **"Why should a user care about this?"** + +**Bad (implementation-focused):** + +> Added `headerMutation` field to `AIServiceBackend` spec + +**Good (user benefit-focused):** + +> Header mutations at backend level: New `headerMutation` field in `AIServiceBackend` enables adding, removing, or modifying HTTP headers for all requests to a backend, useful for adding API keys, tracing headers, or custom metadata without modifying application code. + +#### "So What?" Checklist for Each Item + +Ask these questions: + +1. **What can the user do now that they couldn't before?** +2. **What problem does this solve?** +3. **What's the practical use case?** +4. **How does this make their life easier?** + +#### Reframing Examples + +| Before (Technical) | After (User Benefit) | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| "Support for AWS credential chain" | "Eliminates need for static AWS credentials - automatically uses IRSA, Pod Identity, or Instance Profiles" | +| "Added cross-namespace references" | "Reference backends in different namespaces, enabling multi-tenant setups and organizational separation" | +| "Implemented OpenInference tracing" | "Full request lifecycle visibility with Arize Phoenix compatibility for debugging and evaluation" | +| "New MCPRoute CRD" | "Route MCP requests to backend servers with a single custom resource, enabling unified AI API management" | + +#### Final Quality Pass + +1. **Read aloud**: Does each description flow naturally? +2. **Jargon check**: Would a new user understand the terminology? +3. **Completeness**: Does each item have enough context to be useful? +4. **Consistency**: Is the tone and style consistent throughout? + +### Style Guidelines + +1. **Feature descriptions**: Focus on user benefit, not implementation details +2. **Use `` tags**: For API fields, CRD names, CLI commands, and config values +3. **Use `Link text →`**: For links to documentation +4. **Be specific**: Include version numbers, field names, and concrete examples +5. **Group logically**: Combine related features under themed sections +6. **Migration guidance**: Always provide clear upgrade steps for breaking changes +7. **Dependencies**: Check `go.mod` and `.envoy-version` for current versions +8. **"So what?" test**: Every feature should explain why users should care + +### Checklist + +#### Gathering and Verification + +- [ ] Identified all commits since last release +- [ ] Verified API/CRD changes against `api/v1alpha1/` source files +- [ ] Verified provider changes against `internal/translator/` implementations +- [ ] Verified observability changes against `internal/metrics/` and `internal/tracing/` +- [ ] Verified CLI changes against `cmd/aigw/` source files +- [ ] Verified authentication changes against `internal/backendauth/` +- [ ] Cross-referenced features with test files for accuracy +- [ ] Verified dependency versions from `go.mod` and `.envoy-version` + +#### Content Generation + +- [ ] Categorized changes into appropriate sections +- [ ] Created JSON data file with correct schema +- [ ] Created MDX page file +- [ ] Updated previous release navigation +- [ ] Updated `site/src/data/releases/index.json` lastUpdated date +- [ ] Updated `site/src/pages/release-notes/index.mdx` with new import, allReleases array, and ReleaseCard + +#### Item-by-Item Verification (Step 7) + +- [ ] Each feature item verified against its implementation code +- [ ] Each API change verified against struct definitions in `api/v1alpha1/` +- [ ] Field names match JSON tags exactly +- [ ] Breaking changes have verified migration paths +- [ ] No claims made that aren't supported by the code + +#### Quality and Clarity Review (Step 8) + +- [ ] No duplicate features across sections +- [ ] No redundant items that should be consolidated +- [ ] Each feature passes the "so what?" test (explains user benefit) +- [ ] Descriptions focus on what users can do, not implementation details +- [ ] Consistent tone and style throughout +- [ ] All code references use `` tags +- [ ] Links to documentation are valid + +#### Final Checks + +- [ ] Read through entire release notes for flow and readability +- [ ] Breaking changes include clear migration guidance +- [ ] Deprecations specify removal timeline and alternatives +- [ ] No jargon without explanation diff --git a/.envoy-version b/.envoy-version index bf50e910e6..11a36de972 100644 --- a/.envoy-version +++ b/.envoy-version @@ -1 +1 @@ -1.37.0 +1.38.1 diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4e2172c5a8..2b747c7b56 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -212,8 +212,8 @@ jobs: include: - name: latest envoy_gateway_version: v0.0.0-latest - - name: v1.6.2 - envoy_gateway_version: v1.6.2 + - name: v1.8.1 + envoy_gateway_version: v1.8.1 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v5 @@ -270,7 +270,7 @@ jobs: - run: make test-e2e-upgrade env: # We only need to test the upgrade from the latest stable version of EG. - EG_VERSION: v1.6.0 + EG_VERSION: v1.8.1 K8S_VERSION: ${{ matrix.k8s-version }} test_e2e_inference_extension: @@ -299,7 +299,7 @@ jobs: - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - run: make test-e2e-inference-extension env: - EG_VERSION: v1.6.0 + EG_VERSION: v1.8.1 test_e2e_namespaced: needs: changes @@ -329,7 +329,7 @@ jobs: env: # We only need to test with the latest stable version of EG, since these e2e tests # do not depend on the EG version. - EG_VERSION: v1.6.0 + EG_VERSION: v1.8.1 test_e2e_aigw: needs: changes diff --git a/RELEASES.md b/RELEASES.md index 3df6c3b34b..537fc3258a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -84,7 +84,12 @@ Each non-patch release should start with Release Candidate (RC) phase as follows Pushing a tag will trigger the pipeline to build the release candidate image and the helm chart tagged with the release candidate tag. The release candidate image will be available in the Docker Hub. -4. The release candidate should be tested by the maintainers and the community. If there is any issue, the issue should be fixed in the main branch +4. Generate the release notes using the Claude Code `/release-notes` command (defined in `.claude/commands/release-notes`). + This command analyzes commits since the last release, verifies features against the actual code, and generates + the release notes JSON data file, MDX page, plain-markdown copy, and navigation updates for the docs site. + Review the generated output and make any necessary adjustments. + +5. The release candidate should be tested by the maintainers and the community. If there is any issue, the issue should be fixed in the main branch and the new rc tag should be created. For example, if there is an issue in the release candidate v0.50.0-rc1, replace `v0.50.0-rc1` with `v0.50.0-rc2` in the above command and repeat the process. diff --git a/cmd/aigw/download_envoy.go b/cmd/aigw/download_envoy.go index 40933b72e6..3cc2fe4cbc 100644 --- a/cmd/aigw/download_envoy.go +++ b/cmd/aigw/download_envoy.go @@ -17,7 +17,7 @@ import ( // This matches the version in the .envoy-version file at the repo root. That is ensured in tests. // The reason why we don't use the constant defined in the EG as a library is that when we depend // on the main branch, it will be a non-released development version that func-e may not have. -const envoyVersion = "1.37.0" +const envoyVersion = "1.38.1" // downloadEnvoy downloads the Envoy binary used by Envoy Gateway. func downloadEnvoy(ctx context.Context, funcERun func_e_api.RunFunc, tmpDir, dataHome string, stdout, stderr io.Writer) error { diff --git a/go.mod b/go.mod index dae73f5d78..67bce5632b 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/envoyproxy/ai-gateway // Explicitly specify the Go patch version to be able to purge the CI cache correctly. -go 1.26.2 +go 1.26.4 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 @@ -18,10 +18,10 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/cohere-ai/cohere-go/v2 v2.18.0 github.com/coreos/go-oidc/v3 v3.18.0 - github.com/envoyproxy/gateway v1.7.0 - github.com/envoyproxy/go-control-plane v0.14.0 - github.com/envoyproxy/go-control-plane/envoy v1.37.0 - github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e + github.com/envoyproxy/gateway v1.8.1 + github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14 + github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409050421-3f47accd6e14 + github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260409050421-3f47accd6e14 github.com/go-logr/logr v1.4.3 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/cel-go v0.28.1 @@ -53,7 +53,7 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.28.0 - golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.20.0 golang.org/x/tools v0.45.0 @@ -70,8 +70,9 @@ require ( k8s.io/client-go v0.36.1 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/controller-runtime v0.24.1 - sigs.k8s.io/gateway-api v1.4.1 + sigs.k8s.io/gateway-api v1.5.1 sigs.k8s.io/gateway-api-inference-extension v1.0.2 + sigs.k8s.io/gateway-api/conformance v1.5.1 sigs.k8s.io/yaml v1.6.0 ) @@ -114,7 +115,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect github.com/containers/image/v5 v5.36.2 // indirect github.com/containers/storage v1.59.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect @@ -122,18 +123,17 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/cli v29.2.0+incompatible // indirect + github.com/docker/cli v29.4.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v28.5.2+incompatible // indirect - github.com/docker/docker-credential-helpers v0.9.4 // indirect + github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dominikbraun/graph v0.23.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260115164926-066cbd5b3989 // indirect + github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260409050421-3f47accd6e14 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect - github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7 // indirect + github.com/envoyproxy/ratelimit v1.4.1-0.20260122083618-3fb702589d36 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -143,25 +143,25 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-openapi/analysis v0.24.1 // indirect - github.com/go-openapi/errors v0.22.4 // indirect - github.com/go-openapi/jsonpointer v0.22.4 // indirect - github.com/go-openapi/jsonreference v0.21.4 // indirect - github.com/go-openapi/loads v0.23.2 // indirect - github.com/go-openapi/spec v0.22.3 // indirect - github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/analysis v0.24.3 // indirect + github.com/go-openapi/errors v0.22.7 // indirect + github.com/go-openapi/jsonpointer v0.22.5 // indirect + github.com/go-openapi/jsonreference v0.21.5 // indirect + github.com/go-openapi/loads v0.23.3 // indirect + github.com/go-openapi/spec v0.22.4 // indirect + github.com/go-openapi/strfmt v0.26.1 // indirect github.com/go-openapi/swag v0.23.1 // indirect - github.com/go-openapi/swag/conv v0.25.4 // indirect - github.com/go-openapi/swag/fileutils v0.25.1 // indirect - github.com/go-openapi/swag/jsonname v0.25.4 // indirect - github.com/go-openapi/swag/jsonutils v0.25.4 // indirect - github.com/go-openapi/swag/loading v0.25.4 // indirect - github.com/go-openapi/swag/mangling v0.25.1 // indirect - github.com/go-openapi/swag/stringutils v0.25.4 // indirect - github.com/go-openapi/swag/typeutils v0.25.4 // indirect - github.com/go-openapi/swag/yamlutils v0.25.4 // indirect - github.com/go-openapi/validate v0.25.1 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-openapi/swag/conv v0.25.5 // indirect + github.com/go-openapi/swag/fileutils v0.25.5 // indirect + github.com/go-openapi/swag/jsonname v0.25.5 // indirect + github.com/go-openapi/swag/jsonutils v0.25.5 // indirect + github.com/go-openapi/swag/loading v0.25.5 // indirect + github.com/go-openapi/swag/mangling v0.25.5 // indirect + github.com/go-openapi/swag/stringutils v0.25.5 // indirect + github.com/go-openapi/swag/typeutils v0.25.5 // indirect + github.com/go-openapi/swag/yamlutils v0.25.5 // indirect + github.com/go-openapi/validate v0.25.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.7.0 // indirect @@ -182,10 +182,9 @@ require ( github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect - github.com/lyft/gostats v0.4.1 // indirect + github.com/lyft/gostats v0.4.14 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/miekg/dns v1.1.72 // indirect + github.com/mailru/easyjson v0.9.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect @@ -199,8 +198,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/ohler55/ojg v1.28.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect + github.com/ohler55/ojg v1.28.1 // indirect + github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -231,12 +230,11 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/yuin/gopher-lua v1.1.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect - go.mongodb.org/mongo-driver v1.17.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect @@ -281,7 +279,7 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09 // indirect - sigs.k8s.io/mcs-api v0.3.0 // indirect + sigs.k8s.io/mcs-api v0.4.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) diff --git a/go.sum b/go.sum index c7ac8b62cf..2524e427f7 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= -github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= +github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= +github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= github.com/containers/image/v5 v5.36.2 h1:GcxYQyAHRF/pLqR4p4RpvKllnNL8mOBn0eZnqJbfTwk= github.com/containers/image/v5 v5.36.2/go.mod h1:b4GMKH2z/5t6/09utbse2ZiLK/c72GuGLFdp7K69eA4= github.com/containers/storage v1.59.1 h1:11Zu68MXsEQGBBd+GadPrHPpWeqjKS8hJDGiAHgIqDs= @@ -142,14 +142,12 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= -github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.4.1+incompatible h1:02RT8QqqwtGRn+6SYypv8IUEbD/ltY6sfKCJIoUcGzk= +github.com/docker/cli v29.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= -github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.9.4 h1:76ItO69/AP/V4yT9V4uuuItG0B1N8hvt0T0c0NN/DzI= -github.com/docker/docker-credential-helpers v0.9.4/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= +github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= +github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -162,20 +160,20 @@ github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/gateway v1.7.0 h1:noVz4fADhljSKpQFaspb25+C3kC8bBBisQZle9W0ei4= -github.com/envoyproxy/gateway v1.7.0/go.mod h1:U/UXS8G6Q5SldcKubP8PZazD9OsNT0Mf1XzCVig0KkQ= -github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260115164926-066cbd5b3989 h1:KTd1TJym7dgV1L1XlxXeJNct7rJI3xTV+iuArq40wm0= -github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260115164926-066cbd5b3989/go.mod h1:+fG/snSdlOxU+5RWuuKSYxF9zusT3Duy1MDbETA44Bo= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e h1:EHL6eLDhQduyYGEKh+QSXE7s7Yhg/hpeeHFT0ET0gBw= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e/go.mod h1:buWyXJdrI6ayYbeGm3upu3Qf/qHHrdWfUHKnVrTD+vM= +github.com/envoyproxy/gateway v1.8.1 h1:i8POtrPtel3n0iGP55MNy6U1rTW4msyhkEWjXAF09ts= +github.com/envoyproxy/gateway v1.8.1/go.mod h1:l8bHAnc8sMfNNIcBm5p7CojxkF2AfbxKCjbPkaXIqEE= +github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14 h1:7g8SJv4OrVcLT4yfkzIbsTcwLBwyLu8gKb/yCf3Loxk= +github.com/envoyproxy/go-control-plane v0.14.1-0.20260409050421-3f47accd6e14/go.mod h1:18SVzvkoF8AL2O7baVikhojMZ+7rFPh3o8tOOsBVyok= +github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260409050421-3f47accd6e14 h1:VszH+75Lfplgo/ZDOe79HOGnLHAgPHWqFjMl7AdQEWw= +github.com/envoyproxy/go-control-plane/contrib v1.36.1-0.20260409050421-3f47accd6e14/go.mod h1:29VWPXU81Y5hg3S89D3zXhbOgqgh93Os+W911d6SxP8= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409050421-3f47accd6e14 h1:zEzMNlk4Kb4GpwKt2pmEc2B5+iM9rcmUYoB0mGHhXyU= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409050421-3f47accd6e14/go.mod h1:5yRfenlmRH8sxKrhXyiFtK8BDz3syDWcFm81rkCcATM= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260409050421-3f47accd6e14 h1:128xSbKG9xp2W6JAyfb2Q2pDrEC5bhtUcfYpJZf6OdA= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260409050421-3f47accd6e14/go.mod h1://utHaGoDyMdS6rB87A76UIaRn+Ss9dS2ZJ5rM2psGU= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7 h1:yz9/p/8QVPuEjPqRfZDXJmRaURKpKkxCZXUhl22i+cU= -github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7/go.mod h1:NmJBO+gDMvSQWvcSWq8wmlgkDmHHAkx1SCxEGva5hKU= +github.com/envoyproxy/ratelimit v1.4.1-0.20260122083618-3fb702589d36 h1:nEi1OH2qhE8NtcuBgO/uKpTw/P0nVu4i8mZvL6oD9CQ= +github.com/envoyproxy/ratelimit v1.4.1-0.20260122083618-3fb702589d36/go.mod h1:n61Cm3Lex7sdo0PcQ4UMO12rYb4VT9XZaYC1bA+BUFc= github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= @@ -186,8 +184,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= -github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk= +github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -200,52 +198,52 @@ github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/analysis v0.24.1 h1:Xp+7Yn/KOnVWYG8d+hPksOYnCYImE3TieBa7rBOesYM= -github.com/go-openapi/analysis v0.24.1/go.mod h1:dU+qxX7QGU1rl7IYhBC8bIfmWQdX4Buoea4TGtxXY84= -github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM= -github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= -github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= -github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= -github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= -github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/loads v0.23.2 h1:rJXAcP7g1+lWyBHC7iTY+WAF0rprtM+pm8Jxv1uQJp4= -github.com/go-openapi/loads v0.23.2/go.mod h1:IEVw1GfRt/P2Pplkelxzj9BYFajiWOtY2nHZNj4UnWY= -github.com/go-openapi/spec v0.22.3 h1:qRSmj6Smz2rEBxMnLRBMeBWxbbOvuOoElvSvObIgwQc= -github.com/go-openapi/spec v0.22.3/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs= -github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= -github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= +github.com/go-openapi/analysis v0.24.3 h1:a1hrvMr8X0Xt69KP5uVTu5jH62DscmDifrLzNglAayk= +github.com/go-openapi/analysis v0.24.3/go.mod h1:Nc+dWJ/FxZbhSow5Yh3ozg5CLJioB+XXT6MdLvJUsUw= +github.com/go-openapi/errors v0.22.7 h1:JLFBGC0Apwdzw3484MmBqspjPbwa2SHvpDm0u5aGhUA= +github.com/go-openapi/errors v0.22.7/go.mod h1://QW6SD9OsWtH6gHllUCddOXDL0tk0ZGNYHwsw4sW3w= +github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= +github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= +github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= +github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= +github.com/go-openapi/loads v0.23.3 h1:g5Xap1JfwKkUnZdn+S0L3SzBDpcTIYzZ5Qaag0YDkKQ= +github.com/go-openapi/loads v0.23.3/go.mod h1:NOH07zLajXo8y55hom0omlHWDVVvCwBM/S+csCK8LqA= +github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ= +github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ= +github.com/go-openapi/strfmt v0.26.1 h1:7zGCHji7zSYDC2tCXIusoxYQz/48jAf2q+sF6wXTG+c= +github.com/go-openapi/strfmt v0.26.1/go.mod h1:Zslk5VZPOISLwmWTMBIS7oiVFem1o1EI6zULY8Uer7Y= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= -github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= -github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/fileutils v0.25.1 h1:rSRXapjQequt7kqalKXdcpIegIShhTPXx7yw0kek2uU= -github.com/go-openapi/swag/fileutils v0.25.1/go.mod h1:+NXtt5xNZZqmpIpjqcujqojGFek9/w55b3ecmOdtg8M= -github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= -github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= -github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= -github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= -github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/mangling v0.25.1 h1:XzILnLzhZPZNtmxKaz/2xIGPQsBsvmCjrJOWGNz/ync= -github.com/go-openapi/swag/mangling v0.25.1/go.mod h1:CdiMQ6pnfAgyQGSOIYnZkXvqhnnwOn997uXZMAd/7mQ= -github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= -github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= -github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= -github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= -github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= -github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-openapi/validate v0.25.1 h1:sSACUI6Jcnbo5IWqbYHgjibrhhmt3vR6lCzKZnmAgBw= -github.com/go-openapi/validate v0.25.1/go.mod h1:RMVyVFYte0gbSTaZ0N4KmTn6u/kClvAFp+mAVfS/DQc= +github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g= +github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k= +github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk= +github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc= +github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= +github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo= +github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo= +github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU= +github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g= +github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw= +github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY= +github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M= +github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII= +github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E= +github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc= +github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ= +github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.1 h1:NZOrZmIb6PTv5LTFxr5/mKV/FjbUzGE7E6gLz7vFoOQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.1/go.mod h1:r7dwsujEHawapMsxA69i+XMGZrQ5tRauhLAjV/sxg3Q= +github.com/go-openapi/testify/v2 v2.4.1 h1:zB34HDKj4tHwyUQHrUkpV0Q0iXQ6dUCOQtIqn8hE6Iw= +github.com/go-openapi/testify/v2 v2.4.1/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/validate v0.25.2 h1:12NsfLAwGegqbGWr2CnvT65X/Q2USJipmJ9b7xDJZz0= +github.com/go-openapi/validate v0.25.2/go.mod h1:Pgl1LpPPGFnZ+ys4/hTlDiRYQdI1ocKypgE+8Q8BLfY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -267,8 +265,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= -github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= -github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -315,14 +313,12 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/lyft/gostats v0.4.1 h1:oR6p4HRCGxt0nUntmZIWmYMgyothBi3eZH2A71vRjsc= -github.com/lyft/gostats v0.4.1/go.mod h1:Tpx2xRzz4t+T2Tx0xdVgIoBdR2UMVz+dKnE3X01XSd8= +github.com/lyft/gostats v0.4.14 h1:xmP4yMfDvEKtlNZEcS2sYz0cvnps1ri337ZEEbw3ab8= +github.com/lyft/gostats v0.4.14/go.mod h1:cJWqEVL8JIewIJz/olUIios2F1q06Nc51hXejPQmBH0= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= -github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -355,15 +351,15 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ohler55/ojg v1.28.0 h1:8xClBgMIRRJGDUC9xNe7NprP4kD2C3mQMeon3wY4KXA= -github.com/ohler55/ojg v1.28.0/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/ohler55/ojg v1.28.1 h1:Xy93DelhLSZNeWv8GPKtP6qMqkUlZlAxBP/AQcC5RfY= +github.com/ohler55/ojg v1.28.1/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= +github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= +github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y= -github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= -github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= -github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/onsi/ginkgo/v2 v2.28.0 h1:Rrf+lVLmtlBIKv6KrIGJCjyY8N36vDVcutbGJkyqjJc= +github.com/onsi/ginkgo/v2 v2.28.0/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= github.com/openai/openai-go/v3 v3.35.0 h1:109x3epXMSE423KW2euR506GGFezcEt0s87MoWejpH0= @@ -372,6 +368,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -454,10 +451,10 @@ github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9R github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/tsaarni/certyaml v0.10.0 h1:8ZWHO4Zg4VHUf7YblZNju44PcG5M+YtlJawiArYUHRs= -github.com/tsaarni/certyaml v0.10.0/go.mod h1:rI1wDTE/VQIglHOyGbjfvqb+5mWTVT5uLFVDDcT1sq8= -github.com/tsaarni/x500dn v1.0.0 h1:LvaWTkqRpse4VHBhB5uwf3wytokK4vF9IOyNAEyiA+U= -github.com/tsaarni/x500dn v1.0.0/go.mod h1:QaHa3EcUKC4dfCAZmj8+ZRGLKukWgpGv9H3oOCsAbcE= +github.com/tsaarni/certyaml v0.11.0 h1:qpiXKPCGZvQaYf3ParnvLcxMZmWqBIeU4XDAywfQUHw= +github.com/tsaarni/certyaml v0.11.0/go.mod h1:AiWJjkISlmC8shtMWPxsY9vMFpu+VYB8arwWr8079f4= +github.com/tsaarni/x500dn v1.1.0 h1:+rgqGj7LQEkdIIRLsYJm5S6M2dDBscb6/xiEcGW678s= +github.com/tsaarni/x500dn v1.1.0/go.mod h1:vzfi5pu5wr1eeFf9/0rIr5Bc1kxeyes4jFMCcp0wfCk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ulikunitz/lz v0.6.11 h1:KX3Wk0shhb9X7xo0gjmjGYgqOJ22ykxVmnvt6rBaG6E= @@ -478,8 +475,8 @@ github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zI github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= -github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= +github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= @@ -496,8 +493,6 @@ go.etcd.io/etcd/server/v3 v3.6.8 h1:U2strdSEy1U8qcSzRIdkYpvOPtBy/9i/IfaaCI9flZ4= go.etcd.io/etcd/server/v3 v3.6.8/go.mod h1:88dCtwUnSirkUoJbflQxxWXqtBSZa6lSG0Kuej+dois= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= -go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= -go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 h1:w3zlHYETbDwXyWHZlyyR58ZC39XGi8rAhkBgUgJ9d5w= @@ -577,8 +572,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= -golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= -golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= @@ -691,16 +686,18 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= -sigs.k8s.io/gateway-api v1.4.1 h1:NPxFutNkKNa8UfLd2CMlEuhIPMQgDQ6DXNKG9sHbJU8= -sigs.k8s.io/gateway-api v1.4.1/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= +sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= +sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= sigs.k8s.io/gateway-api-inference-extension v1.0.2 h1:JF/SCOSixHtYrXVIuBzmmjMOotWTT2SQSnVYKC9smmk= sigs.k8s.io/gateway-api-inference-extension v1.0.2/go.mod h1:JzP+NDgySAyRrLME1OzPyKIhKJQuo5gEN8Lawikp90g= +sigs.k8s.io/gateway-api/conformance v1.5.1 h1:5eruSMKcwKnkX42PFek8oO6BgPNBD5FbWbTcRV76KIw= +sigs.k8s.io/gateway-api/conformance v1.5.1/go.mod h1:mcvYR0Zll1i5hmcKn+jNbWdZTBls6s5GU+FPUFIceXw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09 h1:JQbPOwLjSztom+aSDQIi6UZq8V0Gbv7BjAlYQSgycCI= sigs.k8s.io/kubectl-validate v0.0.5-0.20250915070809-d2f2d68fba09/go.mod h1:n5AZlk9qPlp2JHChA4a4N/+lU3bJJ3X1Cig6VGQeqwE= -sigs.k8s.io/mcs-api v0.3.0 h1:LjRvgzjMrvO1904GP6XBJSnIX221DJMyQlZOYt9LAnM= -sigs.k8s.io/mcs-api v0.3.0/go.mod h1:zZ5CK8uS6HaLkxY4HqsmcBHfzHuNMrY2uJy8T7jffK4= +sigs.k8s.io/mcs-api v0.4.1 h1:rUygPnCZVS5xiZCzAi54Ngs9on6UQr7MNfx4uJXR2kA= +sigs.k8s.io/mcs-api v0.4.1/go.mod h1:zZ5CK8uS6HaLkxY4HqsmcBHfzHuNMrY2uJy8T7jffK4= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aigatewayroutes.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aigatewayroutes.yaml index 50ae8e086e..8c7ecfe3e7 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aigatewayroutes.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aigatewayroutes.yaml @@ -576,8 +576,14 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: Value is the value of HTTP Header - to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -714,8 +720,14 @@ spec: - RegularExpression type: string value: - description: Value is the value of HTTP Header to - be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -1462,8 +1474,14 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: Value is the value of HTTP Header - to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -1600,8 +1618,14 @@ spec: - RegularExpression type: string value: - description: Value is the value of HTTP Header to - be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml index 80b66ea552..859f27f002 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml @@ -289,7 +289,14 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: Value is the value of HTTP Header to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -697,7 +704,14 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: Value is the value of HTTP Header to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_backendsecuritypolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_backendsecuritypolicies.yaml index 80e171f5da..159f6f7c29 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_backendsecuritypolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_backendsecuritypolicies.yaml @@ -478,6 +478,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -753,6 +769,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -932,8 +987,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -950,6 +1006,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -965,6 +1034,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -1064,10 +1150,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -1100,6 +1187,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -1148,6 +1241,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -1161,6 +1256,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -1204,6 +1334,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -1328,6 +1510,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -1362,7 +1578,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -1382,12 +1598,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -1429,6 +1649,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -1438,22 +1689,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -1612,6 +1892,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -1705,6 +1991,11 @@ spec: - message: only one of clientID or clientIDRef must be set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - awsRoleArn - oidc @@ -2067,6 +2358,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -2342,6 +2649,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -2521,8 +2867,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -2539,6 +2886,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -2554,6 +2914,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -2653,10 +3030,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -2689,6 +3067,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -2737,6 +3121,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -2750,6 +3136,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -2793,6 +3214,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -2917,6 +3390,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -2951,7 +3458,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -2971,12 +3478,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -3018,6 +3529,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -3027,22 +3569,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -3201,6 +3772,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -3294,6 +3871,11 @@ spec: - message: only one of clientID or clientIDRef must be set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - oidc type: object @@ -3626,6 +4208,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -3904,6 +4502,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less + than or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -4085,8 +4722,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -4103,6 +4741,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -4118,6 +4769,23 @@ spec: time between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -4221,10 +4889,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is - marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -4258,8 +4927,14 @@ spec: passive: description: Passive passive check configuration properties: - baseEjectionTime: - default: 30s + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean + baseEjectionTime: + default: 30s description: BaseEjectionTime defines the base duration for which a host will be ejected on consecutive failures. @@ -4306,6 +4981,8 @@ spec: the maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -4319,6 +4996,43 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING + frames. + properties: + idleInterval: + description: IdleInterval specifies + how long a connection must be idle + before a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how + often to send HTTP/2 PING frames + to keep the connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how + long to wait for a PING response + before considering the connection + dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -4362,6 +5076,59 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has + not reported load metrics in this + long, stop using the reported weight. + Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than + 100ms are capped at 100ms. Default + 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -4490,6 +5257,40 @@ spec: must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -4524,7 +5325,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -4544,12 +5345,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -4593,6 +5398,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig + defines the weight for a specific + locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -4602,22 +5438,53 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' + ? has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? + has(self.dynamicModule) : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin + for RoundRobin, LeastRequest, and BackendUtilization load balancers. - rule: 'self.type == ''ConsistentHash'' ? - !has(self.zoneAware) : true ' + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing + is not supported for ConsistentHash load + balancers. Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: PreferLocal zone-aware routing + is not currently supported for BackendUtilization + load balancers. Only WeightedZones can + be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' + && has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? + !has(self.zoneAware) : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) + : true' + - message: EndpointOverride is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? + !has(self.endpointOverride) : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -4777,6 +5644,12 @@ spec: is received from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -4873,6 +5746,11 @@ spec: set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - oidc type: object @@ -5529,6 +6407,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -5804,6 +6698,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -5983,8 +6916,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -6001,6 +6935,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -6016,6 +6963,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -6115,10 +7079,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -6151,6 +7116,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -6199,6 +7170,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -6212,6 +7185,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -6255,6 +7263,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -6379,6 +7439,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -6413,7 +7507,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -6433,12 +7527,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -6480,6 +7578,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -6489,22 +7618,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -6663,6 +7821,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -6756,6 +7920,11 @@ spec: - message: only one of clientID or clientIDRef must be set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - awsRoleArn - oidc @@ -7118,6 +8287,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -7393,6 +8578,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -7572,8 +8796,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -7590,6 +8815,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -7605,6 +8843,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -7704,10 +8959,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -7740,6 +8996,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -7788,6 +9050,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -7801,6 +9065,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -7844,6 +9143,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -7968,6 +9319,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -8002,7 +9387,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -8022,12 +9407,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -8069,6 +9458,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -8078,22 +9498,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -8252,6 +9701,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -8345,6 +9800,11 @@ spec: - message: only one of clientID or clientIDRef must be set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - oidc type: object @@ -8677,6 +10137,22 @@ spec: via the Authorization header Bearer scheme to the upstream. If not specified, defaults to false. type: boolean + forwardIDToken: + description: |- + ForwardIDToken configures forwarding of the OIDC ID token to the upstream. + + If the configured header is "Authorization", EG forwards the ID token using + the "Bearer " prefix. For any other header, EG forwards the raw token value. + If not specified, the ID token will not be forwarded. + properties: + header: + description: Header is the upstream request header + that will carry the ID token. + minLength: 1 + type: string + required: + - header + type: object logoutPath: description: |- The path to log a user out, clearing their credential cookies. @@ -8955,6 +10431,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less + than or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -9136,8 +10651,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -9154,6 +10670,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -9169,6 +10698,23 @@ spec: time between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -9272,10 +10818,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is - marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -9309,6 +10856,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -9357,6 +10910,8 @@ spec: the maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -9370,6 +10925,43 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING + frames. + properties: + idleInterval: + description: IdleInterval specifies + how long a connection must be idle + before a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how + often to send HTTP/2 PING frames + to keep the connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how + long to wait for a PING response + before considering the connection + dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -9413,6 +11005,59 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has + not reported load metrics in this + long, stop using the reported weight. + Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than + 100ms are capped at 100ms. Default + 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -9541,6 +11186,40 @@ spec: must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -9575,7 +11254,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -9595,12 +11274,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -9644,6 +11327,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig + defines the weight for a specific + locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -9653,22 +11367,53 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' + ? has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? + has(self.dynamicModule) : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin + for RoundRobin, LeastRequest, and BackendUtilization load balancers. - rule: 'self.type == ''ConsistentHash'' ? - !has(self.zoneAware) : true ' + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing + is not supported for ConsistentHash load + balancers. Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: PreferLocal zone-aware routing + is not currently supported for BackendUtilization + load balancers. Only WeightedZones can + be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' + && has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? + !has(self.zoneAware) : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) + : true' + - message: EndpointOverride is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? + !has(self.endpointOverride) : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -9828,6 +11573,12 @@ spec: is received from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -9924,6 +11675,11 @@ spec: set rule: (has(self.clientID) && !has(self.clientIDRef)) || (!has(self.clientID) && has(self.clientIDRef)) + - message: forwardAccessToken cannot be true when forwardIDToken.header + is Authorization + rule: '!(has(self.forwardAccessToken) && self.forwardAccessToken + && has(self.forwardIDToken) && self.forwardIDToken.header.lowerAscii() + == ''authorization'')' required: - oidc type: object diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_mcproutes.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_mcproutes.yaml index 810470271e..d77a726f82 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_mcproutes.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_mcproutes.yaml @@ -353,7 +353,14 @@ spec: - RegularExpression type: string value: - description: Value is the value of HTTP Header to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -1180,6 +1187,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or + equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -1358,8 +1404,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -1376,6 +1423,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines the + http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -1391,6 +1451,23 @@ spec: active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -1490,9 +1567,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines the - number of unhealthy health checks required - before a backend host is marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -1525,6 +1604,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines the @@ -1573,6 +1658,8 @@ spec: percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -1586,6 +1673,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures HTTP/2 + connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how long + a connection must be idle before a PING + is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the connection + alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long to + wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -1629,6 +1751,57 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not reported + load metrics in this long, stop using the + reported weight. Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights are + recalculated. Values less than 100ms are + capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -1751,6 +1924,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -1785,7 +1992,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -1805,12 +2012,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -1852,6 +2063,36 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -1861,21 +2102,48 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? has(self.backendUtilization) + : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported for - RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported for - LeastRequest, Random, and RoundRobin load balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) : true' + - message: PreferLocal zone-aware routing is not supported + for ConsistentHash load balancers. Use weightedZones + instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not currently + supported for BackendUtilization load balancers. + Only WeightedZones can be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: ZoneAware routing is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for DynamicModule + load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -2033,6 +2301,12 @@ spec: upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -2339,6 +2613,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or + equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -2517,8 +2830,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -2535,6 +2849,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines the + http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -2550,6 +2877,23 @@ spec: active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -2649,9 +2993,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines the - number of unhealthy health checks required - before a backend host is marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -2684,6 +3030,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines the @@ -2732,6 +3084,8 @@ spec: percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -2745,6 +3099,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures HTTP/2 + connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how long + a connection must be idle before a PING + is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the connection + alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long to + wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -2788,6 +3177,57 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not reported + load metrics in this long, stop using the + reported weight. Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights are + recalculated. Values less than 100ms are + capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -2910,6 +3350,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -2944,7 +3418,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -2964,12 +3438,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -3011,6 +3489,36 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -3020,21 +3528,48 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? has(self.backendUtilization) + : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported for - RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported for - LeastRequest, Random, and RoundRobin load balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) : true' + - message: PreferLocal zone-aware routing is not supported + for ConsistentHash load balancers. Use weightedZones + instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not currently + supported for BackendUtilization load balancers. + Only WeightedZones can be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: ZoneAware routing is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for DynamicModule + load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -3192,6 +3727,12 @@ spec: upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -3232,6 +3773,15 @@ spec: For example, if the original request path is "/hello", and the path specified here is "/auth", then the path of the authorization request will be "/auth/hello". If the path is not specified, the path of the authorization request will be "/hello". + Only one of Path or PathOverride can be set. + type: string + pathOverride: + description: |- + PathOverride replaces the original request path in the authorization request. + If set, the path will be overridden to this value during authorization. + For example, if the original request path is "/hello", and PathOverride is set to "/auth", + then the path of the authorization request will be "/auth". + Only one of Path or PathOverride can be set. type: string type: object x-kubernetes-validations: @@ -3248,6 +3798,30 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''multicluster.x-k8s.io'' || f.group == ''gateway.envoyproxy.io'')) : true' + - message: only one of path or pathOverride can be specified + rule: '!(has(self.path) && has(self.pathOverride))' + includeRouteMetadata: + description: |- + IncludeRouteMetadata sends Envoy Gateway's built-in route metadata to the + external authorization service as context. + + This includes Envoy Gateway's built-in metadata for the selected route in + the "envoy-gateway" metadata namespace. + + The metadata is exposed under the "resources" field as a list of route + resource objects. For example: + + envoy-gateway: + resources: + - kind: HTTPRoute + name: backend + namespace: default + annotations: + foo: bar + + The resource object may include fields such as kind, namespace, name, + sectionName, and supported route annotations. + type: boolean recomputeRoute: description: |- RecomputeRoute clears the route cache and recalculates the routing decision. @@ -3255,6 +3829,51 @@ spec: route matching decisions. If the recomputation selects a new route, features targeting the new matched route will be applied. type: boolean + statusOnError: + description: |- + Sets the HTTP status that is returned when the authorization service returns an error + or cannot be reached. Defaults to 403 Forbidden. + Only 4xx and 5xx status codes are supported. + enum: + - 400 + - 401 + - 402 + - 403 + - 404 + - 405 + - 406 + - 407 + - 408 + - 409 + - 410 + - 411 + - 412 + - 413 + - 414 + - 415 + - 416 + - 417 + - 421 + - 422 + - 423 + - 424 + - 426 + - 428 + - 429 + - 431 + - 500 + - 501 + - 502 + - 503 + - 504 + - 505 + - 506 + - 507 + - 508 + - 510 + - 511 + format: int32 + type: integer timeout: description: |- Timeout defines the timeout for requests to the external authorization service. @@ -3634,6 +4253,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -3813,8 +4471,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -3831,6 +4490,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -3846,6 +4518,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -3945,10 +4634,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -3981,6 +4671,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -4029,6 +4725,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -4042,6 +4740,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -4085,6 +4818,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -4209,6 +4994,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -4243,7 +5062,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -4263,12 +5082,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -4310,6 +5133,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -4319,22 +5173,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -4493,6 +5376,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -5014,7 +5903,14 @@ spec: - RegularExpression type: string value: - description: Value is the value of HTTP Header to be matched. + description: |- + Value is the value of HTTP Header to be matched. + + Must consist of printable US-ASCII characters, optionally separated + by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + + maxLength: 4096 minLength: 1 type: string @@ -5841,6 +6737,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or + equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -6019,8 +6954,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -6037,6 +6973,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines the + http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -6052,6 +7001,23 @@ spec: active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -6151,9 +7117,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines the - number of unhealthy health checks required - before a backend host is marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -6186,6 +7154,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines the @@ -6234,6 +7208,8 @@ spec: percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -6247,6 +7223,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures HTTP/2 + connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how long + a connection must be idle before a PING + is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the connection + alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long to + wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -6290,6 +7301,57 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not reported + load metrics in this long, stop using the + reported weight. Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights are + recalculated. Values less than 100ms are + capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -6412,6 +7474,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -6446,7 +7542,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -6466,12 +7562,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -6513,6 +7613,36 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -6522,21 +7652,48 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? has(self.backendUtilization) + : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported for - RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported for - LeastRequest, Random, and RoundRobin load balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) : true' + - message: PreferLocal zone-aware routing is not supported + for ConsistentHash load balancers. Use weightedZones + instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not currently + supported for BackendUtilization load balancers. + Only WeightedZones can be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: ZoneAware routing is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for DynamicModule + load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -6694,6 +7851,12 @@ spec: upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -7000,6 +8163,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than or + equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -7178,8 +8380,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -7196,6 +8399,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines the + http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -7211,6 +8427,23 @@ spec: active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -7310,9 +8543,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines the - number of unhealthy health checks required - before a backend host is marked unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -7345,6 +8580,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines the @@ -7393,6 +8634,8 @@ spec: percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -7406,6 +8649,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures HTTP/2 + connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how long + a connection must be idle before a PING + is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the connection + alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long to + wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -7449,6 +8727,57 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not reported + load metrics in this long, stop using the + reported weight. Defaults to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights are + recalculated. Values less than 100ms are + capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -7571,6 +8900,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -7605,7 +8968,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -7625,12 +8988,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -7672,6 +9039,36 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -7681,21 +9078,48 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? has(self.backendUtilization) + : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported for - RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported for - LeastRequest, Random, and RoundRobin load balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) : true' + - message: PreferLocal zone-aware routing is not supported + for ConsistentHash load balancers. Use weightedZones + instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not currently + supported for BackendUtilization load balancers. + Only WeightedZones can be used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: ZoneAware routing is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for DynamicModule + load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -7853,6 +9277,12 @@ spec: upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. @@ -7893,6 +9323,15 @@ spec: For example, if the original request path is "/hello", and the path specified here is "/auth", then the path of the authorization request will be "/auth/hello". If the path is not specified, the path of the authorization request will be "/hello". + Only one of Path or PathOverride can be set. + type: string + pathOverride: + description: |- + PathOverride replaces the original request path in the authorization request. + If set, the path will be overridden to this value during authorization. + For example, if the original request path is "/hello", and PathOverride is set to "/auth", + then the path of the authorization request will be "/auth". + Only one of Path or PathOverride can be set. type: string type: object x-kubernetes-validations: @@ -7909,6 +9348,30 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''multicluster.x-k8s.io'' || f.group == ''gateway.envoyproxy.io'')) : true' + - message: only one of path or pathOverride can be specified + rule: '!(has(self.path) && has(self.pathOverride))' + includeRouteMetadata: + description: |- + IncludeRouteMetadata sends Envoy Gateway's built-in route metadata to the + external authorization service as context. + + This includes Envoy Gateway's built-in metadata for the selected route in + the "envoy-gateway" metadata namespace. + + The metadata is exposed under the "resources" field as a list of route + resource objects. For example: + + envoy-gateway: + resources: + - kind: HTTPRoute + name: backend + namespace: default + annotations: + foo: bar + + The resource object may include fields such as kind, namespace, name, + sectionName, and supported route annotations. + type: boolean recomputeRoute: description: |- RecomputeRoute clears the route cache and recalculates the routing decision. @@ -7916,6 +9379,51 @@ spec: route matching decisions. If the recomputation selects a new route, features targeting the new matched route will be applied. type: boolean + statusOnError: + description: |- + Sets the HTTP status that is returned when the authorization service returns an error + or cannot be reached. Defaults to 403 Forbidden. + Only 4xx and 5xx status codes are supported. + enum: + - 400 + - 401 + - 402 + - 403 + - 404 + - 405 + - 406 + - 407 + - 408 + - 409 + - 410 + - 411 + - 412 + - 413 + - 414 + - 415 + - 416 + - 417 + - 421 + - 422 + - 423 + - 424 + - 426 + - 428 + - 429 + - 431 + - 500 + - 501 + - 502 + - 503 + - 504 + - 505 + - 506 + - 507 + - 508 + - 510 + - 511 + format: int32 + type: integer timeout: description: |- Timeout defines the timeout for requests to the external authorization service. @@ -8295,6 +9803,45 @@ spec: minimum: 0 type: integer type: object + retryBudget: + description: |- + RetryBudget provides settings for retry budget, which limits the number of retries in a given percentage. + RetryBudget take precedence over maxParallelRetries. + properties: + minRetryConcurrency: + description: |- + MinRetryConcurrency specifies the minimum retry concurrency allowed for the retry budget. + For example, a budget of 20% with a minimum retry concurrency of 3 + will allow 5 active retries while there are 25 active requests. + If there are 2 active requests, there are still 3 active retries + allowed because of the minimum retry concurrency. + Defaults to 3. + format: int32 + type: integer + percent: + description: |- + Percent specifies the limit on concurrent retries as a percentage [0, 100] of + the sum of active requests and active pending requests. + properties: + denominator: + default: 100 + format: int32 + minimum: 1 + type: integer + numerator: + format: int32 + minimum: 0 + type: integer + required: + - numerator + type: object + x-kubernetes-validations: + - message: numerator must be less than + or equal to denominator + rule: self.numerator <= self.denominator + required: + - percent + type: object type: object connection: description: Connection includes backend connection @@ -8474,8 +10021,9 @@ spec: type: array hostname: description: |- - Hostname defines the HTTP host that will be requested during health checking. - Default: HTTPRoute or GRPCRoute hostname. + Hostname defines the HTTP Host header used for active HTTP health checks. + Host selection uses this order: this field, the associated Backend endpoint + hostname if available, then the effective Route hostname. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -8492,6 +10040,19 @@ spec: maxLength: 1024 minLength: 1 type: string + retriableStatuses: + description: |- + RetriableStatuses defines a list of HTTP response statuses considered retriable. + Responses matching these statuses count towards the unhealthy threshold but + do not result in the host being considered immediately unhealthy. + The expected statuses take precedence for any range overlaps with this field. + items: + description: HTTPStatus defines + the http status code. + maximum: 599 + minimum: 100 + type: integer + type: array required: - path type: object @@ -8507,6 +10068,23 @@ spec: between active health checks. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + overrides: + description: |- + Overrides defines the configuration of the overriding health check settings for all endpoints + in the backend cluster. This allows customization of port and other settings that may differ + from the main service configuration. + properties: + port: + description: |- + Port overrides the health check port. + If not set, the endpoint's serving port is used for health checks. + This is useful when health checks are served on a different port than + the main service port (e.g., port 443 for service, port 9090 for health checks). + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object tcp: description: |- TCP defines the configuration of tcp health checker. @@ -8606,10 +10184,11 @@ spec: type: string unhealthyThreshold: default: 3 - description: UnhealthyThreshold defines - the number of unhealthy health checks - required before a backend host is marked - unhealthy. + description: |- + UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + Without RetriableStatuses configured, any health check failure results in the host being immediately + considered unhealthy. When RetriableStatuses is set, health checks returning those statuses are retried + up to this threshold before the host is marked unhealthy. format: int32 minimum: 1 type: integer @@ -8642,6 +10221,12 @@ spec: passive: description: Passive passive check configuration properties: + alwaysEjectOneEndpoint: + default: false + description: |- + AlwaysEjectOneEndpoint defines whether at least one host should be ejected, + regardless of MaxEjectionPercent. + type: boolean baseEjectionTime: default: 30s description: BaseEjectionTime defines @@ -8690,6 +10275,8 @@ spec: maximum percentage of hosts in a cluster that can be ejected. format: int32 + maximum: 100 + minimum: 0 type: integer splitExternalLocalOriginErrors: default: false @@ -8703,6 +10290,41 @@ spec: description: HTTP2 provides HTTP/2 configuration for backend connections. properties: + connectionKeepalive: + description: ConnectionKeepalive configures + HTTP/2 connection keepalive using PING frames. + properties: + idleInterval: + description: IdleInterval specifies how + long a connection must be idle before + a PING is sent. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + interval: + description: Interval specifies how often + to send HTTP/2 PING frames to keep the + connection alive. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + intervalJitter: + description: |- + IntervalJitter specifies a random jitter percentage added to each interval. + Defaults to 15% if not specified. + format: int32 + maximum: 100 + minimum: 0 + type: integer + timeout: + description: Timeout specifies how long + to wait for a PING response before considering + the connection dead. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object + x-kubernetes-validations: + - message: timeout must be less than interval + rule: '!has(self.timeout) || !has(self.interval) + || duration(self.timeout) < duration(self.interval)' initialConnectionWindowSize: allOf: - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ @@ -8746,6 +10368,58 @@ spec: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints. Defaults to `LeastRequest`. properties: + backendUtilization: + description: |- + BackendUtilization defines the configuration when the load balancer type is + set to BackendUtilization. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least this long before the endpoint weight will be used. + Default is 10s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + errorUtilizationPenaltyPercent: + description: |- + ErrorUtilizationPenaltyPercent adjusts endpoint weights based on the error rate (eps/qps). + This is expressed as a percentage-based integer where 100 represents 1.0, 150 represents 1.5, etc. + + For example: + - 100 => 1.0x + - 120 => 1.2x + - 200 => 2.0x + + Must be non-negative. + format: int32 + minimum: 0 + type: integer + keepResponseHeaders: + default: false + description: |- + KeepResponseHeaders keeps the ORCA load report headers/trailers before sending the response to the client. + Defaults to false. + type: boolean + metricNamesForComputingUtilization: + description: |- + Metric names used to compute utilization if application_utilization is not set. + For map fields in ORCA proto, use the form ".", e.g., "named_metrics.foo". + items: + type: string + type: array + weightExpirationPeriod: + description: If a given endpoint has not + reported load metrics in this long, + stop using the reported weight. Defaults + to 3m. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + weightUpdatePeriod: + description: How often endpoint weights + are recalculated. Values less than 100ms + are capped at 100ms. Default 1s. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + type: object consistentHash: description: |- ConsistentHash defines the configuration when the load balancer type is @@ -8870,6 +10544,40 @@ spec: the queryParams field must be set. rule: 'self.type == ''QueryParams'' ? has(self.queryParams) : !has(self.queryParams)' + dynamicModule: + description: |- + DynamicModule defines the configuration when the load balancer type is + set to DynamicModule. The referenced module must be registered in the + EnvoyProxy resource's dynamicModules allowlist. + properties: + config: + description: |- + Config is optional configuration for the module's load balancer + implementation. This is serialized and passed to the module's + initialization function. + x-kubernetes-preserve-unknown-fields: true + lbPolicyName: + description: |- + LBPolicyName identifies a specific load balancer implementation within + the dynamic module. A single shared library can contain multiple LB + policy implementations. This value is passed to the module's + initialization function to select the appropriate implementation. + maxLength: 253 + minLength: 1 + type: string + name: + description: |- + Name references a dynamic module registered in the EnvoyProxy resource's + dynamicModules list. The referenced module must exist in the registry; + otherwise, the policy will be rejected. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ + type: string + required: + - lbPolicyName + - name + type: object endpointOverride: description: |- EndpointOverride defines the configuration for endpoint override. @@ -8904,7 +10612,7 @@ spec: description: |- SlowStart defines the configuration related to the slow start load balancer policy. If set, during slow start window, traffic sent to the newly added hosts will gradually increase. - Currently this is only supported for RoundRobin and LeastRequest load balancers + Supported for RoundRobin, LeastRequest, and BackendUtilization load balancers. properties: window: description: |- @@ -8924,12 +10632,16 @@ spec: "ConsistentHash", "LeastRequest", "Random", - "RoundRobin". + "RoundRobin", + "BackendUtilization", + "DynamicModule". enum: - ConsistentHash - LeastRequest - Random - RoundRobin + - BackendUtilization + - DynamicModule type: string zoneAware: description: ZoneAware defines the configuration @@ -8971,6 +10683,37 @@ spec: minimum: 0 type: integer type: object + weightedZones: + description: |- + WeightedZones configures weight-based traffic distribution across locality zones. + Traffic is distributed proportionally based on the sum of all zone weights. + items: + description: WeightedZoneConfig defines + the weight for a specific locality + zone. + properties: + weight: + description: |- + Weight defines the weight for this locality. + Higher values receive more traffic. The actual traffic distribution + is proportional to this value relative to other localities. + format: int32 + type: integer + zone: + description: |- + Zone specifies the topology zone this weight applies to. + The value should match the topology.kubernetes.io/zone label + of the nodes where endpoints are running. + Zones not listed in the configuration receive a default weight of 1. + type: string + required: + - weight + - zone + type: object + type: array + x-kubernetes-list-map-keys: + - zone + x-kubernetes-list-type: map type: object required: - type @@ -8980,22 +10723,51 @@ spec: consistentHash field needs to be set. rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash) : !has(self.consistentHash)' + - message: If LoadBalancer type is BackendUtilization, + backendUtilization field needs to be set. + rule: 'self.type == ''BackendUtilization'' ? + has(self.backendUtilization) : !has(self.backendUtilization)' + - message: If LoadBalancer type is DynamicModule, + dynamicModule field needs to be set. + rule: 'self.type == ''DynamicModule'' ? has(self.dynamicModule) + : !has(self.dynamicModule)' - message: Currently SlowStart is only supported - for RoundRobin and LeastRequest load balancers. - rule: 'self.type in [''Random'', ''ConsistentHash''] - ? !has(self.slowStart) : true ' - - message: Currently ZoneAware is only supported - for LeastRequest, Random, and RoundRobin load - balancers. - rule: 'self.type == ''ConsistentHash'' ? !has(self.zoneAware) - : true ' + for RoundRobin, LeastRequest, and BackendUtilization + load balancers. + rule: 'self.type in [''Random'', ''ConsistentHash'', + ''DynamicModule''] ? !has(self.slowStart) + : true' + - message: PreferLocal zone-aware routing is not + supported for ConsistentHash load balancers. + Use weightedZones instead. + rule: 'self.type == ''ConsistentHash'' && has(self.zoneAware) + ? !has(self.zoneAware.preferLocal) : true' + - message: PreferLocal zone-aware routing is not + currently supported for BackendUtilization + load balancers. Only WeightedZones can be + used with BackendUtilization. + rule: 'self.type == ''BackendUtilization'' && + has(self.zoneAware) ? !has(self.zoneAware.preferLocal) + : true' + - message: ZoneAware routing is not supported + for DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.zoneAware) + : true' + - message: ZoneAware PreferLocal and WeightedZones + cannot be specified together. + rule: 'has(self.zoneAware) ? !(has(self.zoneAware.preferLocal) + && has(self.zoneAware.weightedZones)) : true' + - message: EndpointOverride is not supported for + DynamicModule load balancers. + rule: 'self.type == ''DynamicModule'' ? !has(self.endpointOverride) + : true' proxyProtocol: description: ProxyProtocol enables the Proxy Protocol when communicating with the backend. properties: version: description: |- - Version of ProxyProtol + Version of ProxyProtocol Valid ProxyProtocolVersion values are "V1" "V2" @@ -9154,6 +10926,12 @@ spec: from the upstream. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + streamIdleTimeout: + description: |2- + The stream idle timeout defines the amount of time a stream can exist without any upstream or downstream activity. + If not specified, StreamIdleTimeout is inherited from the listener-level setting, which can be configured via ClientTrafficPolicy. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string type: object tcp: description: Timeout settings for TCP. diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_quotapolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_quotapolicies.yaml index 7960e1881a..81621a0773 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_quotapolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_quotapolicies.yaml @@ -144,7 +144,7 @@ spec: required: - name type: object - maxItems: 16 + maxItems: 64 type: array methods: description: |- @@ -250,6 +250,12 @@ spec: description: SourceCIDR is the client IP Address range to match on. properties: + invert: + default: false + description: |- + Invert specifies whether the source range match result will be inverted. + When true, the rule matches when the client IP is not in the specified range(s). + type: boolean type: default: Exact enum: diff --git a/release-notes/v0.7.0.md b/release-notes/v0.7.0.md new file mode 100644 index 0000000000..9eaec52e4a --- /dev/null +++ b/release-notes/v0.7.0.md @@ -0,0 +1,128 @@ +# Envoy AI Gateway v0.7.0 + +> Plain-markdown copy of the v0.7.0 release notes, suitable for pasting into the GitHub release body. The full rendered version lives at https://aigateway.envoyproxy.io/release-notes/v0.7. + +Envoy AI Gateway v0.7.0 adds **hostname-based routing** to `AIGatewayRoute`, enabling multi-tenant deployments where different hostnames expose different model sets through a single Gateway. A new **Anthropic Messages → AWS Bedrock Converse** translator lets Anthropic-native clients reach Bedrock without switching protocols. OpenAI **audio transcription and translation** endpoints arrive alongside **Azure OpenAI Responses API** support. Quota-aware rate limiting takes its first step with **backend rate limit filter injection** for `QuotaPolicy`. **Claude Opus 4.7** gains full reasoning support including the `display` parameter and `xhigh` effort tier. Anthropic-to-OpenAI translation now handles **reasoning blocks and images** end-to-end. MCP `tools/list` responses **respect authorization rules**, and multimodal support grows with `audio_url` and `video_url` content types. Several SSE streaming and provider translation bugs are fixed. + +## ✨ New Features + +### Multi-Tenant Hostname Routing + +- **Hostname-based model scoping on `AIGatewayRoute`** — Serve different model sets from a single Gateway by assigning hostnames to each `AIGatewayRoute`. The `/v1/models` endpoint automatically returns only the models declared by routes matching the request's `Host` header, so tenants on `teamA.ai.example.com` and `teamB.ai.example.com` each see their own catalog without separate Gateways. Wildcard hostnames (`*.ai.example.com`) are supported following the Gateway API hostname matching rules. + +### Provider Translation + +- **Anthropic `/v1/messages` → AWS Bedrock Converse API** — Send requests in Anthropic Messages format and have them translated to Bedrock's Converse and ConverseStream APIs automatically. Supports text, images, tool use, thinking blocks, and streaming — so Anthropic-native clients can reach any Bedrock model without changing their integration. Complements the existing OpenAI → Bedrock Converse and Anthropic → Bedrock InvokeModel paths. +- **Reasoning and image support for Anthropic-to-OpenAI translation** — The Anthropic `/v1/messages` → OpenAI `/v1/chat/completions` path now handles thinking/reasoning content and image blocks end-to-end. Thinking config (enabled/disabled/adaptive) passes through, thinking and redacted_thinking blocks are preserved in multi-turn conversations, and image blocks (base64 and URL) convert to OpenAI `image_url` format. Previously these were silently dropped. +- **Claude Opus 4.7 and Mythos Preview reasoning** — Full support for Claude Opus 4.7's reasoning features: the `display` parameter (`summarized`/`omitted`) controls thinking content visibility, and `xhigh` joins the reasoning effort tiers for long-horizon agentic and coding tasks. Both `claude-opus-4-7` and `claude-mythos-preview` models are recognized for effort-based thinking control. +- **Custom request paths for Anthropic backends via `prefix`** — The `prefix` field on `VersionedAPISchema` now works for Anthropic-schema backends, producing endpoints like `/{prefix}/messages` instead of the default `/v1/messages`. Useful for routing to Anthropic-compatible providers that use a non-standard path. +- **Anthropic `anthropic-beta` header forwarded to AWSAnthropic** — The `anthropic-beta` request header is now mapped into the `anthropic_beta` body field when routing to AWSAnthropic backends, so beta features like extended thinking and token counting work through the gateway without manual body rewriting. + +### OpenAI API Compatibility + +- **Audio transcription and translation endpoints** — Full data-plane support for OpenAI's `/v1/audio/transcriptions` (Whisper transcription) and `/v1/audio/translations` (Whisper translation) endpoints. These accept `multipart/form-data` requests containing audio files, enabling speech-to-text workloads to flow through the gateway with the same auth, rate limiting, and observability as other traffic. +- **Azure OpenAI Responses API** — The OpenAI-compatible `/v1/responses` endpoint now works with Azure OpenAI backends, routing requests to Azure's `/openai/responses?api-version=...` path while preserving existing request and response handling. Azure users get Responses API support without changing client code. +- **`audio_url` and `video_url` content types** — OpenAI chat completion requests can now include `audio_url` and `video_url` content parts, enabling multimodal audio and video inputs for compatible backends like vLLM with phi-4-mm and Qwen 3.5 models. + +### Quota-Aware Routing + +- **Backend quota rate limit filter injection** — First step toward quota-aware routing: the controller now injects a backend rate limit filter when a `QuotaPolicy` is attached to an `AIServiceBackend`. The QuotaPolicy controller reconciles the policy, builds rate limit descriptor trees, and configures the rate limit service. This enables per-backend request throttling based on upstream provider quotas. + +### MCP Gateway + +- **Authorization-filtered `tools/list` responses** — MCP `tools/list` now applies the same authorization rules used by `tools/call`, omitting tools the caller isn't authorized to invoke. Prevents unauthorized callers from discovering tool names and avoids wasted LLM turns on tools that would fail at call time. + +### Observability + +- **Smarter log redaction preserves developer-authored metadata** — Debug log redaction (`--enableRedaction`) no longer masks developer-authored schema metadata that was previously over-redacted: tool definition `description` and `parameters`, tool call `function.name`, `response_format.json_schema`, and `guided_json` are now visible in debug logs. User-provided content and AI-generated text remain redacted, making debug logs significantly more useful without compromising privacy. + +## 🔗 API Updates + +- **`AIGatewayRoute.spec.hostnames`** — New optional field accepting a list of hostnames for hostname-based request filtering. When specified, the generated `HTTPRoute` includes these hostnames, and the `/v1/models` endpoint scopes its response to models from matching routes. Follows Gateway API hostname semantics including wildcard support. +- **`AIGatewayRoute.spec.rules` capped at 15** — Maximum rules per `AIGatewayRoute` reduced from 128 to 15 to match the Gateway API `HTTPRoute` limit (one slot is reserved for a controller-injected catch-all rule). To configure more rules on the same Gateway, split them across multiple `AIGatewayRoute` resources. +- **`VersionedAPISchema.prefix` supported for Anthropic** — The `prefix` field now applies to Anthropic-schema backends in addition to OpenAI. The `version` field is ignored for Anthropic; use `prefix` for custom paths. Note: `prefix` is ignored for `AWSAnthropic` and `GCPAnthropic` as these override paths internally. +- **`QuotaPolicy` rate limit filter injection (runtime enforcement)** — The `QuotaPolicy` CRD (introduced as API-only in v0.6) now has its first runtime behavior: when attached to an `AIServiceBackend`, a backend rate limit filter is injected to enforce quota-based throttling. Full quota-aware routing across multiple backends is planned for future releases. + +## 🐛 Bug Fixes + +- **SSE parser handles fields without space after colon** — The SSE event parser now correctly handles fields formatted as `data:{json}` (no space after the colon), in addition to the standard `data: {json}`. Fixes silent field drops when proxying responses from providers that omit the optional space. +- **Responses API streaming SSE buffering** — OpenAI Responses API and speech streaming translators now buffer incomplete SSE events across response body chunks instead of treating each chunk as self-contained. Fixes dropped or mangled events when TCP segment boundaries split an SSE event mid-frame. +- **Responses API token usage from incomplete and failed streams** — Token usage is now captured from `response.incomplete` and `response.failed` SSE events, not just `response.completed`. Streams that hit `max_output_tokens` or encounter post-generation failures no longer report zero tokens. +- **Nil output guard in AWS Bedrock response translator** — Bedrock can return HTTP 200 with no `output` field (e.g. guardrail interventions or `UnknownOperationException`). Previously this caused a nil-pointer panic in the ext-proc; now it returns a clean error to the caller. +- **Comprehensive Gemini finish-reason mapping** — Gemini finish reasons like `SAFETY`, `BLOCKLIST`, `RECITATION`, `MALFORMED_FUNCTION_CALL`, and others now map to their correct OpenAI equivalents instead of all falling through to `content_filter`. Unknown reasons map to `error` rather than silently misreporting as a content filter event. +- **Empty delta in GCP Vertex AI streaming chunks** — Streaming response chunks from GCP Vertex AI that lack candidate content now emit an empty `delta` object instead of omitting the field, conforming to the OpenAI streaming contract and fixing parse errors in strict clients. +- **Typeless assistant output messages in Responses API** — Multi-turn Responses API inputs that include assistant messages without an explicit `type: "message"` field (e.g. from OpenCode) now parse correctly. Previously these were treated as easy-input messages, causing unmarshalling failures on `output_text` content blocks. + +## 📖 Upgrade Guidance + +### Using Hostname-Based Routing + +To serve different model sets per hostname, add `hostnames` to your `AIGatewayRoute`: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: team-a-route +spec: + hostnames: + - "team-a.ai.example.com" + rules: + - matches: + - headers: + - name: x-ai-eg-model + value: gpt-4o + backendRefs: + - name: openai-backend +``` + +Routes without `hostnames` remain accessible on all hosts. When at least one route uses hostname scoping, the `/v1/models` endpoint automatically returns only the models for the matching host. + +### Rules-Per-Route Limit + +`AIGatewayRoute.spec.rules` is now capped at 15 (down from 128) to match the Gateway API HTTPRoute limit. If you have routes with more than 15 rules, split them across multiple `AIGatewayRoute` resources attached to the same Gateway. + +### Using Anthropic Prefix + +If you route Anthropic traffic to a provider with a non-standard path, use the `prefix` field: + +```yaml +schema: + name: Anthropic + prefix: /custom/v2 # produces /custom/v2/messages +``` + +Note: `prefix` is ignored for `AWSAnthropic` and `GCPAnthropic` backends as they override paths internally. + +### Adopting Claude Opus 4.7 Reasoning + +If you use Claude Opus 4.7 or Mythos Preview models, note that `display` defaults to `omitted` (unlike earlier Claude models which default to `summarized`). To receive summarized thinking content, set `display: "summarized"` explicitly. The new `xhigh` reasoning effort tier is available for long-horizon agentic tasks. + +## 📦 Dependency Versions + +| Dependency | Version | +| ------------------------------- | ------- | +| Go | 1.26.2 | +| Envoy Gateway | v1.7.0 | +| Envoy Proxy | v1.37 | +| Gateway API | v1.4.1 | +| Gateway API Inference Extension | v1.0.2 | +| MCP Go SDK | v1.6.0 | + +## 🙏 Acknowledgements + +We extend our gratitude to all contributors who made this release possible. Special thanks to: + +- The growing community of adopters for their valuable feedback and production insights +- Everyone who reported bugs, submitted PRs, and participated in design discussions +- The Envoy Gateway team for their continued collaboration + +## 🔮 What's Next + +We're already working on features for future releases: + +- **Full quota-aware routing** — building on the rate limit filter injection landed in v0.7 to route around rate-limited upstreams automatically across multiple backends +- **MCPBackend CRD** — a dedicated custom resource for MCP backend servers, decoupling MCP backend configuration from MCPRoute +- **Expanded multimodal support** — additional audio, video, and image generation backends across cloud providers +- **Deeper MCP authorization** — finer-grained policy across tools, resources, and prompts +- **More provider translation paths** — filling coverage gaps across Anthropic, Bedrock, and Vertex AI diff --git a/site/src/data/releases/index.json b/site/src/data/releases/index.json index 48505b7796..7498de3a99 100644 --- a/site/src/data/releases/index.json +++ b/site/src/data/releases/index.json @@ -2,7 +2,7 @@ "metadata": { "title": "Release Notes", "description": "Release Notes for Envoy AI Gateway", - "lastUpdated": "2026-05-01" + "lastUpdated": "2026-06-04" }, "community": { "slack": "https://envoyproxy.slack.com/archives/C07Q4N24VAA", diff --git a/site/src/data/releases/v0.6.json b/site/src/data/releases/v0.6.json index 855ace5b0d..5318baf9aa 100644 --- a/site/src/data/releases/v0.6.json +++ b/site/src/data/releases/v0.6.json @@ -3,7 +3,7 @@ "version": "v0.6", "title": "Envoy AI Gateway v0.6.x", "subtitle": "v1beta1 API graduation, AWS Bedrock InvokeModel for Claude, Anthropic endpoint on OpenAI backends, Gemini embeddings and context caching, MCP per-backend header forwarding, GKE Workload Identity, and request/response body redaction", - "badge": "Latest", + "badge": "Stable", "badgeType": "milestone" }, "releases": [ @@ -297,6 +297,7 @@ } ], "navigation": { - "previous": { "version": "v0.5.x Series", "path": "/release-notes/v0.5" } + "previous": { "version": "v0.5.x Series", "path": "/release-notes/v0.5" }, + "next": { "version": "v0.7.x Series", "path": "/release-notes/v0.7" } } } diff --git a/site/src/data/releases/v0.7.json b/site/src/data/releases/v0.7.json new file mode 100644 index 0000000000..680cb4adb2 --- /dev/null +++ b/site/src/data/releases/v0.7.json @@ -0,0 +1,186 @@ +{ + "series": { + "version": "v0.7", + "title": "Envoy AI Gateway v0.7.x", + "subtitle": "Hostname-based routing, Anthropic-to-Bedrock Converse translation, audio transcription/translation endpoints, Azure OpenAI Responses API, quota-aware rate limiting, Claude Opus 4.7 reasoning, and MCP authorization-aware tool listing.", + "badge": "Latest", + "badgeType": "milestone" + }, + "releases": [ + { + "version": "v0.7.0", + "date": "June 4, 2026", + "type": "minor", + "tags": [ + { "text": "Hostname Routing", "type": "feature" }, + { "text": "Anthropic → Bedrock", "type": "feature" }, + { "text": "Audio Transcription", "type": "feature" }, + { "text": "Azure Responses API", "type": "feature" }, + { "text": "Quota Rate Limiting", "type": "feature" }, + { "text": "Opus 4.7 Reasoning", "type": "feature" }, + { "text": "MCP Auth Filtering", "type": "feature" }, + { "text": "Multimodal A/V", "type": "feature" } + ], + "overview": "Envoy AI Gateway v0.7.0 adds hostname-based routing to AIGatewayRoute, enabling multi-tenant deployments where different hostnames expose different model sets through a single Gateway. A new Anthropic Messages → AWS Bedrock Converse translator lets Anthropic-native clients reach Bedrock without switching protocols. OpenAI audio transcription and translation endpoints arrive alongside Azure OpenAI Responses API support. Quota-aware rate limiting takes its first step with backend rate limit filter injection for QuotaPolicy. Claude Opus 4.7 gains full reasoning support including the display parameter and xhigh effort tier. Anthropic-to-OpenAI translation now handles reasoning blocks and images end-to-end. MCP tools/list responses respect authorization rules, and multimodal support grows with audio_url and video_url content types. Several SSE streaming and provider translation bugs are fixed.", + "features": [ + { + "title": "Multi-Tenant Hostname Routing", + "items": [ + { + "title": "Hostname-based model scoping on AIGatewayRoute", + "description": "Serve different model sets from a single Gateway by assigning hostnames to each AIGatewayRoute. The /v1/models endpoint automatically returns only the models declared by routes matching the request's Host header, so tenants on teamA.ai.example.com and teamB.ai.example.com each see their own catalog without separate Gateways. Wildcard hostnames (*.ai.example.com) are supported following the Gateway API hostname matching rules." + } + ] + }, + { + "title": "Provider Translation", + "items": [ + { + "title": "Anthropic /v1/messages → AWS Bedrock Converse API", + "description": "Send requests in Anthropic Messages format and have them translated to Bedrock's Converse and ConverseStream APIs automatically. Supports text, images, tool use, thinking blocks, and streaming — so Anthropic-native clients can reach any Bedrock model without changing their integration. Complements the existing OpenAI → Bedrock Converse and Anthropic → Bedrock InvokeModel paths." + }, + { + "title": "Reasoning and image support for Anthropic-to-OpenAI translation", + "description": "The Anthropic /v1/messages → OpenAI /v1/chat/completions path now handles thinking/reasoning content and image blocks end-to-end. Thinking config (enabled/disabled/adaptive) passes through, thinking and redacted_thinking blocks are preserved in multi-turn conversations, and image blocks (base64 and URL) convert to OpenAI image_url format. Previously these were silently dropped." + }, + { + "title": "Claude Opus 4.7 and Mythos Preview reasoning", + "description": "Full support for Claude Opus 4.7's reasoning features: the display parameter (summarized/omitted) controls thinking content visibility, and xhigh joins the reasoning effort tiers for long-horizon agentic and coding tasks. Both claude-opus-4-7 and claude-mythos-preview models are recognized for effort-based thinking control." + }, + { + "title": "Custom request paths for Anthropic backends via prefix", + "description": "The prefix field on VersionedAPISchema now works for Anthropic-schema backends, producing endpoints like /{prefix}/messages instead of the default /v1/messages. Useful for routing to Anthropic-compatible providers that use a non-standard path." + }, + { + "title": "Anthropic anthropic-beta header forwarded to AWSAnthropic", + "description": "The anthropic-beta request header is now mapped into the anthropic_beta body field when routing to AWSAnthropic backends, so beta features like extended thinking and token counting work through the gateway without manual body rewriting." + } + ] + }, + { + "title": "OpenAI API Compatibility", + "items": [ + { + "title": "Audio transcription and translation endpoints", + "description": "Full data-plane support for OpenAI's /v1/audio/transcriptions (Whisper transcription) and /v1/audio/translations (Whisper translation) endpoints. These accept multipart/form-data requests containing audio files, enabling speech-to-text workloads to flow through the gateway with the same auth, rate limiting, and observability as other traffic." + }, + { + "title": "Azure OpenAI Responses API", + "description": "The OpenAI-compatible /v1/responses endpoint now works with Azure OpenAI backends, routing requests to Azure's /openai/responses?api-version=... path while preserving existing request and response handling. Azure users get Responses API support without changing client code." + }, + { + "title": "audio_url and video_url content types", + "description": "OpenAI chat completion requests can now include audio_url and video_url content parts, enabling multimodal audio and video inputs for compatible backends like vLLM with phi-4-mm and Qwen 3.5 models." + } + ] + }, + { + "title": "Quota-Aware Routing", + "items": [ + { + "title": "Backend quota rate limit filter injection", + "description": "First step toward quota-aware routing: the controller now injects a backend rate limit filter when a QuotaPolicy is attached to an AIServiceBackend. The QuotaPolicy controller reconciles the policy, builds rate limit descriptor trees, and configures the rate limit service. This enables per-backend request throttling based on upstream provider quotas." + } + ] + }, + { + "title": "MCP Gateway", + "items": [ + { + "title": "Authorization-filtered tools/list responses", + "description": "MCP tools/list now applies the same authorization rules used by tools/call, omitting tools the caller isn't authorized to invoke. Prevents unauthorized callers from discovering tool names and avoids wasted LLM turns on tools that would fail at call time." + } + ] + }, + { + "title": "Observability", + "items": [ + { + "title": "Smarter log redaction preserves developer-authored metadata", + "description": "Debug log redaction (--enableRedaction) no longer masks developer-authored schema metadata that was previously over-redacted: tool definition description and parameters, tool call function.name, response_format.json_schema, and guided_json are now visible in debug logs. User-provided content and AI-generated text remain redacted, making debug logs significantly more useful without compromising privacy." + } + ] + } + ], + "apiChanges": [ + { + "title": "AIGatewayRoute.spec.hostnames", + "description": "New optional field accepting a list of hostnames for hostname-based request filtering. When specified, the generated HTTPRoute includes these hostnames, and the /v1/models endpoint scopes its response to models from matching routes. Follows Gateway API hostname semantics including wildcard support." + }, + { + "title": "AIGatewayRoute.spec.rules capped at 15", + "description": "Maximum rules per AIGatewayRoute reduced from 128 to 15 to match the Gateway API HTTPRoute limit (one slot is reserved for a controller-injected catch-all rule). To configure more rules on the same Gateway, split them across multiple AIGatewayRoute resources." + }, + { + "title": "VersionedAPISchema.prefix supported for Anthropic", + "description": "The prefix field now applies to Anthropic-schema backends in addition to OpenAI. The version field is ignored for Anthropic; use prefix for custom paths. Note: prefix is ignored for AWSAnthropic and GCPAnthropic as these override paths internally." + }, + { + "title": "QuotaPolicy rate limit filter injection (runtime enforcement)", + "description": "The QuotaPolicy CRD (introduced as API-only in v0.6) now has its first runtime behavior: when attached to an AIServiceBackend, a backend rate limit filter is injected to enforce quota-based throttling. Full quota-aware routing across multiple backends is planned for future releases." + } + ], + "deprecations": [], + "bugFixes": [ + { + "title": "SSE parser handles fields without space after colon", + "description": "The SSE event parser now correctly handles fields formatted as data:{json} (no space after the colon), in addition to the standard data: {json}. Fixes silent field drops when proxying responses from providers that omit the optional space." + }, + { + "title": "Responses API streaming SSE buffering", + "description": "OpenAI Responses API and speech streaming translators now buffer incomplete SSE events across response body chunks instead of treating each chunk as self-contained. Fixes dropped or mangled events when TCP segment boundaries split an SSE event mid-frame." + }, + { + "title": "Responses API token usage from incomplete and failed streams", + "description": "Token usage is now captured from response.incomplete and response.failed SSE events, not just response.completed. Streams that hit max_output_tokens or encounter post-generation failures no longer report zero tokens." + }, + { + "title": "Nil output guard in AWS Bedrock response translator", + "description": "Bedrock can return HTTP 200 with no output field (e.g. guardrail interventions or UnknownOperationException). Previously this caused a nil-pointer panic in the ext-proc; now it returns a clean error to the caller." + }, + { + "title": "Comprehensive Gemini finish-reason mapping", + "description": "Gemini finish reasons like SAFETY, BLOCKLIST, RECITATION, MALFORMED_FUNCTION_CALL, and others now map to their correct OpenAI equivalents instead of all falling through to content_filter. Unknown reasons map to error rather than silently misreporting as a content filter event." + }, + { + "title": "Empty delta in GCP Vertex AI streaming chunks", + "description": "Streaming response chunks from GCP Vertex AI that lack candidate content now emit an empty delta object instead of omitting the field, conforming to the OpenAI streaming contract and fixing parse errors in strict clients." + }, + { + "title": "Typeless assistant output messages in Responses API", + "description": "Multi-turn Responses API inputs that include assistant messages without an explicit type: \"message\" field (e.g. from OpenCode) now parse correctly. Previously these were treated as easy-input messages, causing unmarshalling failures on output_text content blocks." + } + ], + "breakingChanges": [], + "dependencies": [ + { + "title": "Go 1.26.2", + "description": "Continues on Go 1.26.2, same as v0.6." + }, + { + "title": "Envoy Gateway v1.7.0", + "description": "Built on Envoy Gateway v1.7.0 for proven data plane capabilities." + }, + { + "title": "Envoy v1.37", + "description": "Leveraging Envoy Proxy v1.37.0 for battle-tested networking." + }, + { + "title": "Gateway API v1.4.1", + "description": "Support for Gateway API v1.4.1 specifications." + }, + { + "title": "Gateway API Inference Extension v1.0.2", + "description": "Continued integration with Gateway API Inference Extension v1.0.2 for intelligent endpoint selection." + }, + { + "title": "MCP Go SDK v1.6.0", + "description": "Updated from v1.4.1 to v1.6.0 for the latest MCP protocol features and fixes." + } + ] + } + ], + "navigation": { + "previous": { "version": "v0.6.x Series", "path": "/release-notes/v0.6" } + } +} diff --git a/site/src/pages/release-notes/index.mdx b/site/src/pages/release-notes/index.mdx index 7fe3079207..27f972f29b 100644 --- a/site/src/pages/release-notes/index.mdx +++ b/site/src/pages/release-notes/index.mdx @@ -11,10 +11,12 @@ import v03Data from '../../data/releases/v0.3.json'; import v04Data from '../../data/releases/v0.4.json'; import v05Data from '../../data/releases/v0.5.json'; import v06Data from '../../data/releases/v0.6.json'; +import v07Data from '../../data/releases/v0.7.json'; import indexData from '../../data/releases/index.json'; import styles from '../../components/ReleaseNotes/ReleaseNotes.module.css'; export const allReleases = [ + ...v07Data.releases, ...v06Data.releases, ...v05Data.releases, ...v04Data.releases, @@ -39,15 +41,27 @@ export const getMarkerStyle = (type) => { Stay up-to-date with the latest improvements, features, and fixes in Envoy AI Gateway.
+ release.version).join(', ')} + /> + release.version).join(', ')} /> diff --git a/site/src/pages/release-notes/v0.7.mdx b/site/src/pages/release-notes/v0.7.mdx new file mode 100644 index 0000000000..9f6b1908fd --- /dev/null +++ b/site/src/pages/release-notes/v0.7.mdx @@ -0,0 +1,158 @@ +--- +title: Envoy AI Gateway v0.7.x Release Series +description: Hostname-based routing, Anthropic-to-Bedrock Converse translation, audio transcription/translation endpoints, Azure OpenAI Responses API, quota-aware rate limiting, Claude Opus 4.7 reasoning, and MCP authorization-aware tool listing +toc_min_heading_level: 2 +toc_max_heading_level: 4 +--- + +import Link from '@docusaurus/Link'; +import { + ReleaseSeriesLayout, + ReleaseHeader, + FeatureSectionCard, + PatchRelease, + ItemList, + Dependencies +} from '../../components/ReleaseNotes'; +import releaseData from '../../data/releases/v0.7.json'; +import React from 'react'; + +export const { series, releases, navigation } = releaseData; +export const mainRelease = releases[0]; +export const patchReleases = releases.slice(1).reverse(); + + + + + +{mainRelease.overview} + + + +## ✨ New Features + +{mainRelease.features.map((featureSection, index) => ( + +))} + +## 🔗 API Updates + +{mainRelease.apiChanges.length > 0 && ( + +)} + +{mainRelease.deprecations?.length > 0 && ( + <> + ### Deprecations + + +)} + +{mainRelease.breakingChanges?.length > 0 && ( + <> + ## ⚠️ Breaking Changes + + +)} + +## 🐛 Bug Fixes + +{mainRelease.bugFixes?.length > 0 && ( + +)} + +## 📖 Upgrade Guidance + +### Using Hostname-Based Routing + +To serve different model sets per hostname, add `hostnames` to your `AIGatewayRoute`: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: team-a-route +spec: + hostnames: + - "team-a.ai.example.com" + rules: + - matches: + - headers: + - name: x-ai-eg-model + value: gpt-4o + backendRefs: + - name: openai-backend +``` + +Routes without `hostnames` remain accessible on all hosts. When at least one route uses hostname scoping, the `/v1/models` endpoint automatically returns only the models for the matching host. + +### Rules-Per-Route Limit + +`AIGatewayRoute.spec.rules` is now capped at 15 (down from 128) to match the Gateway API HTTPRoute limit. If you have routes with more than 15 rules, split them across multiple `AIGatewayRoute` resources attached to the same Gateway. + +### Using Anthropic Prefix + +If you route Anthropic traffic to a provider with a non-standard path, use the `prefix` field: + +```yaml +schema: + name: Anthropic + prefix: /custom/v2 # produces /custom/v2/messages +``` + +Note: `prefix` is ignored for `AWSAnthropic` and `GCPAnthropic` backends as they override paths internally. + +### Adopting Claude Opus 4.7 Reasoning + +If you use Claude Opus 4.7 or Mythos Preview models, note that `display` defaults to `omitted` (unlike earlier Claude models which default to `summarized`). To receive summarized thinking content, set `display: "summarized"` explicitly. The new `xhigh` reasoning effort tier is available for long-horizon agentic tasks. + +## 📦 Dependencies Versions + + + +## ⏩ Patch Releases + +{patchReleases.map((release, index) => ( + +))} + +## 🙏 Acknowledgements + +We extend our gratitude to all contributors who made this release possible. Special thanks to: + +- The growing community of adopters for their valuable feedback and production insights +- Everyone who reported bugs, submitted PRs, and participated in design discussions +- The Envoy Gateway team for their continued collaboration + +## 🔮 What's Next + +We're already working on features for future releases: + +- **Full quota-aware routing** — building on the rate limit filter injection landed in v0.7 to route around rate-limited upstreams automatically across multiple backends +- **MCPBackend CRD** — a dedicated custom resource for MCP backend servers, decoupling MCP backend configuration from MCPRoute +- **Expanded multimodal support** — additional audio, video, and image generation backends across cloud providers +- **Deeper MCP authorization** — finer-grained policy across tools, resources, and prompts +- **More provider translation paths** — filling coverage gaps across Anthropic, Bedrock, and Vertex AI + + diff --git a/site/versioned_docs/version-0.7/_vars.json b/site/versioned_docs/version-0.7/_vars.json new file mode 100644 index 0000000000..8541486ead --- /dev/null +++ b/site/versioned_docs/version-0.7/_vars.json @@ -0,0 +1,7 @@ +{ + "aigwVersion": "0.0.0-latest", + "aigwGitRef": "main", + "egVersion": "0.0.0-latest", + "egMinVersion": "1.7.0", + "k8sMinVersion": "1.32" +} diff --git a/site/versioned_docs/version-0.7/api/api.mdx b/site/versioned_docs/version-0.7/api/api.mdx new file mode 100644 index 0000000000..378c4cdad0 --- /dev/null +++ b/site/versioned_docs/version-0.7/api/api.mdx @@ -0,0 +1,4716 @@ +--- +id: api_references +title: API Reference +toc_min_heading_level: 2 +toc_max_heading_level: 4 +--- + +## Versions +* [aigateway.envoyproxy.io/v1alpha1](#aigatewayenvoyproxyiov1alpha1) +* [aigateway.envoyproxy.io/v1beta1](#aigatewayenvoyproxyiov1beta1) + + +## aigateway.envoyproxy.io/v1alpha1 + +Package v1alpha1 contains API schema definitions for the aigateway.envoyproxy.io +API group. + + +## Resource Kinds + +### Available Kinds +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroute) +- [AIGatewayRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutelist) +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackend) +- [AIServiceBackendList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendlist) +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicy) +- [BackendSecurityPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicylist) +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfig) +- [GatewayConfigList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfiglist) +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproute) +- [MCPRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutelist) +- [QuotaPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicy) +- [QuotaPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicylist) + +### Kind Definitions +#### AIGatewayRoute + + + +**Appears in:** +- [AIGatewayRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutelist) + +AIGatewayRoute combines multiple AIServiceBackends and attaching them to Gateway(s) resources. + +This serves as a way to define a "unified" AI API for a Gateway which allows downstream +clients to use a single schema API to interact with multiple AI backends. + +Envoy AI Gateway will generate the following k8s resources corresponding to the AIGatewayRoute: + + - HTTPRoute of the Gateway API as a top-level resource to bind all backends. + The name of the HTTPRoute is the same as the AIGatewayRoute. + - HTTPRouteFilter of the Envoy Gateway API per namespace for automatic hostname rewrite. + The name of the HTTPRouteFilter is `ai-eg-host-rewrite-${AIGatewayRoute.Name}`. + +All of these resources are created in the same namespace as the AIGatewayRoute. Note that this is the implementation +detail subject to change. If you want to customize the default behavior of the Envoy AI Gateway, you can use these +resources as a reference and create your own resources. Alternatively, you can use EnvoyPatchPolicy API of the Envoy +Gateway to patch the generated resources. For example, you can configure the retry fallback behavior by attaching +BackendTrafficPolicy API of Envoy Gateway to the generated HTTPRoute. + +##### Fields + + + + + + + + +#### AIGatewayRouteList + + + + +AIGatewayRouteList contains a list of AIGatewayRoute. + +##### Fields + + + + + + + + +#### AIServiceBackend + + + +**Appears in:** +- [AIServiceBackendList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendlist) + +AIServiceBackend is a resource that represents a single backend for AIGatewayRoute. +A backend is a service that handles traffic with a concrete API specification. + +A AIServiceBackend is "attached" to a Backend which is either a k8s Service or a Backend resource of the Envoy Gateway. + +When a backend with an attached AIServiceBackend is used as a routing target in the AIGatewayRoute (more precisely, the +HTTPRouteSpec defined in the AIGatewayRoute), the ai-gateway will generate the necessary configuration to do +the backend specific logic in the final HTTPRoute. + +##### Fields + + + + + + + + +#### AIServiceBackendList + + + + +AIServiceBackendList contains a list of AIServiceBackends. + +##### Fields + + + + + + + + +#### BackendSecurityPolicy + + + +**Appears in:** +- [BackendSecurityPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicylist) + +BackendSecurityPolicy specifies configuration for authentication and authorization rules on the traffic +exiting the gateway to the backend. + +##### Fields + + + + + + + + +#### BackendSecurityPolicyList + + + + +BackendSecurityPolicyList contains a list of BackendSecurityPolicy + +##### Fields + + + + + + + + +#### GatewayConfig + + + +**Appears in:** +- [GatewayConfigList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfiglist) + +GatewayConfig provides configuration for the AI Gateway external processor +container that is deployed alongside the Gateway. + +A GatewayConfig is referenced by a Gateway via the annotation +"aigateway.envoyproxy.io/gateway-config". The GatewayConfig must be in the +same namespace as the Gateway that references it. + +This allows gateway-level configuration of the external processor, including +environment variables (e.g., for tracing configuration) and resource requirements. + +Multiple Gateways can reference the same GatewayConfig to share configuration. + +Environment Variable Precedence: +When merging environment variables, the following precedence applies (highest to lowest): + 1. GatewayConfig.Spec.ExtProc.Kubernetes.Env (this resource) + 2. Global controller flags (extProcExtraEnvVars) + +If the same environment variable name exists in both sources, the GatewayConfig +value takes precedence. + +##### Fields + + + + + + + + +#### GatewayConfigList + + + + +GatewayConfigList contains a list of GatewayConfig. + +##### Fields + + + + + + + + +#### MCPRoute + + + +**Appears in:** +- [MCPRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutelist) + +MCPRoute defines how to route MCP requests to the backend MCP servers. + +This serves as a way to define a "unified" AI API for a Gateway which allows downstream +clients to use a single schema API to interact with multiple MCP backends. + +##### Fields + + + + + + + + +#### MCPRouteList + + + + +MCPRouteList contains a list of MCPRoute. + +##### Fields + + + + + + + + +#### QuotaPolicy + + + +**Appears in:** +- [QuotaPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicylist) + +QuotaPolicy specifies token quota configuration for inference services. +Providing a list of backends in the AIGatewayRouteRule allows failover to a different service +if token quota for a service had been exceeded. + +##### Fields + + + + + + + + +#### QuotaPolicyList + + + + +QuotaPolicyList contains a list of QuotaPolicy + +##### Fields + + + + + + + + +## Supporting Types + +### Available Types +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterule) +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [AIGatewayRouteRuleMatch](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulematch) +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutespec) +- [AIGatewayRouteStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutestatus) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) +- [AIServiceBackendStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendstatus) +- [APISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-apischema) +- [AWSCredentialsFile](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-awscredentialsfile) +- [AWSOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-awsoidcexchangetoken) +- [AzureOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-azureoidcexchangetoken) +- [BackendSecurityPolicyAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyapikey) +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyawscredentials) +- [BackendSecurityPolicyAnthropicAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyanthropicapikey) +- [BackendSecurityPolicyAzureAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyazureapikey) +- [BackendSecurityPolicyAzureCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyazurecredentials) +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicygcpcredentials) +- [BackendSecurityPolicyOIDC](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyoidc) +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) +- [BackendSecurityPolicyStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicystatus) +- [BackendSecurityPolicyType](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicytype) +- [GCPCredentialsFile](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpcredentialsfile) +- [GCPOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpoidcexchangetoken) +- [GCPServiceAccountImpersonationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpserviceaccountimpersonationconfig) +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpworkloadidentityfederationconfig) +- [GCPWorkloadIdentityProvider](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpworkloadidentityprovider) +- [GatewayConfigExtProc](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigextproc) +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) +- [GatewayConfigStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigstatus) +- [HTTPBodyField](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-httpbodyfield) +- [HTTPBodyMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) +- [HTTPHeaderMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-httpheadermutation) +- [JWKS](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-jwks) +- [JWTSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-jwtsource) +- [LLMRequestCost](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-llmrequestcost) +- [LLMRequestCostType](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-llmrequestcosttype) +- [MCPAuthorizationSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationsource) +- [MCPAuthorizationTarget](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationtarget) +- [MCPBackendAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpbackendapikey) +- [MCPBackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpbackendsecuritypolicy) +- [MCPHeaderForward](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpheaderforward) +- [MCPRouteAuthorization](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteauthorization) +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteauthorizationrule) +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutebackendref) +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteoauth) +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutesecuritypolicy) +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) +- [MCPRouteStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) +- [MCPToolFilter](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) +- [PerModelQuota](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) +- [ProtectedResourceMetadata](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) +- [QuotaBucketMode](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) +- [QuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) +- [QuotaPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicyspec) +- [QuotaPolicyStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicystatus) +- [QuotaRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotarule) +- [QuotaValue](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotavalue) +- [ServiceQuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-servicequotadefinition) +- [ToolCall](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-toolcall) +- [VersionedAPISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-versionedapischema) + +### Type Definitions +#### AIGatewayRouteRule + + + +**Appears in:** +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutespec) + +AIGatewayRouteRule is a rule that defines the routing behavior of the AIGatewayRoute. + +##### Fields + + + + + + +#### AIGatewayRouteRuleBackendRef + + + +**Appears in:** +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterule) + +AIGatewayRouteRuleBackendRef is a reference to a backend with a weight. +It can reference either an AIServiceBackend or an InferencePool resource. + +##### Fields + + + + + + +#### AIGatewayRouteRuleMatch + + + +**Appears in:** +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterule) + + + +##### Fields + + + + + + +#### AIGatewayRouteSpec + + + +**Appears in:** +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroute) + +AIGatewayRouteSpec details the AIGatewayRoute configuration. + +##### Fields + + + + + + +#### AIGatewayRouteStatus + + + +**Appears in:** +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroute) + +AIGatewayRouteStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### AIServiceBackendSpec + + + +**Appears in:** +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackend) + +AIServiceBackendSpec details the AIServiceBackend configuration. + +##### Fields + + + + + + +#### AIServiceBackendStatus + + + +**Appears in:** +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackend) + +AIServiceBackendStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### APISchema + +**Underlying type:** string + +**Appears in:** +- [VersionedAPISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-versionedapischema) + +APISchema defines the API schema. + + + +##### Possible Values + + +#### AWSCredentialsFile + + + +**Appears in:** +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyawscredentials) + +AWSCredentialsFile specifies the credentials file to use for the AWS provider. +Envoy reads the secret file, and the profile to use is specified by the Profile field. + +##### Fields + + + + + + +#### AWSOIDCExchangeToken + + + +**Appears in:** +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyawscredentials) + +AWSOIDCExchangeToken specifies credentials to obtain oidc token from a sso server. +For AWS, the controller will query STS to obtain AWS AccessKeyId, SecretAccessKey, and SessionToken, +and store them in a temporary credentials file. + +##### Fields + + + + + + +#### AzureOIDCExchangeToken + + + +**Appears in:** +- [BackendSecurityPolicyAzureCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyazurecredentials) + +AzureOIDCExchangeToken specifies credentials to obtain oidc token from a sso server. +For Azure, the controller will query Azure Entra ID to get an Azure Access Token, +and store them in a secret. + +##### Fields + + + + + + +#### BackendSecurityPolicyAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyAPIKey specifies the API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAWSCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyAWSCredentials contains the supported authentication mechanisms to access AWS. + +When neither CredentialsFile nor OIDCExchangeToken is specified, the AWS SDK's default credential chain +will be used. This automatically supports various authentication methods in the following order: + 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) + 2. EKS Pod Identity - automatically rotates credentials for pods in EKS clusters + 3. IAM Roles for Service Accounts (IRSA) - injects credentials via mounted service account tokens + 4. EC2 instance metadata (IAM instance roles) + 5. ECS task roles + +The default credential chain is recommended for Kubernetes deployments as it supports automatic +credential rotation without manual configuration. Credentials are refreshed automatically when +they approach expiration (typically hourly for IRSA and Pod Identity). + +##### Fields + + + + + + +#### BackendSecurityPolicyAnthropicAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyAnthropicAPIKey specifies the Anthropic API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAzureAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyAzureAPIKey specifies the Azure OpenAI API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAzureCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyAzureCredentials contains the supported authentication mechanisms to access Azure. +Only one of ClientSecretRef or OIDCExchangeToken must be specified. Credentials will not be generated if +neither are set. + +##### Fields + + + + + + +#### BackendSecurityPolicyGCPCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyGCPCredentials contains the supported authentication mechanisms to access GCP. + +##### Fields + + + + + + +#### BackendSecurityPolicyOIDC + + + +**Appears in:** +- [AWSOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-awsoidcexchangetoken) +- [AzureOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-azureoidcexchangetoken) +- [GCPOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpoidcexchangetoken) +- [GCPWorkloadIdentityProvider](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpworkloadidentityprovider) + +BackendSecurityPolicyOIDC specifies OIDC related fields. + +##### Fields + + + + + + +#### BackendSecurityPolicySpec + + + +**Appears in:** +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicy) + +BackendSecurityPolicySpec specifies authentication rules on access the provider from the Gateway. +Only one mechanism to access a backend(s) can be specified. + +Only one type of BackendSecurityPolicy can be defined. + +##### Fields + + + + + + +#### BackendSecurityPolicyStatus + + + +**Appears in:** +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicy) + +BackendSecurityPolicyStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### BackendSecurityPolicyType + +**Underlying type:** string + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) + +BackendSecurityPolicyType specifies the type of auth mechanism used to access a backend. + + + +##### Possible Values + + +#### GCPCredentialsFile + + + +**Appears in:** +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicygcpcredentials) + +GCPCredentialsFile specifies the service account key json file to authenticate with GCP provider. + +##### Fields + + + + + + +#### GCPOIDCExchangeToken + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPServiceAccountImpersonationConfig + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPWorkloadIdentityFederationConfig + + + +**Appears in:** +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicygcpcredentials) + + + +##### Fields + + + + + + + +#### GatewayConfigExtProc + + + +**Appears in:** +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) + +GatewayConfigExtProc holds runtime-specific configuration for the external processor. + +##### Fields + + + + + + +#### GatewayConfigSpec + + + +**Appears in:** +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfig) + +GatewayConfigSpec defines the configuration for the AI Gateway. + +##### Fields + + + + + + +#### GatewayConfigStatus + + + +**Appears in:** +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfig) + +GatewayConfigStatus defines the observed state of GatewayConfig. + +##### Fields + + + + + + +#### HTTPBodyField + + + +**Appears in:** +- [HTTPBodyMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) + +HTTPBodyField represents a JSON field name and value for body mutation + +##### Fields + + + + + + +#### HTTPBodyMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will be applied to the request + +##### Fields + + + + + + +#### HTTPHeaderMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request + +##### Fields + + + + + + +#### JWKS + + + +**Appears in:** +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteoauth) + +JWKS defines how to obtain JSON Web Key Sets (JWKS) either from a remote HTTP/HTTPS endpoint or from a local source. + +##### Fields + + + + + + +#### JWTSource + + + +**Appears in:** +- [MCPAuthorizationSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationsource) + +JWTSource defines the MCP authorization source for JWT tokens. +At least one of scopes or claims must be provided. +Scopes and claims are AND-ed: when both are specified, both sets must match. + +##### Fields + + + + + + +#### LLMRequestCost + + + +**Appears in:** +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutespec) +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) + +LLMRequestCost configures each request cost. + +##### Fields + + + + + + +#### LLMRequestCostType + +**Underlying type:** string + +**Appears in:** +- [LLMRequestCost](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-llmrequestcost) + +LLMRequestCostType specifies the type of the LLMRequestCost. + + + +##### Possible Values + + +#### MCPAuthorizationSource + + + +**Appears in:** +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteauthorizationrule) + +MCPAuthorizationSource defines the source of an authorization rule. + +##### Fields + + + + + + +#### MCPAuthorizationTarget + + + +**Appears in:** +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteauthorizationrule) + +MCPAuthorizationTarget defines the target of an authorization rule. + +##### Fields + + + + + + +#### MCPBackendAPIKey + + + +**Appears in:** +- [MCPBackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpbackendsecuritypolicy) + +MCPBackendAPIKey defines the configuration for the API Key Authentication to a backend. +When both `header` and `queryParam` are unspecified, the API key will be injected into the "Authorization" header by default. + +##### Fields + + + + + + +#### MCPBackendSecurityPolicy + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutebackendref) + +MCPBackendSecurityPolicy defines the security policy for a backend MCP server. + +##### Fields + + + + + + +#### MCPHeaderForward + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutebackendref) + +MCPHeaderForward specifies a header to extract from the incoming request and forward to a backend. + +##### Fields + + + + + + +#### MCPRouteAuthorization + + + +**Appears in:** +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutesecuritypolicy) + +MCPRouteAuthorization defines the authorization configuration for a MCPRoute. + +##### Fields + + + + + + +#### MCPRouteAuthorizationRule + + + +**Appears in:** +- [MCPRouteAuthorization](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteauthorization) + +MCPRouteAuthorizationRule defines an authorization rule for MCPRoute based on the MCP authorization spec. +Reference: https://modelcontextprotocol.io/specification/draft/basic/authorization#scope-challenge-handling + +##### Fields + + + + + + +#### MCPRouteBackendRef + + + +**Appears in:** +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) + +MCPRouteBackendRef wraps a EG's BackendObjectReference to reference an MCP server. + +##### Fields + + + + + + +#### MCPRouteOAuth + + + +**Appears in:** +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutesecuritypolicy) + +MCPRouteOAuth defines a MCP spec compatible OAuth authentication configuration for a MCPRoute. +Reference: https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization + +##### Fields + + + + + + +#### MCPRouteSecurityPolicy + + + +**Appears in:** +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) + +MCPRouteSecurityPolicy defines the security policy for a MCPRoute. + +##### Fields + + + + + + +#### MCPRouteSpec + + + +**Appears in:** +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproute) + +MCPRouteSpec details the MCPRoute configuration. + +##### Fields + + + + + + +#### MCPRouteStatus + + + +**Appears in:** +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproute) + +MCPRouteStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### MCPToolFilter + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcproutebackendref) + +MCPToolFilter filters tools using include and exclude patterns with exact matches or regular expressions. +Exclude rules take precedence over include rules (deny-wins). When both include and exclude are specified, +a tool must match an include rule AND not match any exclude rule to be allowed. + +##### Fields + + + + + + +#### PerModelQuota + + + +**Appears in:** +- [QuotaPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicyspec) + + + +##### Fields + + + + + + +#### ProtectedResourceMetadata + + + +**Appears in:** +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcprouteoauth) + +ProtectedResourceMetadata represents the Protected Resource Metadata of the MCP server as per RFC 9728. + +References: +* https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-server-location +* https://datatracker.ietf.org/doc/html/rfc9728#name-protected-resource-metadata + +##### Fields + + + + + + +#### QuotaBucketMode + +**Underlying type:** string + +**Appears in:** +- [QuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) + +QuotaBucketMode specifies whether the default and per request buckets values are exclusive or inclusive. + + + + +##### Possible Values + + +#### QuotaDefinition + + + +**Appears in:** +- [PerModelQuota](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) + +QuotaDefinition specified expression for computing request cost and rules for matching requests to quota buckets. + +##### Fields + + + + + + +#### QuotaPolicySpec + + + +**Appears in:** +- [QuotaPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicy) + +QuotaPolicySpec specifies rules for computing token based costs of requests. + +##### Fields + + + + + + +#### QuotaPolicyStatus + + + +**Appears in:** +- [QuotaPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicy) + +QuotaPolicyStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### QuotaRule + + + +**Appears in:** +- [QuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) + + + +##### Fields + + + + + + +#### QuotaValue + + + +**Appears in:** +- [QuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) +- [QuotaRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotarule) +- [ServiceQuotaDefinition](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-servicequotadefinition) + +QuotaValue defines the quota limits using sliding window. + +##### Fields + + + + + + +#### ServiceQuotaDefinition + + + +**Appears in:** +- [QuotaPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-quotapolicyspec) + + + +##### Fields + + + + + + +#### ToolCall + + + +**Appears in:** +- [MCPAuthorizationTarget](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationtarget) + +ToolCall represents a tool call in the MCP authorization target. + +##### Fields + + + + + + +#### VersionedAPISchema + + + +**Appears in:** +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +VersionedAPISchema defines the API schema of either AIGatewayRoute (the input) or AIServiceBackend (the output). + +This allows the ai-gateway to understand the input and perform the necessary transformation +depending on the API schema pair (input, output). + +Note that this is vendor specific, and the stability of the API schema is not guaranteed by +the ai-gateway, but by the vendor via proper versioning. + +##### Fields + + + + + + + +## aigateway.envoyproxy.io/v1beta1 + +Package v1beta1 contains API schema definitions for the aigateway.envoyproxy.io +API group. This is the beta version of the API, preferred over v1alpha1 for new resources. + + +## Resource Kinds + +### Available Kinds +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroute) +- [AIGatewayRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutelist) +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackend) +- [AIServiceBackendList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendlist) +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicy) +- [BackendSecurityPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicylist) +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) +- [GatewayConfigList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfiglist) +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproute) +- [MCPRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutelist) + +### Kind Definitions +#### AIGatewayRoute + + + +**Appears in:** +- [AIGatewayRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutelist) + +AIGatewayRoute combines multiple AIServiceBackends and attaching them to Gateway(s) resources. + +This serves as a way to define a "unified" AI API for a Gateway which allows downstream +clients to use a single schema API to interact with multiple AI backends. + +Envoy AI Gateway will generate the following k8s resources corresponding to the AIGatewayRoute: + + - HTTPRoute of the Gateway API as a top-level resource to bind all backends. + The name of the HTTPRoute is the same as the AIGatewayRoute. + - HTTPRouteFilter of the Envoy Gateway API per namespace for automatic hostname rewrite. + The name of the HTTPRouteFilter is `ai-eg-host-rewrite-${AIGatewayRoute.Name}`. + +All of these resources are created in the same namespace as the AIGatewayRoute. Note that this is the implementation +detail subject to change. If you want to customize the default behavior of the Envoy AI Gateway, you can use these +resources as a reference and create your own resources. Alternatively, you can use EnvoyPatchPolicy API of the Envoy +Gateway to patch the generated resources. For example, you can configure the retry fallback behavior by attaching +BackendTrafficPolicy API of Envoy Gateway to the generated HTTPRoute. + +##### Fields + + + + + + + + +#### AIGatewayRouteList + + + + +AIGatewayRouteList contains a list of AIGatewayRoute. + +##### Fields + + + + + + + + +#### AIServiceBackend + + + +**Appears in:** +- [AIServiceBackendList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendlist) + +AIServiceBackend is a resource that represents a single backend for AIGatewayRoute. +A backend is a service that handles traffic with a concrete API specification. + +A AIServiceBackend is "attached" to a Backend which is either a k8s Service or a Backend resource of the Envoy Gateway. + +When a backend with an attached AIServiceBackend is used as a routing target in the AIGatewayRoute (more precisely, the +HTTPRouteSpec defined in the AIGatewayRoute), the ai-gateway will generate the necessary configuration to do +the backend specific logic in the final HTTPRoute. + +##### Fields + + + + + + + + +#### AIServiceBackendList + + + + +AIServiceBackendList contains a list of AIServiceBackends. + +##### Fields + + + + + + + + +#### BackendSecurityPolicy + + + +**Appears in:** +- [BackendSecurityPolicyList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicylist) + +BackendSecurityPolicy specifies configuration for authentication and authorization rules on the traffic +exiting the gateway to the backend. + +##### Fields + + + + + + + + +#### BackendSecurityPolicyList + + + + +BackendSecurityPolicyList contains a list of BackendSecurityPolicy + +##### Fields + + + + + + + + +#### GatewayConfig + + + +**Appears in:** +- [GatewayConfigList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfiglist) + +GatewayConfig provides configuration for the AI Gateway external processor +container that is deployed alongside the Gateway. + +A GatewayConfig is referenced by a Gateway via the annotation +"aigateway.envoyproxy.io/gateway-config". The GatewayConfig must be in the +same namespace as the Gateway that references it. + +This allows gateway-level configuration of the external processor, including +environment variables (e.g., for tracing configuration) and resource requirements. + +Multiple Gateways can reference the same GatewayConfig to share configuration. + +Environment Variable Precedence: +When merging environment variables, the following precedence applies (highest to lowest): + 1. GatewayConfig.Spec.ExtProc.Kubernetes.Env (this resource) + 2. Global controller flags (extProcExtraEnvVars) + +If the same environment variable name exists in both sources, the GatewayConfig +value takes precedence. + +##### Fields + + + + + + + + +#### GatewayConfigList + + + + +GatewayConfigList contains a list of GatewayConfig. + +##### Fields + + + + + + + + +#### MCPRoute + + + +**Appears in:** +- [MCPRouteList](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutelist) + +MCPRoute defines how to route MCP requests to the backend MCP servers. + +This serves as a way to define a "unified" AI API for a Gateway which allows downstream +clients to use a single schema API to interact with multiple MCP backends. + +##### Fields + + + + + + + + +#### MCPRouteList + + + + +MCPRouteList contains a list of MCPRoute. + +##### Fields + + + + + + + + +## Supporting Types + +### Available Types +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterule) +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulebackendref) +- [AIGatewayRouteRuleMatch](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulematch) +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutespec) +- [AIGatewayRouteStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutestatus) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) +- [AIServiceBackendStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendstatus) +- [APISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-apischema) +- [AWSCredentialsFile](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-awscredentialsfile) +- [AWSOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-awsoidcexchangetoken) +- [AzureOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-azureoidcexchangetoken) +- [BackendSecurityPolicyAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyapikey) +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyawscredentials) +- [BackendSecurityPolicyAnthropicAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyanthropicapikey) +- [BackendSecurityPolicyAzureAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyazureapikey) +- [BackendSecurityPolicyAzureCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyazurecredentials) +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicygcpcredentials) +- [BackendSecurityPolicyOIDC](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyoidc) +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) +- [BackendSecurityPolicyStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicystatus) +- [BackendSecurityPolicyType](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicytype) +- [GCPCredentialsFile](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpcredentialsfile) +- [GCPOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpoidcexchangetoken) +- [GCPServiceAccountImpersonationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpserviceaccountimpersonationconfig) +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) +- [GCPWorkloadIdentityProvider](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityprovider) +- [GatewayConfigExtProc](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigextproc) +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) +- [GatewayConfigStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigstatus) +- [HTTPBodyField](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-httpbodyfield) +- [HTTPBodyMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation) +- [HTTPHeaderMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-httpheadermutation) +- [JWKS](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-jwks) +- [JWTSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-jwtsource) +- [LLMRequestCost](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-llmrequestcost) +- [LLMRequestCostType](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-llmrequestcosttype) +- [MCPAuthorizationSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpauthorizationsource) +- [MCPAuthorizationTarget](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpauthorizationtarget) +- [MCPBackendAPIKey](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpbackendapikey) +- [MCPBackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpbackendsecuritypolicy) +- [MCPHeaderForward](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpheaderforward) +- [MCPRouteAuthorization](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteauthorization) +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteauthorizationrule) +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutebackendref) +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteoauth) +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutesecuritypolicy) +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) +- [MCPRouteStatus](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) +- [MCPToolFilter](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [ProtectedResourceMetadata](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) +- [ToolCall](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-toolcall) +- [VersionedAPISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) + +### Type Definitions +#### AIGatewayRouteRule + + + +**Appears in:** +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutespec) + +AIGatewayRouteRule is a rule that defines the routing behavior of the AIGatewayRoute. + +##### Fields + + + + + + +#### AIGatewayRouteRuleBackendRef + + + +**Appears in:** +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterule) + +AIGatewayRouteRuleBackendRef is a reference to a backend with a weight. +It can reference either an AIServiceBackend or an InferencePool resource. + +##### Fields + + + + + + +#### AIGatewayRouteRuleMatch + + + +**Appears in:** +- [AIGatewayRouteRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterule) + + + +##### Fields + + + + + + +#### AIGatewayRouteSpec + + + +**Appears in:** +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroute) + +AIGatewayRouteSpec details the AIGatewayRoute configuration. + +##### Fields + + + + + + +#### AIGatewayRouteStatus + + + +**Appears in:** +- [AIGatewayRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroute) + +AIGatewayRouteStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### AIServiceBackendSpec + + + +**Appears in:** +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackend) + +AIServiceBackendSpec details the AIServiceBackend configuration. + +##### Fields + + + + + + +#### AIServiceBackendStatus + + + +**Appears in:** +- [AIServiceBackend](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackend) + +AIServiceBackendStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### APISchema + +**Underlying type:** string + +**Appears in:** +- [VersionedAPISchema](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) + +APISchema defines the API schema. + + + +##### Possible Values + + +#### AWSCredentialsFile + + + +**Appears in:** +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyawscredentials) + +AWSCredentialsFile specifies the credentials file to use for the AWS provider. +Envoy reads the secret file, and the profile to use is specified by the Profile field. + +##### Fields + + + + + + +#### AWSOIDCExchangeToken + + + +**Appears in:** +- [BackendSecurityPolicyAWSCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyawscredentials) + +AWSOIDCExchangeToken specifies credentials to obtain oidc token from a sso server. +For AWS, the controller will query STS to obtain AWS AccessKeyId, SecretAccessKey, and SessionToken, +and store them in a temporary credentials file. + +##### Fields + + + + + + +#### AzureOIDCExchangeToken + + + +**Appears in:** +- [BackendSecurityPolicyAzureCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyazurecredentials) + +AzureOIDCExchangeToken specifies credentials to obtain oidc token from a sso server. +For Azure, the controller will query Azure Entra ID to get an Azure Access Token, +and store them in a secret. + +##### Fields + + + + + + +#### BackendSecurityPolicyAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyAPIKey specifies the API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAWSCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyAWSCredentials contains the supported authentication mechanisms to access AWS. + +When neither CredentialsFile nor OIDCExchangeToken is specified, the AWS SDK's default credential chain +will be used. This automatically supports various authentication methods in the following order: + 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) + 2. EKS Pod Identity - automatically rotates credentials for pods in EKS clusters + 3. IAM Roles for Service Accounts (IRSA) - injects credentials via mounted service account tokens + 4. EC2 instance metadata (IAM instance roles) + 5. ECS task roles + +The default credential chain is recommended for Kubernetes deployments as it supports automatic +credential rotation without manual configuration. Credentials are refreshed automatically when +they approach expiration (typically hourly for IRSA and Pod Identity). + +##### Fields + + + + + + +#### BackendSecurityPolicyAnthropicAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyAnthropicAPIKey specifies the Anthropic API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAzureAPIKey + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyAzureAPIKey specifies the Azure OpenAI API key. + +##### Fields + + + + + + +#### BackendSecurityPolicyAzureCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyAzureCredentials contains the supported authentication mechanisms to access Azure. +Only one of ClientSecretRef or OIDCExchangeToken must be specified. Credentials will not be generated if +neither are set. + +##### Fields + + + + + + +#### BackendSecurityPolicyGCPCredentials + + + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyGCPCredentials contains the supported authentication mechanisms to access GCP. + +When neither CredentialsFile nor WorkloadIdentityFederationConfig is specified, Application Default +Credentials (ADC) will be used. This supports GKE Workload Identity, environment variables, and +default service account credentials when running on GCP. + +##### Fields + + + + + + +#### BackendSecurityPolicyOIDC + + + +**Appears in:** +- [AWSOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-awsoidcexchangetoken) +- [AzureOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-azureoidcexchangetoken) +- [GCPOIDCExchangeToken](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpoidcexchangetoken) +- [GCPWorkloadIdentityProvider](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityprovider) + +BackendSecurityPolicyOIDC specifies OIDC related fields. + +##### Fields + + + + + + +#### BackendSecurityPolicySpec + + + +**Appears in:** +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicy) + +BackendSecurityPolicySpec specifies authentication rules on access the provider from the Gateway. +Only one mechanism to access a backend(s) can be specified. + +Only one type of BackendSecurityPolicy can be defined. + +##### Fields + + + + + + +#### BackendSecurityPolicyStatus + + + +**Appears in:** +- [BackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicy) + +BackendSecurityPolicyStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### BackendSecurityPolicyType + +**Underlying type:** string + +**Appears in:** +- [BackendSecurityPolicySpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) + +BackendSecurityPolicyType specifies the type of auth mechanism used to access a backend. + + + +##### Possible Values + + +#### GCPCredentialsFile + + + +**Appears in:** +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicygcpcredentials) + +GCPCredentialsFile specifies the service account key json file to authenticate with GCP provider. + +##### Fields + + + + + + +#### GCPOIDCExchangeToken + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPServiceAccountImpersonationConfig + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPWorkloadIdentityFederationConfig + + + +**Appears in:** +- [BackendSecurityPolicyGCPCredentials](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicygcpcredentials) + + + +##### Fields + + + + + + + +#### GatewayConfigExtProc + + + +**Appears in:** +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) + +GatewayConfigExtProc holds runtime-specific configuration for the external processor. + +##### Fields + + + + + + +#### GatewayConfigSpec + + + +**Appears in:** +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) + +GatewayConfigSpec defines the configuration for the AI Gateway. + +##### Fields + + + + + + +#### GatewayConfigStatus + + + +**Appears in:** +- [GatewayConfig](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) + +GatewayConfigStatus defines the observed state of GatewayConfig. + +##### Fields + + + + + + +#### HTTPBodyField + + + +**Appears in:** +- [HTTPBodyMutation](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation) + +HTTPBodyField represents a JSON field name and value for body mutation + +##### Fields + + + + + + +#### HTTPBodyMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) + +HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will be applied to the request + +##### Fields + + + + + + +#### HTTPHeaderMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) + +HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request + +##### Fields + + + + + + +#### JWKS + + + +**Appears in:** +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteoauth) + +JWKS defines how to obtain JSON Web Key Sets (JWKS) either from a remote HTTP/HTTPS endpoint or from a local source. + +##### Fields + + + + + + +#### JWTSource + + + +**Appears in:** +- [MCPAuthorizationSource](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpauthorizationsource) + +JWTSource defines the MCP authorization source for JWT tokens. +At least one of scopes or claims must be provided. +Scopes and claims are AND-ed: when both are specified, both sets must match. + +##### Fields + + + + + + +#### LLMRequestCost + + + +**Appears in:** +- [AIGatewayRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aigatewayroutespec) +- [GatewayConfigSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) + +LLMRequestCost configures each request cost. + +##### Fields + + + + + + +#### LLMRequestCostType + +**Underlying type:** string + +**Appears in:** +- [LLMRequestCost](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-llmrequestcost) + +LLMRequestCostType specifies the type of the LLMRequestCost. + + + +##### Possible Values + + +#### MCPAuthorizationSource + + + +**Appears in:** +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteauthorizationrule) + +MCPAuthorizationSource defines the source of an authorization rule. + +##### Fields + + + + + + +#### MCPAuthorizationTarget + + + +**Appears in:** +- [MCPRouteAuthorizationRule](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteauthorizationrule) + +MCPAuthorizationTarget defines the target of an authorization rule. + +##### Fields + + + + + + +#### MCPBackendAPIKey + + + +**Appears in:** +- [MCPBackendSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpbackendsecuritypolicy) + +MCPBackendAPIKey defines the configuration for the API Key Authentication to a backend. +When both `header` and `queryParam` are unspecified, the API key will be injected into the "Authorization" header by default. + +##### Fields + + + + + + +#### MCPBackendSecurityPolicy + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutebackendref) + +MCPBackendSecurityPolicy defines the security policy for a backend MCP server. + +##### Fields + + + + + + +#### MCPHeaderForward + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutebackendref) + +MCPHeaderForward specifies a header to extract from the incoming request and forward to a backend. + +##### Fields + + + + + + +#### MCPRouteAuthorization + + + +**Appears in:** +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutesecuritypolicy) + +MCPRouteAuthorization defines the authorization configuration for a MCPRoute. + +##### Fields + + + + + + +#### MCPRouteAuthorizationRule + + + +**Appears in:** +- [MCPRouteAuthorization](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteauthorization) + +MCPRouteAuthorizationRule defines an authorization rule for MCPRoute based on the MCP authorization spec. +Reference: https://modelcontextprotocol.io/specification/draft/basic/authorization#scope-challenge-handling + +##### Fields + + + + + + +#### MCPRouteBackendRef + + + +**Appears in:** +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) + +MCPRouteBackendRef wraps a EG's BackendObjectReference to reference an MCP server. + +##### Fields + + + + + + +#### MCPRouteOAuth + + + +**Appears in:** +- [MCPRouteSecurityPolicy](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutesecuritypolicy) + +MCPRouteOAuth defines a MCP spec compatible OAuth authentication configuration for a MCPRoute. +Reference: https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization + +##### Fields + + + + + + +#### MCPRouteSecurityPolicy + + + +**Appears in:** +- [MCPRouteSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) + +MCPRouteSecurityPolicy defines the security policy for a MCPRoute. + +##### Fields + + + + + + +#### MCPRouteSpec + + + +**Appears in:** +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproute) + +MCPRouteSpec details the MCPRoute configuration. + +##### Fields + + + + + + +#### MCPRouteStatus + + + +**Appears in:** +- [MCPRoute](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproute) + +MCPRouteStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### MCPToolFilter + + + +**Appears in:** +- [MCPRouteBackendRef](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcproutebackendref) + +MCPToolFilter filters tools using include and exclude patterns with exact matches or regular expressions. +Exclude rules take precedence over include rules (deny-wins). When both include and exclude are specified, +a tool must match an include rule AND not match any exclude rule to be allowed. + +##### Fields + + + + + + +#### ProtectedResourceMetadata + + + +**Appears in:** +- [MCPRouteOAuth](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcprouteoauth) + +ProtectedResourceMetadata represents the Protected Resource Metadata of the MCP server as per RFC 9728. + +References: +* https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-server-location +* https://datatracker.ietf.org/doc/html/rfc9728#name-protected-resource-metadata + +##### Fields + + + + + + +#### ToolCall + + + +**Appears in:** +- [MCPAuthorizationTarget](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-mcpauthorizationtarget) + +ToolCall represents a tool call in the MCP authorization target. + +##### Fields + + + + + + +#### VersionedAPISchema + + + +**Appears in:** +- [AIServiceBackendSpec](#github.1485827954.workers.dev-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) + +VersionedAPISchema defines the API schema of either AIGatewayRoute (the input) or AIServiceBackend (the output). + +This allows the ai-gateway to understand the input and perform the necessary transformation +depending on the API schema pair (input, output). + +Note that this is vendor specific, and the stability of the API schema is not guaranteed by +the ai-gateway, but by the vendor via proper versioning. + +##### Fields + + + + + + diff --git a/site/versioned_docs/version-0.7/capabilities/gateway-config.md b/site/versioned_docs/version-0.7/capabilities/gateway-config.md new file mode 100644 index 0000000000..f1228001ff --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/gateway-config.md @@ -0,0 +1,247 @@ +--- +id: gateway-config +title: Gateway Configuration +sidebar_position: 2 +--- + +# Gateway Configuration + +The `GatewayConfig` CRD provides gateway-scoped configuration for the AI Gateway external processor container. This allows you to configure environment variables and resource requirements at the Gateway level, rather than at the route level. + +## Overview + +Use `GatewayConfig` when you need to: + +- Configure per-gateway OpenTelemetry tracing settings +- Set resource requirements (CPU/memory) for the external processor +- Share configuration across multiple Gateways +- Configure environment variables for different gateway instances without affecting others + +## Usage + +### Creating a GatewayConfig + +Create a `GatewayConfig` resource with your desired configuration: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GatewayConfig +metadata: + name: my-gateway-config + namespace: default +spec: + extProc: + kubernetes: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://otel-collector:4317" + - name: OTEL_SERVICE_NAME + value: "my-ai-gateway" + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" +``` + +### Referencing from a Gateway + +Reference the `GatewayConfig` from your Gateway using an annotation: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + namespace: default + annotations: + aigateway.envoyproxy.io/gateway-config: my-gateway-config +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + protocol: HTTP + port: 8080 +``` + +:::note +The `GatewayConfig` must be in the same namespace as the Gateway that references it. +::: + +## Configuration Options + +### Environment Variables + +The `spec.extProc.kubernetes.env` field accepts a list of Kubernetes `EnvVar` objects: + +```yaml +spec: + extProc: + kubernetes: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://otel-collector:4317" + - name: OTEL_EXPORTER_OTLP_HEADERS + value: "api-key=your-secret" + - name: LOG_LEVEL + value: "debug" +``` + +### Resource Requirements + +The `spec.extProc.kubernetes.resources` field configures compute resources for the external processor container: + +```yaml +spec: + extProc: + kubernetes: + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" +``` + +If not specified, Kubernetes default resource allocations are used. + +## Environment Variable Precedence + +Environment variables can be configured at multiple levels. The precedence order is (highest to lowest): + +1. **GatewayConfig.spec.extProc.kubernetes.env** - Highest priority +2. **Global controller flags** (`--extproc-extra-env-vars`) - Lower priority + +When the same environment variable is defined at multiple levels, the higher precedence value is used. + +### Example + +If the controller is started with: + +``` +--extproc-extra-env-vars="LOG_LEVEL=info;GLOBAL_VAR=global" +``` + +And a GatewayConfig defines: + +```yaml +env: + - name: LOG_LEVEL + value: "debug" + - name: CONFIG_VAR + value: "config" +``` + +The resulting environment variables will be: + +- `LOG_LEVEL=debug` (GatewayConfig overrides global) +- `GLOBAL_VAR=global` (from global) +- `CONFIG_VAR=config` (from GatewayConfig) + +## Shared Configuration + +Multiple Gateways can reference the same `GatewayConfig`: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GatewayConfig +metadata: + name: shared-config +spec: + extProc: + kubernetes: + env: + - name: OTEL_SERVICE_NAME + value: "ai-gateway-cluster" +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway-1 + annotations: + aigateway.envoyproxy.io/gateway-config: shared-config +spec: + # ... +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway-2 + annotations: + aigateway.envoyproxy.io/gateway-config: shared-config +spec: + # ... +``` + +## Migration from Route-Level Configuration + +The route-level resource configuration (`AIGatewayRoute.spec.filterConfig.externalProcessor.resources`) is deprecated. Migrate to `GatewayConfig`: + +### Before (deprecated) + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: my-route +spec: + filterConfig: + type: ExternalProcessor + externalProcessor: + resources: + requests: + cpu: "100m" + memory: "128Mi" +``` + +### After (recommended) + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GatewayConfig +metadata: + name: my-config +spec: + extProc: + kubernetes: + resources: + requests: + cpu: "100m" + memory: "128Mi" +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + annotations: + aigateway.envoyproxy.io/gateway-config: my-config +spec: + # ... +``` + +## Status + +The `GatewayConfig` status reports the validity of the configuration: + +```yaml +status: + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: "GatewayConfig reconciled successfully" +``` + +Possible condition types: + +- `Accepted`: The configuration is valid and applied +- `NotAccepted`: The configuration has validation errors + +## See Also + +- [Tracing](./observability/tracing.md) - Configure distributed tracing for AI Gateway +- [Metrics](./observability/metrics.md) - Configure metrics collection +- [Examples](https://github.com/envoyproxy/ai-gateway/tree/main/examples/gateway-config) - Example YAML files diff --git a/site/versioned_docs/version-0.7/capabilities/index.md b/site/versioned_docs/version-0.7/capabilities/index.md new file mode 100644 index 0000000000..bd5f9fd52d --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/index.md @@ -0,0 +1,59 @@ +--- +id: capabilities +title: Capabilities +sidebar_position: 3 +--- + +# Envoy AI Gateway Capabilities + +Welcome to the Envoy AI Gateway capabilities documentation! This section provides detailed information about the various features and capabilities that Envoy AI Gateway offers to help you manage and optimize your AI/LLM traffic. + +## LLM Providers Integrations + +Support for various Large Language Model providers: + +- **[Connecting to AI Providers](./llm-integrations/connect-providers.md)**: Learn how to establish connectivity with any supported AI provider +- **[Supported Providers](./llm-integrations/supported-providers.md)**: Compatible AI/LLM service providers +- **[Supported Endpoints](./llm-integrations/supported-endpoints.md)**: Available API endpoints and operations +- **[Vendor-Specific Fields](./llm-integrations/vendor-specific-fields.md)**: Use backend-specific parameters and access provider-unique capabilities in your OpenAI-compatible requests +- **[Prompt Caching](./llm-integrations/prompt-caching.md)**: Provider-agnostic prompt caching using unified cache_control API + +## Inference Optimization + +Advanced inference optimization capabilities for AI/LLM workloads: + +- **[InferencePool Support](./inference/inferencepool-support.md)**: Intelligent routing and load balancing for inference endpoints +- **[HTTPRoute + InferencePool](./inference/httproute-inferencepool.md)**: Basic inference routing with standard Gateway API +- **[AIGatewayRoute + InferencePool](./inference/aigatewayroute-inferencepool.md)**: Advanced AI-specific routing with enhanced features + +## Gateway Configuration + +- **[GatewayConfig](./gateway-config.md)**: Gateway-scoped configuration for the external processor (env vars, resources, shared settings) +- **[Scaling](./scaling.md)**: Configure multiple controller replicas and horizontal pod autoscaling for production deployments + +## Traffic Management + +Comprehensive traffic handling and routing capabilities: + +- **[Model Virtualization](./traffic/model-virtualization.md)**: Abstract and virtualize AI models +- **[Provider Fallback](./traffic/provider-fallback.md)**: Automatic failover between AI providers +- **[Usage-based Rate Limiting](./traffic/usage-based-ratelimiting.md)**: Token-aware rate limiting for AI workloads +- **[Header and Body Mutations](./traffic/header-body-mutations.md)**: Customize HTTP headers and JSON body fields per backend or route + +## Security + +Robust security features for AI gateway deployments: + +- **[Upstream Authentication](./security/upstream-auth.mdx)**: Secure authentication to upstream AI services + +## Model Context Protocol (MCP) + +Connect AI agents to external tools and data sources: + +- **[MCP Gateway](./mcp/)**: Server multiplexing, tool routing, OAuth authentication, and observability for MCP workloads + +## Observability + +Monitoring and observability tools for AI workloads: + +- **[Metrics](./observability/metrics.md)**: Comprehensive metrics collection and monitoring diff --git a/site/versioned_docs/version-0.7/capabilities/inference/aigatewayroute-inferencepool.md b/site/versioned_docs/version-0.7/capabilities/inference/aigatewayroute-inferencepool.md new file mode 100644 index 0000000000..32d6d35aec --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/inference/aigatewayroute-inferencepool.md @@ -0,0 +1,633 @@ +--- +id: aigatewayroute-inferencepool +title: AIGatewayRoute + InferencePool Guide +sidebar_position: 3 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# AIGatewayRoute + InferencePool Guide + +This guide demonstrates how to use InferencePool with AIGatewayRoute for advanced AI-specific inference routing. This approach provides enhanced features like model-based routing, token rate limiting, and advanced observability. + +## Prerequisites + +Before starting, ensure you have: + +1. **Kubernetes cluster** with Gateway API support +2. **Envoy AI Gateway** installed and configured + +## Step 1: Install Gateway API Inference Extension + +Install the Gateway API Inference Extension CRDs and controller: + +```bash +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.1/manifests.yaml +``` + +After installing InferencePool CRD, enable InferencePool support in Envoy Gateway, restart the deployment, and wait for it to be ready: + + +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/inference-pool/config.yaml + +kubectl rollout restart -n envoy-gateway-system deployment/envoy-gateway + +kubectl wait --timeout=2m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available`} + + +## Step 2: Ensure Envoy Gateway is configured for InferencePool + +See [Envoy Gateway Installation Guide](../../getting-started/prerequisites.md#additional-features-rate-limiting-inferencepool-etc) + +## Step 3: Deploy Inference Backends + +Deploy sample inference backends and related resources: + +```bash +# Deploy vLLM simulation backend +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/v1.0.1/config/manifests/vllm/sim-deployment.yaml + +# Deploy InferenceObjective +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/refs/tags/v1.0.1/config/manifests/inferenceobjective.yaml + +# Deploy InferencePool resources +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/v1.0.1/config/manifests/inferencepool-resources.yaml +``` + +> **Note**: These deployments create the `vllm-llama3-8b-instruct` InferencePool and related resources that are referenced in the AIGatewayRoute configuration below. + +## Step 4: Create Custom InferencePool Resources + +Create additional inference backends with custom EndpointPicker configuration: + +```yaml +cat < +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/inference-pool/config.yaml + +kubectl rollout restart -n envoy-gateway-system deployment/envoy-gateway + +kubectl wait --timeout=2m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available`} + + +## Step 2: Ensure Envoy Gateway is configured for InferencePool + +See [Envoy Gateway Installation Guide](../../getting-started/prerequisites.md#additional-features-rate-limiting-inferencepool-etc) + +## Step 3: Deploy Inference Backend + +Deploy a sample inference backend that will serve as your inference endpoints: + +```bash +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/v1.0.1/config/manifests/vllm/sim-deployment.yaml +``` + +This creates a simulated vLLM deployment with multiple replicas that can handle inference requests. + +> **Note**: This deployment creates the `vllm-llama3-8b-instruct` InferencePool and related resources that are referenced in the HTTPRoute configuration below. + +## Step 4: Create InferenceObjective + +Create an InferenceObjective resource to define the model configuration: + +```bash +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/refs/tags/v1.0.1/config/manifests/inferenceobjective.yaml +``` + +## Step 5: Create InferencePool Resources + +Deploy the InferencePool and related resources: + +```bash +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/v1.0.1/config/manifests/inferencepool-resources.yaml +``` + +This creates: + +- InferencePool resource defining the endpoint selection criteria +- Endpoint Picker Provider (EPP) deployment for intelligent routing with advanced scheduling plugins +- Associated services and configurations +- RBAC permissions for accessing InferencePool and Pod resources + +## Step 6: Configure Gateway and HTTPRoute + +Create a Gateway and HTTPRoute that uses the InferencePool: + +```yaml +cat <>Envoy: Request with model header + Envoy->>RLS: Check Rate Limit + RLS-->>Envoy: Rate limit OK + Envoy->>Processor: Router-level ExtProc Request + Note over Processor: Extract Model Name + Processor-->>Envoy: ClearRouteCache + Envoy->>EPP: Router-level ExtProc Request + Note over EPP: Pick Optimal Endpoint + EPP-->>Envoy: Add x-gateway-destination-endpoint header + loop Retry/Fallback loop + Note over Envoy: Forward to Selected Endpoint + Envoy->>Processor: Upstream-level ExtProc Request + Note over Processor: Request Transform + Processor-->>Envoy: Transformed Request + Envoy->>Backend: Forward Request + Backend-->>Envoy: Response + end + Envoy->>Processor: Process Response + Note over Processor: Extract Token Usage & Metrics + Processor-->>Envoy: Add Usage Metadata + Envoy->>RLS: Update Rate Limit Budget + RLS-->>Envoy: Budget Updated + Envoy->>Client: Response +``` + +### HTTPRoute + InferencePool Flow + +```mermaid +sequenceDiagram + participant Client as Client + participant Envoy as Envoy Proxy + participant EPP as Endpoint Picker Provider + participant Backend as Inference Backend + + Client->>Envoy: Request + Envoy->>EPP: Router-level ExtProc Request + Note over EPP: Pick Optimal Endpoint + EPP-->>Envoy: Add x-gateway-destination-endpoint header + loop Retry/Fallback loop + Note over Envoy: Forward to Selected Endpoint + Envoy->>Backend: Forward Request + Backend-->>Envoy: Response + end + Envoy->>Client: Response +``` + +## Integration Approaches + +Envoy AI Gateway supports two ways to use InferencePool: + +### 1. HTTPRoute + InferencePool + +Direct integration with standard Gateway API HTTPRoute for simple inference routing scenarios. + +**Use Cases:** + +- Simple inference workloads without complex AI-specific requirements +- Direct OpenAI-compatible API forwarding + +### 2. AIGatewayRoute + InferencePool + +Enhanced integration with AI Gateway's custom route type for advanced AI-specific features. + +**Use Cases:** + +- Multi-model routing based on request content +- Token-based rate limiting +- Advanced observability and metrics +- Request/response transformation + +## AIGatewayRoute Advantages + +AIGatewayRoute provides several advantages over standard HTTPRoute when used with InferencePool: + +### Advanced OpenAI Routing + +- Built-in OpenAI API schema validation +- Seamless integration with OpenAI SDKs +- Route multiple models in a single listener +- Mix InferencePool and traditional backends +- Automatic model extraction from request body + +### AI-Specific Features + +- Token-based rate limiting +- Model performance metrics +- Cost tracking and management +- Request/response transformation + +## Getting Started + +To use InferencePool with Envoy AI Gateway, you'll need to: + +1. **Install Prerequisites**: + - Deploy Gateway API Inference Extension CRD. + - Configure Envoy Gateway with the InferencePool support. See [Envoy Gateway Installation Guide](../../getting-started/prerequisites.md#additional-features-rate-limiting-inferencepool-etc) for details. +2. **Configure InferencePool**: Define your inference endpoints and selection criteria +3. **Set up Routes**: Configure either HTTPRoute or AIGatewayRoute to use the InferencePool +4. **Deploy Endpoint Picker**: Install and configure the endpoint picker provider + +The following sections provide detailed guides for both integration approaches with complete examples and step-by-step instructions. diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/connect-providers.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/connect-providers.md new file mode 100644 index 0000000000..8a95257b88 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/connect-providers.md @@ -0,0 +1,554 @@ +--- +id: connect-providers +title: Connecting to AI Providers +sidebar_position: 10 +--- + +# Connecting to AI Providers + +Envoy AI Gateway provides a unified interface for connecting to multiple AI providers through a standardized configuration approach. This page explains the fundamental concepts, resources, and relationships required to establish connectivity with any supported AI provider. + +## Overview + +Establishing connectivity with an AI provider involves configuring three key Kubernetes resources that work together to enable secure, scalable access to AI services: + +1. **AIServiceBackend** - Defines the backend service and its API schema +2. **BackendSecurityPolicy** - Configures authentication credentials +3. **AIGatewayRoute** - Routes client requests to the appropriate backends + +These resources provide a consistent configuration model regardless of which AI provider you're connecting to, whether it's OpenAI, AWS Bedrock, Azure OpenAI, or any other [supported provider](./supported-providers). + +## Core Resources for Provider Connectivity + +### AIServiceBackend + +The `AIServiceBackend` resource represents an individual AI service backend and serves as the bridge between your gateway and the AI provider's API. + +#### Purpose and Configuration + +- **API Schema Definition**: Specifies which API format the backend expects (OpenAI v1, AWS Bedrock, Azure OpenAI, etc.) +- **Backend Reference**: Points to the Envoy Gateway Backend resource +- **Security Integration**: Links to authentication policies for upstream services + +#### Key Fields + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIServiceBackend +metadata: + name: my-provider-backend +spec: + # API schema the backend expects + schema: + name: OpenAI # Provider API format + version: "v1" # API version (optional for OpenAI, defaults to "v1") + + # Reference to the Envoy Gateway Backend resource + backendRef: + name: my-provider-backend + kind: Backend + group: gateway.envoyproxy.io +``` + +#### Schema Configuration Examples + +Different providers require different schema configurations: + +| Provider | Schema Configuration | +| -------------------------- | --------------------------------------------------------- | +| OpenAI | `{"name":"OpenAI","version":"v1"}` | +| AWS Bedrock | `{"name":"AWSBedrock"}` | +| Azure OpenAI | `{"name":"AzureOpenAI","version":"2025-01-01-preview"}` | +| GCP Vertex AI | `{"name":"GCPVertexAI"}` | +| GCP Anthropic on Vertex AI | `{"name":"GCPAnthropic", "version": "vertex-2023-10-16"}` | + +:::tip +Many providers offer OpenAI-compatible APIs, which allows them to use the OpenAI schema configuration with provider-specific version paths. +::: + +### BackendSecurityPolicy + +The `BackendSecurityPolicy` resource configures authentication credentials needed to access upstream AI services securely. + +#### Purpose and Configuration + +- **Credential Management**: Stores API keys, cloud credentials, or other authentication mechanisms +- **Security Isolation**: Keeps sensitive credentials separate from routing configuration +- **Provider Flexibility**: Supports multiple authentication types for different providers + +#### Authentication Types + +##### API Key Authentication + +Commonly used across many providers + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: openai-auth +spec: + type: APIKey + apiKey: + secretRef: + name: openai-secret + namespace: default +``` + +:::note +The secret must contain the API key with the key name `"apiKey"`. +::: + +##### AWS Credentials + +Used when connecting to AWS Bedrock. Supports three authentication methods: + +**Option 1: EKS Pod Identity or IRSA (Recommended for production)** + +When running on EKS, the AWS SDK automatically uses the default credential chain, which includes EKS Pod Identity and IRSA. Simply configure the region: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: bedrock-auth +spec: + type: AWSCredentials + awsCredentials: + region: us-east-1 + # No credentialsFile needed - automatically uses: + # - EKS Pod Identity (if Pod Identity association exists) + # - IRSA (if ServiceAccount has eks.amazonaws.com/role-arn annotation) +``` + +See the [Connect AWS Bedrock guide](../../getting-started/connect-providers/aws-bedrock.md) for detailed setup instructions. + +**Option 2: Static Credentials (Development/Testing)** + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: bedrock-auth +spec: + type: AWSCredentials + awsCredentials: + region: us-east-1 + credentialsFile: + secretRef: + name: aws-secret + namespace: default + profile: default # Optional, defaults to "default" +``` + +:::note +When using static credentials, the secret must contain the AWS credentials file with the key name `"credentials"`. +::: + +##### Azure Credentials + +Used for connecting to Azure OpenAI + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: azure-auth +spec: + type: AzureCredentials + azureCredentials: + clientID: "your-azure-client-id" + tenantID: "your-azure-tenant-id" + clientSecretRef: + name: azure-secret + namespace: default +``` + +:::note +The secret must contain the Azure client secret with the key name `"client-secret"`. +::: + +##### GCP Credentials + +Used for connecting to GCP Vertex AI and Anthropic on GCP. Supports three authentication methods: + +1. Application Default Credentials (Recommended for GKE): + When running on GKE with Workload Identity configured, you can use Application Default Credentials (ADC) without managing service account keys. Simply specify the project and region: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: gcp-auth-adc + namespace: default +spec: + type: GCPCredentials + gcpCredentials: + projectName: "your-gcp-project" + region: "us-central1" +``` + +ADC automatically supports GKE Workload Identity, the `GOOGLE_APPLICATION_CREDENTIALS` environment variable, and default service account credentials when running on GCP. + +2. Service Account Key Files: + A service account key file is a JSON file containing a private key that authenticates as a service account. + You create a service account in GCP, generate a key file, download it, and then store it in the k8s secret referenced by BackendSecurityPolicy. + Envoy AI Gateway uses this key file to generate an access token and authenticate with GCP Vertex AI. + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: gcp-auth-service-account-key + namespace: default +spec: + type: GCPCredentials + gcpCredentials: + projectName: GCP_PROJECT_NAME # Replace with your GCP project name + region: GCP_REGION # Replace with your GCP region + credentialsFile: + secretRef: + name: envoy-ai-gateway-basic-gcp-service-account-key-file +``` + +3. Workload Identity Federation: + Workload Identity Federation is a modern, keyless authentication method that allows workloads running outside of GCP to impersonate a service account using their own native identity. + It leverages a trust relationship between GCP and an external identity provider such as OIDC. + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: gcp-auth +spec: + type: GCPCredentials + gcpCredentials: + projectName: "your-gcp-project" + region: "us-central1" + workloadIdentityFederationConfig: + projectID: "your-gcp-project-id" + workloadIdentityPoolName: "your-workload-identity-pool" + workloadIdentityProviderName: "your-identity-provider" + serviceAccountImpersonation: + serviceAccountName: "your-service-account" + oidcExchangeToken: + oidc: + provider: + issuer: "https://your-oidc-provider.com" + clientID: "your-oidc-client-id" + clientSecret: + name: "gcp-client-secret" + namespace: default +``` + +#### Security Best Practices + +- **Store credentials in Kubernetes Secrets**: Never expose sensitive data in plain text +- **Use principle of least privilege**: Grant only necessary permissions +- **Rotate credentials regularly**: Implement credential rotation policies +- **Separate environments**: Use different credentials for development, staging, and production + +### AIGatewayRoute + +The `AIGatewayRoute` resource defines how client requests are routed to appropriate AI backends and manages the unified API interface. + +#### Purpose and Configuration + +- **Request Routing**: Directs traffic to specific backends based on model names or other criteria +- **API Unification**: Provides a consistent interface regardless of backend provider +- **Request Transformation**: Automatically converts between different API schemas +- **Load Balancing**: Distributes traffic across multiple backends + +#### Basic Configuration + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: multi-provider-route +spec: + # Gateway to attach to + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + + # Routing rules + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4o-mini + backendRefs: + - name: openai-backend + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: claude-3-sonnet + backendRefs: + - name: bedrock-backend +``` + +## Resource Relationships and Data Flow + +Understanding how these resources work together is crucial for successful provider connectivity: + +```mermaid +graph TD + A[Client Request] --> B[AIGatewayRoute] + B --> C{Model Header} + C -->|gpt-4o-mini| D[OpenAI AIServiceBackend] + C -->|claude-3-sonnet| E[Bedrock AIServiceBackend] + D --> F[OpenAI BackendSecurityPolicy] + E --> G[AWS BackendSecurityPolicy] + F --> H[OpenAI API] + G --> I[AWS Bedrock API] + + style A fill:#e1f5fe + style B fill:#f3e5f5 + style D fill:#e8f5e8 + style E fill:#e8f5e8 + style F fill:#fff3e0 + style G fill:#fff3e0 +``` + +### Data Flow Process + +1. **Request Reception**: Client sends a request to the AI Gateway with the OpenAI-compatible format +2. **Route Matching**: AIGatewayRoute examines request headers (like `x-ai-eg-model`) to determine the target backend +3. **Backend Resolution**: The matching rule identifies the appropriate AIServiceBackend +4. **Authentication**: The AIServiceBackend's security policy provides credentials for upstream authentication +5. **Schema Transformation**: If needed, the request is transformed from the input schema to the backend's expected schema +6. **Provider Communication**: The request is forwarded to the actual AI provider with proper authentication +7. **Response Processing**: The provider's response is transformed back to the unified schema format +8. **Client Response**: The standardized response is returned to the client + +## Common Configuration Patterns + +### Single Provider Setup + +For a simple single-provider setup: + +```yaml +# Backend configuration +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIServiceBackend +metadata: + name: openai-backend +spec: + schema: + name: OpenAI + version: v1 + backendRef: + name: openai-backend + kind: Backend + group: gateway.envoyproxy.io + +--- +# Security configuration +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: openai-auth +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai-backend + type: APIKey + apiKey: + secretRef: + name: openai-secret + namespace: default + +--- +# Routing configuration +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: openai-route +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: openai-backend +``` + +### Multi-Provider Setup with Fallback + +For high availability with multiple providers: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: multi-provider-fallback +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4o-mini + backendRefs: + - name: openai-backend + - name: groq-backend # Fallback provider +``` + +### Model-Specific Routing + +For routing different models to specialized providers: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: model-specific-routing +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4o-mini + backendRefs: + - name: openai-backend + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: claude-3-sonnet + backendRefs: + - name: bedrock-backend + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: text-embedding-ada-002 + backendRefs: + - name: openai-embeddings-backend +``` + +Configure model ownership and creation information for the `/models` endpoint: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: model-metadata +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4o-mini + modelsOwnedBy: "OpenAI" + modelsCreatedAt: "2024-01-01T00:00:00Z" + backendRefs: + - name: openai-backend +``` + +## Provider-Specific Considerations + +### OpenAI-Compatible Providers + +Many providers offer OpenAI-compatible APIs: + +- Use OpenAI schema configuration +- Adjust the version field if the provider uses custom paths +- Standard API key authentication typically applies + +### Cloud Provider Integration + +Cloud providers like AWS Bedrock and Azure OpenAI require: + +- Cloud-specific credential types (AWS IAM, Azure Service Principal, GCP Workload Identity) +- Region specification for multi-region services +- Custom schema configurations for native APIs + +### Self-Hosted Models + +Self-hosted models using frameworks like vLLM: + +- Often compatible with OpenAI schema +- May not require authentication (internal networks) +- Custom endpoints through Envoy Gateway Backend resources + +## Validation and Troubleshooting + +### Configuration Validation + +The AI Gateway validates configurations at deployment time: + +- **Schema Compatibility**: Ensures input and output schemas are compatible +- **Resource References**: Validates that referenced resources exist +- **Credential Access**: Verifies that secrets are accessible + +### Status Conditions + +All resources provide status conditions to monitor their health: + +- **Accepted**: Resource is valid and has been accepted by the controller +- **NotAccepted**: Resource has validation errors or configuration issues + +### Common Issues and Solutions + +**Authentication Failures (401/403)** + +- Verify API keys and credentials are correct +- Check secret references and key names +- Ensure credentials have necessary permissions + +**Schema Mismatch Errors** + +- Confirm the backend schema matches the provider's API +- Check version specifications for provider-specific paths +- Review API documentation for schema requirements + +**Routing Issues** + +- Verify header matching rules in AIGatewayRoute +- Check that model names match expected values +- Ensure backend references point to existing AIServiceBackends + +**Backend Reference Errors** + +- Ensure backendRef points to Envoy Gateway Backend resources +- Verify the Backend resource exists and is properly configured +- Check that the group field is set to `gateway.envoyproxy.io` + +## Next Steps + +Now that you understand the connectivity fundamentals: + +- **[Supported Providers](/docs/capabilities/llm-integrations/supported-providers)** - View the complete list of supported providers and their configurations +- **[Supported Endpoints](/docs/capabilities/llm-integrations/supported-endpoints)** - Learn about available API endpoints and their capabilities +- **[Getting Started Guide](/docs/getting-started/connect-providers)** - Follow hands-on tutorials for specific providers +- **[Traffic Management](/docs/capabilities/traffic)** - Configure advanced routing, rate limiting, and fallback strategies +- **[Security](/docs/capabilities/security)** - Implement comprehensive security policies for your AI traffic + +## API Reference + +For detailed information about resource fields and configuration options: + +- [AIServiceBackend API Reference](../../api/api.mdx#aiservicebackend) +- [BackendSecurityPolicy API Reference](../../api/api.mdx#backendsecuritypolicy) +- [AIGatewayRoute API Reference](../../api/api.mdx#aigatewayroute) diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/index.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/index.md new file mode 100644 index 0000000000..ecf9862b40 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/index.md @@ -0,0 +1,22 @@ +--- +id: llm-integrations +title: LLM Provider Integrations +--- + +# LLM Provider Integrations + +Envoy AI Gateway provides integration to multiple LLM Providers. This section provides insight into the providers and endpoints available to integrate with. + +## Provider Support + +- **[Supported Providers](./supported-providers.md)** - Complete list of supported AI/LLM service providers and their configurations +- **[Supported Endpoints](./supported-endpoints.md)** - Available API endpoints and operations across different providers + +## Advanced Features + +- **[Vendor-Specific Fields](./vendor-specific-fields.md)** - Use backend-specific parameters and access provider-unique capabilities in your OpenAI-compatible requests +- **[Prompt Caching](./prompt-caching.md)** - Provider-agnostic prompt caching using unified cache_control API + +## Getting Started with Provider Connectivity + +- **[Connecting to AI Providers](./connect-providers.md)** - Learn the fundamental concepts, resources, and relationships required to establish connectivity with any supported AI provider diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/prompt-caching.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/prompt-caching.md new file mode 100644 index 0000000000..1989498f6d --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/prompt-caching.md @@ -0,0 +1,174 @@ +--- +id: prompt-caching +title: Prompt Caching +sidebar_position: 6 +--- + +# Prompt Caching + +Envoy AI Gateway provides provider-agnostic prompt caching through a unified `cache_control` API. The same cache syntax works across multiple providers: Direct Anthropic, GCP Vertex AI (Claude models), and AWS Bedrock (Claude models). This reduces costs and improves response times by caching frequently-used content like system prompts, tool definitions, and reference documents. + +## Supported Providers + +| Provider | API Schema | Cache Support | +| ---------------------- | --------------------- | ------------- | +| Anthropic (Direct) | `Anthropic` | Native | +| GCP Vertex AI (Claude) | `GCPAnthropic` | Translated | +| AWS Bedrock (Claude) | `AWSBedrockAnthropic` | Translated | + +## How It Works + +- Add `cache_control: {"type": "ephemeral"}` to content blocks in your request. +- AI Gateway translates this to the provider-specific format automatically. +- Cache is maintained per-provider; all providers require a minimum of 1,024 tokens for caching. +- A maximum of 4 cache breakpoints are allowed per request across all providers. +- On cache hit, the provider charges reduced input token costs. + +## Usage + +No gateway-side configuration is needed. Caching is controlled entirely at the request level by adding `cache_control` to the content blocks you want cached. + +### Basic Example: System Prompt Caching + +Cache a system prompt so that subsequent requests reuse the cached content: + +```json +{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a helpful assistant with extensive knowledge...(long system prompt)...", + "cache_control": { "type": "ephemeral" } + } + ] + }, + { + "role": "user", + "content": "What is the capital of France?" + } + ] +} +``` + +### Multiple Cache Points + +You can place up to 4 cache breakpoints in a single request to cache different parts of the conversation: + +```json +{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "System instructions...", + "cache_control": { "type": "ephemeral" } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Reference document content...", + "cache_control": { "type": "ephemeral" } + }, + { + "type": "text", + "text": "Question about the document" + } + ] + } + ] +} +``` + +### Tool Definition Caching + +Cache complex tool schemas that remain the same across requests: + +```json +{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": "Help me search for information about cloud computing trends." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "search_knowledge_base", + "description": "Search through a comprehensive knowledge base...", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Natural language search query" + } + }, + "required": ["query"] + }, + "cache_control": { "type": "ephemeral" } + } + } + ] +} +``` + +## Response Format + +When caching is active, the response includes cache information in the `usage` field: + +```json +{ + "usage": { + "prompt_tokens": 2000, + "completion_tokens": 150, + "prompt_tokens_details": { + "cached_tokens": 1800 + } + } +} +``` + +- `cached_tokens` indicates the number of tokens served from cache at a reduced cost. +- Cache write tokens are tracked internally for billing purposes. + +## Best Practices + +:::tip + +- Place `cache_control` on content that exceeds 1,024 tokens. Content below this threshold will not be cached. +- Cache system prompts, tool definitions, and reference documents that do not change between requests. +- Position cache breakpoints strategically -- cached content must appear at the beginning of the message. +- Monitor `cached_tokens` in responses to verify caching effectiveness and measure cost savings. + ::: + +## Provider-Specific Notes + +:::note + +- **All providers**: Minimum 1,024 tokens per cached block, maximum 4 cache breakpoints per request. +- **Anthropic Direct**: Uses the native `cache_control` field directly with no translation. +- **GCP Vertex AI**: AI Gateway translates `cache_control` to Vertex AI's caching format automatically. +- **AWS Bedrock**: AI Gateway translates `cache_control` to Bedrock's cachePoint format automatically. +- All providers support the `"ephemeral"` cache type. +- Existing requests without `cache_control` continue to work with no changes. + ::: + +## Further Reading + +- [Prompt Caching Examples](https://github.com/envoyproxy/ai-gateway/tree/main/examples/cache) -- Detailed examples with curl commands for each provider. +- [Connecting to GCP Vertex AI](../../getting-started/connect-providers/gcp-vertexai.md) -- Set up GCP Vertex AI as a provider. +- [Connecting to AWS Bedrock](../../getting-started/connect-providers/aws-bedrock.md) -- Set up AWS Bedrock as a provider. diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-endpoints.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-endpoints.md new file mode 100644 index 0000000000..9c503940eb --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-endpoints.md @@ -0,0 +1,456 @@ +--- +id: supported-endpoints +title: Supported API Endpoints +sidebar_position: 9 +--- + +The Envoy AI Gateway provides OpenAI-compatible API endpoints as well as the Anthropic-compatible API for routing and managing LLM/AI traffic. This page documents which OpenAI API endpoints and Anthropic-compatible API endpoints are currently supported and their capabilities. + +## Overview + +The Envoy AI Gateway acts as a proxy that accepts OpenAI-compatible and Anthropic-compatible requests and routes them to various AI providers. While it maintains compatibility with the OpenAI API specification, it currently supports a subset of the full OpenAI API. + +## Supported Endpoints + +### Chat Completions + +**Endpoint:** `POST /v1/chat/completions` + +**Status:** ✅ Fully Supported + +**Description:** Create a chat completion response for the given conversation. + +**Features:** + +- ✅ Streaming and non-streaming responses +- ✅ Function calling +- ✅ Response format specification (including JSON schema) +- ✅ Temperature, top_p, and other sampling parameters +- ✅ System and user messages +- ✅ Audio and video inputs +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing + +**Supported Providers:** + +- OpenAI +- AWS Bedrock (with automatic translation) +- Azure OpenAI (with automatic translation) +- GCP VertexAI (with automatic translation) +- GCP Anthropic (with automatic translation) +- Any OpenAI-compatible provider (Groq, Together AI, Mistral, Tetrate Agent Router Service, etc.) + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +### Anthropic Messages + +**Endpoint:** `POST /anthropic/v1/messages` + +**Status:** ✅ Fully Supported + +**Description:** Send a structured list of input messages with text and/or image content, and the model will generate the next message in the conversation. + +**Features:** + +- ✅ Streaming and non-streaming responses +- ✅ Function calling +- ✅ Extended thinking +- ✅ Response format specification (including JSON schema) +- ✅ Temperature, top_p, and other sampling parameters +- ✅ System and user messages +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing + +**Supported Providers:** + +- Anthropic +- GCP Anthropic +- AWS Anthropic +- AWS Bedrock + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "claude-sonnet-4", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "max_tokens": 100 + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +### Completions + +**Endpoint:** `POST /v1/completions` + +**Status:** ✅ Fully Supported + +**Description:** Create a text completion for the given prompt (legacy endpoint). + +**Features:** + +- ✅ Non-streaming responses +- ✅ Streaming responses +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Temperature, top_p, and other sampling parameters +- ✅ Single and batch prompt processing +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing +- ✅ Full metrics support (token usage, request duration, time to first token, inter-token latency) + +**Supported Providers:** + +- OpenAI +- Any OpenAI-compatible provider that supports completions + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "babbage-002", + "prompt": "def fib(n):\n if n <= 1:\n return n\n else:\n return fib(n-1) + fib(n-2)", + "max_tokens": 25, + "temperature": 0.4, + "top_p": 0.9 + }' \ + $GATEWAY_URL/v1/completions +``` + +### Embeddings + +**Endpoint:** `POST /v1/embeddings` + +**Status:** ✅ Fully Supported + +**Description:** Create embeddings for the given input text. + +**Features:** + +- ✅ Single and batch text embedding +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing + +**Supported Providers:** + +- OpenAI +- AWS Bedrock (Titan models, with automatic translation) +- GCP VertexAI (with automatic translation) +- Any OpenAI-compatible provider that supports embeddings, including Azure OpenAI. + +### Image Generation + +**Endpoint:** `POST /v1/images/generations` + +**Status:** ✅ Supported + +**Description:** Generate one or more images from a text prompt using OpenAI-compatible models. + +**Features:** + +- **Non-streaming responses**: Returns JSON payload with image URLs or base64 content +- **Model selection**: Via request body `model` or `x-ai-eg-model` header +- **Parameters**: `prompt`, `size`, `n`, `quality`, `response_format` +- **Metrics**: Records image count, model, and size; token usage when provided +- **Provider fallback and load balancing** + +**Supported Providers:** + +- OpenAI +- Any OpenAI-compatible provider that supports image generations + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-image-1", + "prompt": "a serene mountain landscape at sunrise in watercolor", + "size": "1024x1024", + "n": 1 + }' \ + $GATEWAY_URL/v1/images/generations +``` + +### Audio Transcriptions + +**Endpoint:** `POST /v1/audio/transcriptions` + +**Status:** ✅ Supported + +**Description:** Transcribe audio into text in the language of the audio. + +**Features:** + +- ✅ Multipart/form-data file upload (OpenAI-compatible) +- ✅ Model selection via form field `model` or `x-ai-eg-model` header +- ✅ Optional parameters: `language`, `prompt`, `response_format`, `temperature`, `timestamp_granularities[]` +- ✅ JSON and verbose JSON response formats +- ✅ Provider fallback and load balancing +- ✅ Model name virtualization (override model names for backends) + +**Supported Providers:** + +- OpenAI +- Any OpenAI-compatible provider that supports audio transcriptions + +**Example:** + +```bash +curl -F "model=whisper-1" \ + -F "file=@audio.mp3" \ + -F "language=en" \ + $GATEWAY_URL/v1/audio/transcriptions +``` + +### Audio Translations + +**Endpoint:** `POST /v1/audio/translations` + +**Status:** ✅ Supported + +**Description:** Translate audio into English text. + +**Features:** + +- ✅ Multipart/form-data file upload (OpenAI-compatible) +- ✅ Model selection via form field `model` or `x-ai-eg-model` header +- ✅ Optional parameters: `prompt`, `response_format`, `temperature` +- ✅ Provider fallback and load balancing +- ✅ Model name virtualization (override model names for backends) + +**Supported Providers:** + +- OpenAI +- Any OpenAI-compatible provider that supports audio translations + +**Example:** + +```bash +curl -F "model=whisper-1" \ + -F "file=@audio.mp3" \ + $GATEWAY_URL/v1/audio/translations +``` + +### Responses + +**Endpoint:** `POST /v1/responses` + +**Status:** ✅ Fully Supported + +**Description:** Creates a model response. Provide text or image inputs to generate text or JSON outputs. Have the model call your own custom code or use built-in tools. + +**Features:** + +- ✅ Streaming and non-streaming responses +- ✅ Function calling +- ✅ MCP Tools support +- ✅ Reasoning +- ✅ Multi-turn conversations +- ✅ Native multimodal support for text and images +- ✅ Response format specification (including JSON schema) +- ✅ Temperature, top_p, and other sampling parameters +- ✅ System and user messages +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing + +**Supported Providers:** + +- OpenAI +- Azure OpenAI with an API version that supports Responses, such as `2025-04-01-preview` +- Any OpenAI-compatible provider (Groq, Together AI, Mistral, Tetrate Agent Router Service, etc.) + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4.1", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is in this image?"}, + { + "type": "input_image", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + } + ] + }' \ + $GATEWAY_URL/v1/responses +``` + +### Rerank + +**Endpoint:** `POST /cohere/v2/rerank` + +**Status:** ✅ Fully Supported + +**Description:** Rerank a list of documents for a given query to return relevance scores and an ordered list. Cohere-compatible API. + +**Features:** + +- ✅ Single-query document reranking +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Token usage tracking and cost calculation +- ✅ Provider fallback and load balancing + +**Supported Providers:** + +- Cohere +- Any Cohere-compatible provider that supports rerank, including vLLM. + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "rerank-english-v3.0", + "query": "What is the capital of France?", + "documents": [ + "Paris is the capital of France.", + "Berlin is the capital of Germany." + ] + }' \ + $GATEWAY_URL/cohere/v2/rerank +``` + +### Models + +**Endpoint:** `GET /v1/models` + +**Description:** List available models configured in the AI Gateway. + +**Features:** + +- ✅ Returns models declared in AIGatewayRoute configurations +- ✅ OpenAI-compatible response format +- ✅ Model metadata (ID, owned_by, created timestamp) + +**Example:** + +```bash +curl $GATEWAY_URL/v1/models +``` + +**Response Format:** + +```json +{ + "object": "list", + "data": [ + { + "id": "gpt-4o-mini", + "object": "model", + "created": 1677610602, + "owned_by": "openai" + } + ] +} +``` + +## Provider-Endpoint Compatibility Table + +The following table summarizes which providers support which endpoints: + +| Provider | Chat Completions | Completions | Embeddings | Image Generation | Anthropic Messages | Rerank | Notes | +| ----------------------------------------------------------------------------------------------------- | :--------------: | :---------: | :--------: | :--------------: | :----------------: | :----: | -------------------------------------------------------------------------------------------------------------------- | +| [OpenAI](https://platform.openai.com/docs/api-reference) | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | | +| [AWS Bedrock](https://docs.aws.amazon.com/bedrock/latest/APIReference/) | ✅ | 🚧 | ✅ | ❌ | ❌ | ❌ | Via API translation (embeddings: Titan models only) | +| [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference) | ✅ | 🚧 | ✅ | ❌ | ⚠️ | ❌ | Via API translation or via [OpenAI-compatible API](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/latest) | +| [Google Gemini](https://ai.google.dev/gemini-api/docs/openai) | ✅ | ⚠️ | ✅ | ⚠️ | ❌ | ❌ | Via OpenAI-compatible API | +| [Groq](https://console.groq.com/docs/openai) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Grok](https://docs.x.ai/docs/api-reference) | ✅ | ⚠️ | ❌ | ⚠️ | ❌ | ❌ | Via OpenAI-compatible API | +| [Together AI](https://docs.together.ai/docs/openai-api-compatibility) | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | Via OpenAI-compatible API | +| [Cohere](https://docs.cohere.com/v2/docs/compatibility-api) | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ✅ | Via OpenAI-compatible API and Cohere V2 API for rerank | +| [Mistral](https://docs.mistral.ai/api/) | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [DeepInfra](https://deepinfra.com/docs/inference) | ✅ | ⚠️ | ✅ | ⚠️ | ❌ | ❌ | Via OpenAI-compatible API | +| [DeepSeek](https://api-docs.deepseek.com/) | ⚠️ | ⚠️ | ❌ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Hunyuan](https://cloud.tencent.com/document/product/1729/111007) | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Tencent LLM Knowledge Engine](https://www.tencentcloud.com/document/product/1255/70381) | ⚠️ | ❌ | ❌ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Tetrate Agent Router Service (TARS)](https://router.tetrate.ai/) | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Google Vertex AI](https://cloud.google.com/vertex-ai/docs/reference/rest) | ✅ | 🚧 | ✅ | ❌ | ❌ | ❌ | Via API translation | +| [Anthropic on Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude) | ✅ | ❌ | 🚧 | ❌ | ✅ | ❌ | Via OpenAI-compatible API and Native Anthropic API | +| [Anthropic on AWS Bedrock](https://aws.amazon.com/bedrock/anthropic/) | 🚧 | ❌ | ❌ | ❌ | ✅ | ❌ | Native Anthropic API | +| [SambaNova](https://docs.sambanova.ai/sambastudio/latest/open-ai-api.html) | ✅ | ⚠️ | ✅ | ❌ | ❌ | ❌ | Via OpenAI-compatible API | +| [Anthropic](https://docs.claude.com/en/home) | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | Via OpenAI-compatible API and Native Anthropic API | + +- ✅ - Supported and Tested on Envoy AI Gateway CI +- ⚠️️ - Expected to work based on provider documentation, but not tested on the CI. +- ❌ - Not supported according to provider documentation. +- 🚧 - Unimplemented, or under active development but planned for future releases + +## Custom endpoint prefixes + +By default, the gateway registers provider endpoints under these prefixes: + +- OpenAI: `/` +- Cohere: `/cohere` +- Anthropic: `/anthropic` + +You can override them via Helm using values under `endpointConfig`: + +```yaml +# values.yaml +endpointConfig: + # Explicit provider roots + openai: "" + cohere: "/cohere" + anthropic: "/anthropic" + # rootPrefix applies to all routes; final paths are /... + # endpointConfig: + # rootPrefix: "/" +``` + +Or with helm CLI: + +```bash +helm upgrade --install ai-gateway envoyproxy/ai-gateway-helm \ + -n envoy-ai-gateway-system --create-namespace \ + --set 'endpointConfig.openai=/' \ + --set 'endpointConfig.cohere=/cohere' \ + --set 'endpointConfig.anthropic=/anthropic' +``` + +Notes: + +- `endpointConfig.rootPrefix` (default `/`) is prepended to all provider prefixes. +- Only these keys are accepted: `openaiPrefix`, `coherePrefix`, `anthropicPrefix`. +- If any key is omitted or empty, defaults are applied as listed above. + +## What's Next + +To learn more about configuring and using the Envoy AI Gateway with these endpoints: + +- **[Supported Providers](./supported-providers.md)** - Complete list of supported AI providers and their configurations +- **[Usage-Based Rate Limiting](../traffic/usage-based-ratelimiting.md)** - Configure token-based rate limiting and cost controls +- **[Provider Fallback](../traffic/provider-fallback.md)** - Set up automatic failover between providers for high availability +- **[Metrics and Monitoring](../observability/metrics.md)** - Monitor usage, costs, and performance metrics + +[issue#609]: https://github.com/envoyproxy/ai-gateway/issues/609 diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-providers.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-providers.md new file mode 100644 index 0000000000..2f7961c6f5 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/supported-providers.md @@ -0,0 +1,43 @@ +--- +id: supported-providers +title: Supported AI Providers +sidebar_position: 8 +--- + +Since the Envoy AI Gateway is designed to provide a Unified API for routing and managing LLM/AI traffic, it supports various AI providers out of the box. +A "support of provider" means two things: the API schema support and the Authentication support. \ +The former can be configured in the `AIServiceBackend` resource's `schema` field, while the latter is configured in the `BackendSecurityPolicy`. + +Below is a table of currently supported providers and their respective configurations. + +| Provider Name | API Schema Config on [AIServiceBackend] | Upstream Authentication Config on [BackendSecurityPolicy] | Status | Note | +| --------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------: | :-------------------------------------------------------: | :----: | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [OpenAI](https://platform.openai.com/docs/api-reference) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [AWS Bedrock](https://docs.aws.amazon.com/bedrock/latest/APIReference/) | `{"name":"AWSBedrock"}` | [AWS Bedrock Credentials] | ✅ | | +| [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference) | `{"name":"AzureOpenAI","version":"2025-01-01-preview"}` or `{"name":"OpenAI", "prefix": "/openai/v1"}` | [Azure Credentials] or [Azure API Key] | ✅ | | +| [Google Gemini on AI Studio](https://ai.google.dev/gemini-api/docs/openai) | `{"name":"OpenAI","prefix":"/v1beta/openai"}` | [API Key] | ✅ | Only the OpenAI compatible endpoint | +| [Google Vertex AI](https://cloud.google.com/vertex-ai/docs/reference/rest) | `{"name":"GCPVertexAI"}` | [GCP Credentials] | ✅ | Supports ADC, Service Account Keys, and Workload Identity Federation | +| [Anthropic on GCP Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude) | `{"name":"GCPAnthropic", "version":"vertex-2023-10-16"}` | [GCP Credentials] | ✅ | Native Anthropic and OpenAI endpoints. Supports ADC, Service Account Keys, and Workload Identity Federation | +| [Groq](https://console.groq.com/docs/openai) | `{"name":"OpenAI","prefix":"/openai/v1"}` | [API Key] | ✅ | | +| [Grok](https://docs.x.ai/docs/api-reference?utm_source=chatgpt.com#chat-completions) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [Together AI](https://docs.together.ai/docs/openai-api-compatibility) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [Cohere](https://docs.cohere.com/v2/docs/compatibility-api) | `{"name":"Cohere","version":"v2"}` or `{"name":"OpenAI","prefix":"/compatibility/v1"}` | [API Key] | ✅ | Supports native Cohere v2 (e.g., /cohere/v2/rerank) and OpenAI-compatible endpoints. | +| [Mistral](https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [DeepInfra](https://deepinfra.com/docs/inference) | `{"name":"OpenAI","prefix":"/v1/openai"}` | [API Key] | ✅ | Only the OpenAI compatible endpoint | +| [DeepSeek](https://api-docs.deepseek.com/) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [Hunyuan](https://cloud.tencent.com/document/product/1729/111007) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [Tencent LLM Knowledge Engine](https://www.tencentcloud.com/document/product/1255/70381?lang=en) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [Tetrate Agent Router Service (TARS)](https://router.tetrate.ai/) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| [SambaNova](https://docs.sambanova.ai/sambastudio/latest/open-ai-api.html) | `{"name":"OpenAI","prefix":"/v1"}` | [API Key] | ✅ | | +| Self-hosted-models | `{"name":"OpenAI","prefix":"/v1"}` | N/A | ⚠️ | Depending on the API schema spoken by self-hosted servers. For example, [vLLM] speaks the OpenAI format. Also, API Key auth can be configured as well. | +| [Anthropic](https://docs.claude.com/en/home) | `{"name":"Anthropic"}` | [Anthropic API Key] | ✅ | Support only Native Anthropic messages endpoint | + +[AIServiceBackend]: api/api.mdx#aiservicebackendspec +[BackendSecurityPolicy]: api/api.mdx#backendsecuritypolicyspec +[API Key]: api/api.mdx#backendsecuritypolicyapikey +[AWS Bedrock Credentials]: api/api.mdx#backendsecuritypolicyawscredentials +[GCP Credentials]: api/api.mdx#backendsecuritypolicygcpcredentials +[Azure Credentials]: api/api.mdx#backendsecuritypolicyazurecredentials +[Azure API Key]: api/api.mdx#backendsecuritypolicyazureapikey +[Anthropic API Key]: api/api.mdx#backendsecuritypolicyanthropicapikey +[vLLM]: https://docs.vllm.ai/en/v0.8.3/serving/openai_compatible_server.html diff --git a/site/versioned_docs/version-0.7/capabilities/llm-integrations/vendor-specific-fields.md b/site/versioned_docs/version-0.7/capabilities/llm-integrations/vendor-specific-fields.md new file mode 100644 index 0000000000..b70a90c766 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/llm-integrations/vendor-specific-fields.md @@ -0,0 +1,163 @@ +--- +id: vendor-specific-fields +title: Extension Fields +--- + +# Extension Fields + +The AI Gateway supports extension fields that allow you to specify unified or backend-specific parameters directly as inline fields in your OpenAI-compatible requests. These fields are applied during the translation process to the target backend's native API format. + +## Overview + +Extension fields enable you to: + +- Use advanced backend-specific features not available in the OpenAI API +- Use unified configuration fields that work across multiple providers not available in the OpenAI API + +### Vendor Extension Fields + +Vendor specific fields are specified as inline fields in your OpenAI request and are applied after the standard OpenAI-to-backend translation. + +### Unified Extension Fields + +For thinking/reasoning capabilities, you can use a unified `thinking` field that automatically translates to the correct backend-specific format: + +- **GCP Vertex AI (Gemini)**: Translates to `generationConfig.thinkingConfig` +- **GCP Anthropic**: Uses `thinking` field directly +- **AWS Bedrock**: Uses `thinking` field directly + +This unified approach allows you to write provider-agnostic requests while still leveraging thinking capabilities. + +## Supported Backends + +The following backends support extension fields: + +### GCP Vertex AI (Gemini) + +- **API Schema Name**: `GCPVertexAI` +- **Supported Fields**: + - `safetySettings`: Configure the safety settings for gemini models that translates to `SafetySetting`. [Gemini Docs](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters) + - `thinking`: Configure thinking process for reasoning models that automatically translates to `generationConfig.thinkingConfig`. [Gemini Docs](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/thinking) +- **Supported Tools**: + - `google_search`: Enable Google Search grounding for Gemini models. Configuration options vary by platform: `exclude_domains` and `blocking_confidence` are Vertex AI only, while `time_range_filter` is Gemini API only. [Google Search Grounding Docs](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/grounding/grounding-with-google-search) + +### GCP Anthropic + +- **API Schema Name**: `GCPAnthropic` +- **Supported Fields**: + - `thinking`: Configuration for enabling Claude's extended thinking. [Anthropic Docs](https://docs.anthropic.com/en/api/messages#body-thinking) + +### AWS Bedrock + +- **API Schema Name**: `AWSBedrock` +- **Supported Fields**: + - `thinking`: Configuration for enabling Anthropic Claude's extended thinking. [AWS Docs](https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-extended-thinking.html) + +## Usage + +Add extension fields directly as inline fields in your OpenAI request: + +### Using Unified Thinking Configuration + +The simplest way to enable thinking capabilities across all providers is to use the unified `thinking` field: + +```json +{ + "model": "gemini-2.5-pro", + "messages": [ + { + "role": "user", + "content": "Explain quantum computing and show me a simple code example." + } + ], + "temperature": 0.7, + "max_tokens": 2000, + "thinking": { + "type": "enabled", + "budget_tokens": 1000, + "includeThoughts": true + } +} +``` + +This configuration will work with any provider that supports thinking, automatically translating to the correct backend format. + +### Using Provider-Specific Fields + +For more fine-grained control or provider-specific features, you can use the vendor-specific fields like `safetySettings` for gemini models: + +```json +{ + "model": "gemini-1.5-pro", + "messages": [ + { + "role": "user", + "content": "Explain quantum computing and show me a simple code example." + } + ], + "temperature": 0.7, + "max_tokens": 2000, + "safetySettings": [ + { + "category": "HARM_CATEGORY_HARASSMENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} +``` + +### Using Google Search Grounding + +To enable Google Search grounding for Gemini models, add `google_search` to the tools array. + +For basic usage without configuration options: + +```json +{ + "model": "gemini-2.0-flash", + "messages": [ + { + "role": "user", + "content": "What are the latest developments in quantum computing?" + } + ], + "tools": [ + { + "type": "google_search" + } + ] +} +``` + +For Vertex AI, you can add filtering options: + +```json +{ + "model": "gemini-2.0-flash", + "messages": [ + { + "role": "user", + "content": "What are the latest developments in quantum computing?" + } + ], + "tools": [ + { + "type": "google_search", + "google_search": { + "exclude_domains": ["example.com"], + "blocking_confidence": "BLOCK_LOW_AND_ABOVE" + } + } + ] +} +``` + +### Field Conflicts + +Vendor fields override translated fields when conflicts occur. + +When using unified thinking configuration, the `thinking` field takes precedence over any provider-specific thinking configurations. + +### Unsupported Fields/Backends + +Fields and Backends other than specified in [Supported Backends](#supported-backends) will be ignored. diff --git a/site/versioned_docs/version-0.7/capabilities/mcp/index.md b/site/versioned_docs/version-0.7/capabilities/mcp/index.md new file mode 100644 index 0000000000..c75d045396 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/mcp/index.md @@ -0,0 +1,417 @@ +--- +id: mcp +title: Model Context Protocol (MCP) Gateway +sidebar_position: 8 +--- + +Envoy AI Gateway provides first-class support for [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), enabling AI agents to securely connect to external tools and data sources. + +This guide provides an overview of the MCP Gateway capabilities and how to configure routing to MCP servers using the `MCPRoute` API. + +## Overview + +Envoy AI Gateway's MCP support allows you to: + +- **Aggregate multiple MCP servers** into a single unified endpoint +- **Apply security policies** including OAuth authentication, fine-grained access control over the tool access, and upstream API key injection +- **Filter tools** to control which capabilities are exposed to clients +- **Leverage Envoy's networking** for load balancing, rate limiting, circuit breaking, and observability + +The MCP Gateway acts as a transparent proxy between MCP clients (AI agents like Claude, Goose, etc.) and backend MCP servers, providing the same production-grade features available for LLM traffic. + +## Key Features + +| Feature | Description | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Streamable HTTP Transport** | Full support for MCP's streamable HTTP transport, aligning with the [June 2025 MCP spec](https://modelcontextprotocol.io/specification/2025-06-18).
Efficient handling of stateful sessions and multi-part JSON-RPC messaging over persistent HTTP connections. | +| **Fine-Grained Authorization** | Native enforcement of [OAuth authentication flows](https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization).
Implement granular access control using JWT claims, scopes, and CEL expressions. | +| **Server Multiplexing & Tool Routing** | Route tool calls to the right MCP backends, aggregating and filtering available tools based on gateway policy.
Dynamically merge streaming notifications from multiple MCP servers into a unified interface. | +| **Upstream Authentication** | Built-in upstream authentication primitives to securely connect to external MCP servers using API keys and header injection. | +| **Full MCP Spec Coverage** | Complete [June 2025 MCP spec](https://modelcontextprotocol.io/specification/2025-06-18) compliance, including support for tool calls, notifications, prompts, resources, and bi-directional server-to-client requests. | +| **Built-in Observability** | OpenTelemetry tracing and Prometheus metrics for all MCP requests, using the same observability stack as LLM traffic. | + +## Architecture + +The MCP Gateway is implemented as a lightweight proxy component within the Envoy AI Gateway sidecar, leveraging Envoy's battle-tested networking stack for all connection handling. + +```mermaid +sequenceDiagram + participant Client as MCP Client
(AI Agent) + participant Gateway as Envoy AI Gateway + participant MCP1 as MCP Server 1 + participant MCP2 as MCP Server 2 + + Client->>Gateway: Initialize session + Gateway->>MCP1: Initialize + Gateway->>MCP2: Initialize + MCP1-->>Gateway: Session ID 1 + MCP2-->>Gateway: Session ID 2 + Gateway-->>Client: Unified Session ID + + Client->>Gateway: List tools + Gateway->>MCP1: List tools + Gateway->>MCP2: List tools + MCP1-->>Gateway: Tools A, B + MCP2-->>Gateway: Tools X, Y + Gateway-->>Client: Filtered & merged tools + + Client->>Gateway: Call tool (server1__toolA) + Gateway->>MCP1: Call toolA + MCP1-->>Gateway: Result + Gateway-->>Client: Result +``` + +**Key architectural aspects:** + +- **Session Management**: The gateway creates unified sessions by encoding multiple backend session IDs, handling reconnection with `Last-Event-ID` support for SSE streams. +- **Notification Handling**: Long-lived SSE streams from multiple MCP servers are merged into a single stream for clients, with proper event ID reconstruction. +- **Request Routing**: Tool names are automatically prefixed with the backend name (e.g., `github__issue_read`) to route calls to the correct upstream server. + +For detailed architecture and design decisions, see the [MCP Gateway proposal](https://github.com/envoyproxy/ai-gateway/tree/main/docs/proposals/006-mcp-gateway). + +## Trying it out + +Before you begin, you'll need to complete the basic setup from the [Basic Usage](/docs/getting-started/basic-usage) guide, which includes installing Envoy Gateway and AI Gateway. + +### Basic MCPRoute Configuration + +The following example demonstrates a basic `MCPRoute` that proxies the GitHub MCP server: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: MCPRoute +metadata: + name: mcp-route + namespace: default +spec: + parentRefs: + - name: aigw-run + kind: Gateway + group: gateway.networking.k8s.io + path: "/mcp" # Clients connect to http://gateway-address/mcp + backendRefs: + - name: github + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp/x/issues/readonly" + securityPolicy: + apiKey: + secretRef: + name: github-token +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: github + namespace: default +spec: + endpoints: + - fqdn: + hostname: api.githubcopilot.com + port: 443 +--- +apiVersion: v1 +kind: Secret +metadata: + name: github-token + namespace: default +type: Opaque +stringData: + apiKey: ghp_your_token_here +``` + +Apply this configuration: + +```shell +kubectl apply -f mcp-route.yaml +``` + +Now clients can connect to `http:///mcp` and access GitHub tools. + +### Tool Filtering + +Control which tools are exposed using the `toolSelector` field. You can use exact matches or regular expressions: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: MCPRoute +metadata: + name: mcp-route + namespace: default +spec: + parentRefs: + - name: aigw-run + kind: Gateway + group: gateway.networking.k8s.io + backendRefs: + # GitHub: only expose issue-related tools + - name: github + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp/x/issues/readonly" + toolSelector: + includeRegex: + - .*issues?.* # Matches issue_read, list_issues, etc. + securityPolicy: + apiKey: + secretRef: + name: github-token + + # Context7: expose specific tools by exact name + - name: context7 + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp" + toolSelector: + include: + - resolve-library-id + - query-docs +``` + +:::note +The `toolSelector` field requires exactly one of `include` or `includeRegex` to be specified. If not specified, all tools from the MCP server are exposed. +::: + +### Server Multiplexing + +The gateway automatically aggregates tools from multiple MCP servers into a single unified interface: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: MCPRoute +metadata: + name: mcp-unified + namespace: default +spec: + parentRefs: + - name: aigw-run + kind: Gateway + group: gateway.networking.k8s.io + path: "/mcp" + backendRefs: + - name: github + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp/x/issues/readonly" + securityPolicy: + apiKey: + secretRef: + name: github-token + - name: context7 + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp" +``` + +Clients will see all tools with prefixed names: + +- `github__issue_read` +- `github__list_issues` +- `context7__resolve-library-id` +- `context7__query-docs` + +### Header Forwarding + +Forward HTTP headers from the client request to specific backend MCP servers. This enables per-user authentication passthrough (e.g., personal access tokens) without requiring OAuth: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: MCPRoute +metadata: + name: mcp-unified + namespace: default +spec: + parentRefs: + - name: aigw-run + kind: Gateway + group: gateway.networking.k8s.io + path: "/mcp" + backendRefs: + - name: atlassian + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp" + forwardHeaders: + - name: X-Atlassian-Jira-Personal-Token + - name: X-Atlassian-Jira-Url + - name: Authorization + backendHeader: X-Original-Auth # optional: rename the header + - name: github + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp" + securityPolicy: + apiKey: + secretRef: + name: github-token +``` + +Each `forwardHeaders` entry specifies: + +- `name` (required): The header to extract from the incoming client request. +- `backendHeader` (optional): A different header name to use when forwarding to the backend. If omitted, the original header name is used. + +Headers are scoped per-backend — during fan-out operations like `tools/list`, only the backends with explicit `forwardHeaders` configuration receive the forwarded headers. Other backends in the same route are unaffected. + +### OAuth Authentication + +Protect your MCP Gateway with OAuth authentication following the [MCP Authorization specification](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization): + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: MCPRoute +metadata: + name: mcp-route + namespace: default +spec: + parentRefs: + - name: aigw-run + kind: Gateway + group: gateway.networking.k8s.io + backendRefs: + - name: github + kind: Backend + group: gateway.envoyproxy.io + path: "/mcp/readonly" + securityPolicy: + oauth: + issuer: "https://keycloak.example.com/realms/master" + audiences: + - "https://api.example.com/mcp" + protectedResourceMetadata: + resource: "https://api.example.com/mcp" + scopesSupported: + - "profile" + - "email" +``` + +The OAuth flow follows the MCP specification's authorization code flow with PKCE: + +```mermaid +sequenceDiagram + participant C as MCP Client + participant G as MCP Gateway + participant A as Authorization Server + + C->>G: MCP request without token + G->>C: HTTP 401 with WWW-Authenticate header + + C->>G: Request Protected Resource Metadata + G->>C: Return OAuth metadata + + C->>A: GET /.well-known/oauth-authorization-server + A->>C: Authorization server metadata + + Note over C: Generate PKCE parameters + C->>A: Authorization request + PKCE + Note over A: User authorizes + A->>C: Authorization code + + C->>A: Token request + code_verifier + A->>C: Access token + + C->>G: MCP request with access token + G->>G: Verify token (JWKS) + Note over G: Token validated + G->>C: MCP response +``` + +### Authorization Policies + +Envoy AI Gateway supports fine-grained access control over tool access using a combination of: + +- **JWT Scopes & Claims**: Validate standard OAuth2 scopes and custom claims +- **Tool Selection**: Restrict access to specific tools +- **CEL Expressions**: Flexible, advanced matching using Common Expression Language (CEL) + +#### Configuration Structure + +Authorization is configured in the `MCPRoute` resource under `spec.securityPolicy.authorization`. + +```yaml +spec: + securityPolicy: + authorization: + rules: + - source: + jwt: + scopes: ["read"] + target: + tools: + - backend: "github" + tool: "list_issues" +``` + +#### Rule Evaluation + +Rules are evaluated in order. The first rule that matches the request (Target, Source, and CEL) determines the action (Allow/Deny). If no rules match, the `defaultAction` is applied. + +#### Matchers + +| Matcher | Description | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Target** | Matches specific tools. Can filter by `backend` and `tool` name. | +| **Source** | Matches JWT properties.
`scopes`: List of required scopes (all must be present).
`claims`: Key-value pairs. Arrays in claims match if _any_ value matches. | +| **CEL** | Advanced expression evaluated against the request context. | + +#### CEL Context + +The following variables are available in CEL expressions: + +| Variable | Description | +| --------------------- | ---------------------------------------------- | +| `request.method` | HTTP method (e.g., "POST") | +| `request.host` | Host header value | +| `request.path` | URL path | +| `request.headers` | Map of headers (lowercased keys, single value) | +| `request.auth.jwt` | Parsed JWT `{claims: ..., scopes: [...]}` | +| `request.mcp.method` | MCP JSON-RPC method (e.g., "tools/call") | +| `request.mcp.backend` | Target backend name | +| `request.mcp.tool` | Target tool name (for tool calls) | +| `request.mcp.params` | Parsed JSON-RPC parameters | + +#### Examples + +**Comprehensive Policy Example** + +This example demonstrates various matching strategies including token scopes, claims, tool targeting, and CEL expressions. + +```yaml +authorization: + rules: + - source: + jwt: + scopes: + - echo + target: + tools: + - backend: mcp-backend + tool: echo + cel: request.mcp.params.arguments.text.matches("^Hello, .*!$") && request.headers["x-tenant-id"] == "t-123" + - source: + jwt: + scopes: + - sum + claims: + - name: tenant + valueType: String + values: + - acme + - globex + - name: org.departments + valueType: StringArray + values: + - engineering + - development + target: + tools: + - backend: mcp-backend + tool: sum +``` + +## See Also + +- [MCP Gateway Proposal](https://github.com/envoyproxy/ai-gateway/tree/main/docs/proposals/006-mcp-gateway) - Detailed architecture and design decisions +- [MCP Specification](https://modelcontextprotocol.io/specification/2025-06-18) - Official Model Context Protocol documentation +- [MCP Example](https://github.com/envoyproxy/ai-gateway/tree/main/examples/mcp) - Complete working example +- [CLI MCP Configuration](/docs/cli/aigwrun#mcp-configuration) - Using MCP with `aigw run` standalone mode diff --git a/site/versioned_docs/version-0.7/capabilities/observability/accesslogs.md b/site/versioned_docs/version-0.7/capabilities/observability/accesslogs.md new file mode 100644 index 0000000000..e8e3bf0fe3 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/observability/accesslogs.md @@ -0,0 +1,275 @@ +--- +id: accesslogs +title: Access Logs With AI/LLM/MCP Metadata +sidebar_position: 8 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +It is possible to configure Envoy Access Logs to include metadata generated by the AI Gateway such as +the selected model and token consumption. This guide walks through the steps to configure the AI Gateway +Routes and MCP Routes to generate LLM token consumption and MCP metadata, and to configure the Envoy Access +Logs to include the AI/LLM/MCP details. + +## LLM Metadata in Access Logs + +Envoy AI Gateway populates information in the filter dynamic metadata under the `io.envoy.ai_gateway` namespace. +This metadata includes information about the selected model, prompt and completion token usage, and other +details about the LLM request, and can be extracted and included in the Envoy Access Logs. + +### AIGatewayRoute configuration + +The contents of the dynamic metadata are configured in the `AIGatewayRoute` resource under the `llmRequestCosts` +field. The `llmRequestCosts` field configures a list of metadata keys to populate in the Envoy filter metadata, to +be later accessed in the access logs. For example: + +```yaml +apiVersion: ai-gateway.envoyproxy.io/v1alpha1 +kind: AIGatewayRoute +metadata: + name: ai-gateway-route +spec: + (...) + llmRequestCosts: + - metadataKey: llm_input_token + type: InputToken + - metadataKey: llm_output_token + type: OutputToken + - metadataKey: llm_total_token + type: TotalToken + rules: + (...) +``` + +### EnvoyProxy Configuration + +Once the `AIGatewayRoute` is configured to populate the dynamic metadata, the Envoy Access Logs can be configured +to include the metadata in the log entries. This can be done by configuring the access logs in the `EnvoyProxy` +resource: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: ai-access-logs + namespace: default +spec: + telemetry: + accessLog: + settings: + - sinks: + - type: File + file: + path: /dev/stdout + format: + type: JSON + json: + # AI specific fields. The properties in the dynamic metadata expressions must match the ones + # defined in the AIGatewayRoute llmRequestCosts field. + gen_ai.request.model: "%REQ(X-AI-EG-MODEL)%" + gen_ai.response.model: "%DYNAMIC_METADATA(io.envoy.ai_gateway:response_model)%" + gen_ai.request.model_override: "%DYNAMIC_METADATA(io.envoy.ai_gateway:model_name_override)%" + gen_ai.provider.name: "%DYNAMIC_METADATA(io.envoy.ai_gateway:backend_name)%" + gen_ai.usage.total_tokens: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_total_token)%" + gen_ai.usage.input_tokens: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_input_token)%" + gen_ai.usage.output_tokens: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_output_token)%" + # Common fields + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + x-envoy-origin-path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + response_flags: "%RESPONSE_FLAGS%" + response_code_details: "%RESPONSE_CODE_DETAILS%" + connection_termination_details: "%CONNECTION_TERMINATION_DETAILS%" + upstream_transport_failure_reason: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + bytes_received: "%BYTES_RECEIVED%" + bytes_sent: "%BYTES_SENT%" + duration: "%DURATION%" + x-envoy-upstream-service-time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" + x-forwarded-for: "%REQ(X-FORWARDED-FOR)%" + user-agent: "%REQ(USER-AGENT)%" + x-request-id: "%REQ(X-REQUEST-ID)%" + ":authority": "%REQ(:AUTHORITY)%" + upstream_host: "%UPSTREAM_HOST%" + upstream_cluster: "%UPSTREAM_CLUSTER%" + upstream_local_address: "%UPSTREAM_LOCAL_ADDRESS%" + downstream_local_address: "%DOWNSTREAM_LOCAL_ADDRESS%" + downstream_remote_address: "%DOWNSTREAM_REMOTE_ADDRESS%" + requested_server_name: "%REQUESTED_SERVER_NAME%" + route_name: "%ROUTE_NAME%" +``` + +In this example we're adding the `gen_ai.*` properties with the values extracted from the filter metadata populated +by the AI Gateway. The `EnvoyProxy` resource must be attached to the `Gateway` object or to the `GatewayClass`. For +example: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: envoy-ai-gateway-basic + namespace: default +spec: + gatewayClassName: envoy-ai-gateway-basic + listeners: + - name: http + protocol: HTTP + port: 80 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: ai-access-logs +``` + +With this configuration, the access log entries will include the AI Gateway metadata and look like this: + +```json +{ + ":authority": "api.router.tetrate.ai", + "bytes_received": 105, + "bytes_sent": 432, + "connection_termination_details": null, + "downstream_local_address": "127.0.0.1:1975", + "downstream_remote_address": "127.0.0.1:64484", + "duration": 1526, + "gen_ai.provider.name": "default/tars/route/aigw-run/rule/0/ref/0", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.request.model_override": null, + "gen_ai.usage.input_tokens": 15, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.total_tokens": 22, + "method": "POST", + "protocol": "HTTP/1.1", + "requested_server_name": null, + "response_code": 200, + "response_code_details": "via_upstream", + "response_flags": "-", + "route_name": "httproute/default/aigw-run/rule/0/match/0/*", + "start_time": "2025-08-30T16:51:37.894Z", + "upstream_cluster": "httproute/default/aigw-run/rule/0", + "upstream_host": "34.128.142.77:443", + "upstream_local_address": "192.168.1.38:64485", + "upstream_transport_failure_reason": null, + "user-agent": "curl/8.7.1", + "x-envoy-origin-path": "/v1/chat/completions", + "x-envoy-upstream-service-time": "451", + "x-forwarded-for": "192.168.1.38", + "x-request-id": "83b63da3-45a9-4196-98b8-1a7f697d6d8d" +} +``` + +### Trying it out + +You can deploy the example to quickly try the access log configuration against a local backend: + + +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/access-log/basic.yaml`} + + +Once everything is applied you can send requests to the gateway and see the access logs in the gateway pod logs. + +## MCP Metadata in Access Logs + +Envoy AI Gateway automatically populates MCP information in the filter dynamic metadata under the `io.envoy.ai_gateway` namespace. +This metadata includes information about the request, MCP session, MCP method call, etc, and can be extracted and +included in the Envoy Access Logs. + +### EnvoyProxy Configuration + +The Envoy Access Logs can be configured to include the MCP metadata in the log entries as shown in the following `EnvoyProxy` +resource: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: ai-access-logs + namespace: default +spec: + telemetry: + accessLog: + settings: + - sinks: + - type: File + file: + path: /dev/stdout + format: + type: JSON + json: + # MCP specific fields + jsonrpc.request.id: "%DYNAMIC_METADATA(io.envoy.ai_gateway:mcp_request_id)%" + mcp.session.id: "%REQ(MCP-SESSION-ID)%" + mcp.method.name: "%DYNAMIC_METADATA(io.envoy.ai_gateway:mcp_method)%" + mcp.tool.name: "%DYNAMIC_METADATA(io.envoy.ai_gateway:mcp_tool_name)%" + mcp.provider.name: "%DYNAMIC_METADATA(io.envoy.ai_gateway:mcp_backend)%" + # Common fields + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + x-envoy-origin-path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + response_flags: "%RESPONSE_FLAGS%" + response_code_details: "%RESPONSE_CODE_DETAILS%" + connection_termination_details: "%CONNECTION_TERMINATION_DETAILS%" + upstream_transport_failure_reason: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + bytes_received: "%BYTES_RECEIVED%" + bytes_sent: "%BYTES_SENT%" + duration: "%DURATION%" + x-envoy-upstream-service-time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" + x-forwarded-for: "%REQ(X-FORWARDED-FOR)%" + user-agent: "%REQ(USER-AGENT)%" + x-request-id: "%REQ(X-REQUEST-ID)%" + ":authority": "%REQ(:AUTHORITY)%" + upstream_host: "%UPSTREAM_HOST%" + upstream_cluster: "%UPSTREAM_CLUSTER%" + upstream_local_address: "%UPSTREAM_LOCAL_ADDRESS%" + downstream_local_address: "%DOWNSTREAM_LOCAL_ADDRESS%" + downstream_remote_address: "%DOWNSTREAM_REMOTE_ADDRESS%" + requested_server_name: "%REQUESTED_SERVER_NAME%" + route_name: "%ROUTE_NAME%" +``` + +In this example we're adding the `mcp_*` properties with the values extracted from the filter metadata populated +by the AI Gateway. The `EnvoyProxy` resource must be attached to the `Gateway` object or to the `GatewayClass` as +explained in the previous section. + +With this configuration, the access log entries will include the AI Gateway metadata and look like this: + +```json +{ + "bytes_received": 165, + "bytes_sent": 10255, + "connection_termination_details": null, + "downstream_local_address": "127.0.0.1:10088", + "downstream_remote_address": "127.0.0.1:58124", + "duration": 653, + "gen_ai.provider.name": null, + "gen_ai.request.model": null, + "gen_ai.request.model_override": null, + "gen_ai.usage.input_tokens": null, + "gen_ai.usage.output_tokens": null, + "mcp.provider.name": "context7", + "mcp.method.name": "tools/call", + "jsonrpc.request.id": "3", + "mcp.session.id": null, + "method": "POST", + "response_code": 200, + "start_time": "2025-10-08T10:42:11.430Z", + "upstream_cluster": "httproute/default/ai-eg-mcp-br-mcp-route-context7/rule/0", + "upstream_host": "13.217.64.66:443", + "upstream_local_address": "192.168.40.127:58172", + "upstream_transport_failure_reason": null, + "user-agent": "Go-http-client/1.1", + "x-envoy-origin-path": "/mcp", + "x-request-id": "cf2c0454-6290-498e-af6c-e32a611b5da1" +} +``` + +### Trying it out + +You can deploy the example to quickly try the access log configuration by following the +[MCP examples](https://github.com/envoyproxy/ai-gateway/tree/main/examples/mcp) and use any of the +provided configurations. diff --git a/site/versioned_docs/version-0.7/capabilities/observability/index.md b/site/versioned_docs/version-0.7/capabilities/observability/index.md new file mode 100644 index 0000000000..3b528f51f3 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/observability/index.md @@ -0,0 +1,16 @@ +--- +id: observability +title: Observability +--- + +Envoy AI Gateway extends the capabilities of Envoy Gateway, and as you run Envoy Gateway you have access to the foundational observability in the Envoy Gateway system. +We recommend you familiarize yourself with the [Envoy Gateway Observability Documentation](https://gateway.envoyproxy.io/docs/tasks/observability/). + +## AI/LLM Observability Features + +The Envoy AI Gateway provides specialized observability capabilities for AI and LLM workloads: + +- **[GenAI Metrics](./metrics.md)** - Prometheus metrics following OpenTelemetry Gen AI semantic conventions for monitoring token usage, latency, and model performance. +- **[GenAI Tracing](./tracing.md)** - OpenTelemetry integration with OpenInference semantic conventions for LLM request tracing and evaluation. +- **[Access Logs with AI/LLM metadata](./accesslogs.md)** - AI metadata produced by the AI gateway (model name, token usage, etc.) can be included in the Envoy Access Logs. +- **[Gateway Configuration](../gateway-config.md)** - Per-gateway configuration of the external processor container, including environment variables for tracing and resource requirements. diff --git a/site/versioned_docs/version-0.7/capabilities/observability/metrics.md b/site/versioned_docs/version-0.7/capabilities/observability/metrics.md new file mode 100644 index 0000000000..171b74383e --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/observability/metrics.md @@ -0,0 +1,121 @@ +--- +id: metrics +title: AI/LLM Metrics +sidebar_position: 6 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +When using the Envoy AI Gateway, it will collect AI specific metrics and expose them to Prometheus for monitoring by default. +This guide provides an overview of the metrics collected by the AI Gateway and how to monitor them using Prometheus. + +## Overview + +Envoy AI Gateway is designed to intercept and process AI/LLM requests, that enables it to collect metrics for monitoring and observability. +Currently, it collects metrics and exports them to Prometheus for monitoring in the OpenTelemetry format as specified by the [OpenTelemetry Gen AI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/attributes-registry/gen-ai/). +Not all metrics are supported yet, but the Envoy AI Gateway will continue to add more metrics in the future. + +### Supported Endpoints + +Metrics are collected for the following LLM endpoints: + +- **`/v1/chat/completions`** - Chat completions (streaming and non-streaming) +- **`/v1/completions`** - Legacy text completions (streaming and non-streaming) +- **`/v1/embeddings`** - Text embeddings +- **`/cohere/v2/rerank`** - Rerank +- **`/anthropic/v1/messages`** - Anthropic messages (streaming and non-streaming) + +For example, the Envoy AI Gateway collects metrics such as: + +- [**`gen_ai.client.token.usage`**](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiclienttokenusage): Number of tokens processed. The attribute `gen_ai.token.type` can be used to differentiate between input, output, and total tokens. +- [**`gen_ai.server.request.duration`**](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiserverrequestduration): Measured from the start of the received request headers in the Envoy AI Gateway filter to the end of the processed response body processing. +- [**`gen_ai.server.time_to_first_token`**](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiservertime_to_first_token): Measured from the start of the received request headers in the Envoy AI Gateway filter to the receiving of the first token in the response body handling. +- [**`gen_ai.server.time_per_output_token`**](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiservertime_per_output_token): The latency between consecutive tokens, if supported, or by chunks/tokens otherwise. + +Each metric comes with some default attributes such as: + +- `gen_ai.operation.name` + - `chat`: For `/v1/chat/completions` endpoint. + - `completion`: For `/v1/completions` endpoint. + - `embedding`: For `/v1/embeddings` endpoint. + - `rerank`: For `/cohere/v2/rerank` endpoint. + - `image_generation`: For `/v1/images/generations` endpoint. + - `messages`: For `/anthropic/v1/messages` endpoint. +- `gen_ai.original.model` - The original model name from the request body +- `gen_ai.request.model` - The model name requested (may be overridden) +- `gen_ai.response.model` - The model name returned in the response +- `gen_ai.provider.name` - The provider name (e.g., `openai`, `anthropic`) + +:::tip + +You can enrich the metrics with custom labels extracted from HTTP request headers. Use `controller.requestHeaderAttributes` for a base mapping shared with spans and access logs, and `controller.metricsRequestHeaderAttributes` for metrics-only mappings. Metrics never default to `session.id` because it is high-cardinality. See [values.yaml](https://github.com/envoyproxy/ai-gateway/blob/main/manifests/charts/ai-gateway-helm/values.yaml) for more details including other configurations. + +::: + +## Trying it out + +Before you begin, you'll need to complete the basic setup from the [Basic Usage](/docs/getting-started/basic-usage) guide. + +Then, you can install the prometheus using the following commands: + + +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/monitoring/monitoring.yaml`} + + +Let's wait for a while until the Prometheus is up and running. + +```shell +kubectl wait --for=condition=ready pod -l app=prometheus -n monitoring +``` + +To access the Prometheus dashboard, you need to port-forward the Prometheus service to your local machine like this: + +```shell +kubectl port-forward -n monitoring svc/prometheus 9090:9090 +``` + +Now open your browser and navigate to `http://localhost:9090` to access the Prometheus dashboard to explore the metrics. + +Alternatively, you can make the following requests to see the raw metrics: + +```shell +curl http://localhost:9090/api/v1/query --data-urlencode \ + 'query=sum(gen_ai_client_token_usage_sum{gateway_envoyproxy_io_owning_gateway_name = "envoy-ai-gateway-basic"}) by (gen_ai_request_model, gen_ai_token_type)' \ + | jq '.data.result[]' +``` + +and then you would get the response like this, assuming you have made some requests with the model `gpt-4o-mini`: + +```json lines +{ + "metric": { + "gen_ai_request_model": "gpt-4o-mini", + "gen_ai_token_type": "input" + }, + "value": [ + 1743105857.684, + "12" + ] +} +{ + "metric": { + "gen_ai_request_model": "gpt-4o-mini", + "gen_ai_token_type": "output" + }, + "value": [ + 1743105857.684, + "13" + ] +} +{ + "metric": { + "gen_ai_request_model": "gpt-4o-mini", + "gen_ai_token_type": "total" + }, + "value": [ + 1743105857.684, + "25" + ] +} +``` diff --git a/site/versioned_docs/version-0.7/capabilities/observability/tracing.md b/site/versioned_docs/version-0.7/capabilities/observability/tracing.md new file mode 100644 index 0000000000..443e0240a0 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/observability/tracing.md @@ -0,0 +1,261 @@ +--- +id: tracing +title: GenAI Distributed Tracing +sidebar_position: 7 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +Envoy AI Gateway's router joins and records distributed traces when supplied +with an [OpenTelemetry](https://opentelemetry.io/) collector endpoint. + +This guide provides an overview of the spans recorded by the AI Gateway and how +export them to your choice of OpenTelemetry collector. + +## Overview + +Envoy AI Gateway's router joins and records distributed traces when supplied +with an [OpenTelemetry](https://opentelemetry.io/) collector endpoint. + +Requests to the OpenAI Chat Completions, Completions (legacy), and Embeddings +endpoints are recorded as Spans which include typical timing and request +details. In addition, there are GenAI attributes representing the LLM or +Embeddings call including full request and response details, defined by +[OpenInference semantic conventions][openinference]. + +OpenInference attributes default to include full request and response data for +both chat completions and embeddings. This can be toggled with configuration, +but when enabled allows systems like [Arize Phoenix][phoenix] to perform +evaluations of production requests captured in OpenTelemetry spans. + +For chat completions, this includes traditional LLM metrics such as correctness +and hallucination detection. For embeddings, it enables agentic RAG evaluations +focused on retrieval and semantic analysis. + +## Trying it out + +Before you begin, you'll need to complete the basic setup from the +[Basic Usage](/docs/getting-started/basic-usage) guide, which includes +installing Envoy Gateway and AI Gateway. + +### Install Phoenix for LLM observability + +```shell +# Install Phoenix using PostgreSQL storage. +helm install phoenix oci://registry-1.docker.io/arizephoenix/phoenix-helm \ + --namespace envoy-ai-gateway-system \ + --set auth.enableAuth=false \ + --set server.port=6006 + +# Wait for Phoenix to be ready (first run may take a few minutes to pull images) +kubectl wait --timeout=5m -n envoy-ai-gateway-system \ + pods -l app=phoenix --for=condition=Ready +``` + +### Configure AI Gateway with OpenTelemetry + +Upgrade your AI Gateway installation with [OpenTelemetry configuration][otel-config]: + + +{`helm upgrade aieg oci://docker.io/envoyproxy/ai-gateway-helm \\ + --version v${vars.aigwVersion} \\ + --namespace envoy-ai-gateway-system \\ + --set "extProc.extraEnvVars[0].name=OTEL_EXPORTER_OTLP_ENDPOINT" \\ + --set "extProc.extraEnvVars[0].value=http://phoenix-svc.envoy-ai-gateway-system:6006" \\ + --set "extProc.extraEnvVars[1].name=OTEL_METRICS_EXPORTER" \\ + --set "extProc.extraEnvVars[1].value=none" +# OTEL_SERVICE_NAME defaults to "ai-gateway" if not set +# OTEL_METRICS_EXPORTER=none because Phoenix only supports traces, not metrics +# Note: Use fully-qualified service name because ext-proc runs in envoy-gateway-system namespace`} + + +Wait for the gateway pod to be ready: + +```shell +kubectl wait --for=condition=Ready -n envoy-gateway-system \ + pods -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic +``` + +### Generate traces + +Make requests to your AI Gateway to generate traces. Follow the instructions +from [Testing the Gateway](/docs/getting-started/basic-usage#testing-the-gateway) +in the Basic Usage guide to: + +1. Set up port forwarding (if needed) +2. Test the chat completions endpoint + +Each request will generate traces that are sent to Phoenix. + +### Check Phoenix is receiving traces + +```bash +kubectl logs -n envoy-ai-gateway-system deployment/phoenix | grep "POST /v1/traces" +``` + +### Access Phoenix UI + +Port-forward to access the Phoenix dashboard: + +```shell +kubectl port-forward -n envoy-ai-gateway-system svc/phoenix-svc 6006:6006 +``` + +Then open http://localhost:6006 in your browser to explore the traces. + +## Privacy Configuration + +Control sensitive data in traces by adding +[OpenInference configuration][openinference-config] to Helm values when you +reconfigure the AI Gateway. There is [similar config][openinference-embeddings] +for embeddings: + +For example, if you are using a `values.yaml` file instead of command line +arguments, you can add the following to control redaction: + +```yaml +extProc: + extraEnvVars: + # Base OTEL configuration... + # Hide sensitive data (all default to false) + - name: OPENINFERENCE_HIDE_INPUTS + value: "true" # Hide input messages to the LLM + - name: OPENINFERENCE_HIDE_OUTPUTS + value: "true" # Hide output messages from the LLM + # Reduce volume for embeddings (all default to false) + - name: OPENINFERENCE_HIDE_EMBEDDINGS_TEXT + value: "true" # Hide embeddings input + - name: OPENINFERENCE_HIDE_EMBEDDINGS_VECTORS + value: "true" # Hide embeddings output +``` + +Note: Hiding inputs/outputs prevents human or LLM-as-a-Judge evaluation of your +LLM requests, such as done with the [Phoenix Evals library][phoenix-evals]. + +## Session Tracking + +Sessions help track and organize related traces across multi-turn conversations +with your AI app. Maintaining context between interactions is key for +observability. + +With sessions, you can: + +- Track a conversation's full history in one thread. +- View inputs/outputs for a given agent. +- Monitor token usage and latency per conversation. + +By tagging spans with a consistent session ID, you get a connected view of +performance across a user's journey. + +The challenge is that requests to the gateway may not send traces, making +grouping difficult. Many GenAI frameworks allow you to set custom HTTP headers +when sending traffic to an LLM. Propagating sessions this way is simpler than +instrumenting applications with tracing code and can still achieve grouping. + +There's no standard name for session ID headers, but there is a common attribute +in OpenTelemetry, [session.id][otel-session], which has special handling in some +OpenTelemetry platforms such as [Phoenix][phoenix-session]. + +To bridge this gap, Envoy AI Gateway lets you map HTTP request headers to +OpenTelemetry attributes. You can define a base mapping shared by metrics, +spans, and access logs, plus optional per-signal mappings for metrics, spans, +and access logs. + +- `controller.requestHeaderAttributes` +- `controller.spanRequestHeaderAttributes` +- `controller.metricsRequestHeaderAttributes` +- `controller.logRequestHeaderAttributes` + +`controller.spanRequestHeaderAttributes` and `controller.logRequestHeaderAttributes` default to `agent-session-id:session.id` when unset (set them to an empty string to disable the default). Metrics never default to `session.id`. + +Both of these use the same value format: a comma-separated list of +`:` pairs. For example, if your session ID header +is `agent-session-id`, you can map it to the standard OpenTelemetry attribute +`session.id` like this: `agent-session-id:session.id`. + +Some metrics systems will be able to do fine-grained aggregation, but not all. +Here's an example of keeping the default session mapping for spans/logs while +only adding a low-cardinality attribute to metrics: + + +{`helm upgrade aieg oci://docker.io/envoyproxy/ai-gateway-helm \\ + --version v${vars.aigwVersion} \\ + --namespace envoy-ai-gateway-system \\ + --reuse-values \\ + --set "controller.metricsRequestHeaderAttributes=x-tenant-id:tenant.id"`} + + +## Cleanup + +To remove Phoenix and disable tracing: + + +{`# Uninstall Phoenix +helm uninstall phoenix -n envoy-ai-gateway-system + +# Disable tracing in AI Gateway + +helm upgrade aieg oci://docker.io/envoyproxy/ai-gateway-helm \\ +--version v${vars.aigwVersion} \\ +--namespace envoy-ai-gateway-system \\ +--reuse-values \\ +--unset extProc.extraEnvVars`} + + +## Per-Gateway Configuration + +For deployments with multiple Gateways that need different tracing configurations, +use the `GatewayConfig` CRD instead of global Helm values. This allows you to: + +- Configure different OTEL endpoints for different Gateways +- Set per-gateway service names for better trace organization +- Override global tracing settings for specific Gateways + +### Example + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GatewayConfig +metadata: + name: production-tracing + namespace: default +spec: + extProc: + kubernetes: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://production-collector:4317" + - name: OTEL_SERVICE_NAME + value: "ai-gateway-production" +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: production-gateway + annotations: + aigateway.envoyproxy.io/gateway-config: production-tracing +spec: + # ... +``` + +See the [Gateway Configuration](../gateway-config.md) guide for detailed information +on `GatewayConfig` usage, including environment variable precedence and shared configurations. + +## See Also + +- [Gateway Configuration](../gateway-config.md) - Per-gateway configuration using GatewayConfig +- [OpenInference Specification][openinference] - GenAI Semantic conventions for traces +- [OpenTelemetry Configuration][otel-config] - Environment variable reference +- [Arize Phoenix Documentation][phoenix] - LLM observability platform + +--- + +[openinference]: https://github.com/Arize-ai/openinference/tree/main/spec +[openinference-config]: https://github.com/Arize-ai/openinference/blob/main/spec/configuration.md +[openinference-embeddings]: https://github.com/Arize-ai/openinference/blob/main/spec/embedding_spans.md +[otel-config]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +[phoenix]: https://docs.arize.com/phoenix +[phoenix-evals]: https://arize.com/docs/phoenix/evaluation/llm-evals +[otel-session]: https://opentelemetry.io/docs/specs/semconv/registry/attributes/session/ +[phoenix-session]: https://arize.com/docs/phoenix/tracing/llm-traces/sessions diff --git a/site/versioned_docs/version-0.7/capabilities/scaling.md b/site/versioned_docs/version-0.7/capabilities/scaling.md new file mode 100644 index 0000000000..efef887b0e --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/scaling.md @@ -0,0 +1,117 @@ +--- +id: scaling +title: Scaling the AI Gateway Controller +sidebar_position: 3 +--- + +# Scaling the AI Gateway Controller + +The AI Gateway controller has two independent components with different scaling +characteristics: + +1. **Kubernetes controller** — reconciles CRDs and performs mutable operations + against the Kubernetes API (requires leader election). +2. **Extension server** — a gRPC server that handles Envoy Gateway extension + calls. It is read-only with respect to the Kubernetes API and scales + horizontally. + +Because Envoy Gateway's xDS server communicates with all Envoy instances, and +each EG replica in turn calls the AIGW extension server, the extension server +can become a bottleneck under load. Running multiple AIGW controller replicas +distributes this load across pods. + +:::note +Leader election applies only to the Kubernetes controller portion. The +extension server starts and serves traffic on every replica, including +non-leader pods, so all replicas handle extension server requests from the +moment they are ready. +::: + +## Recommended Configuration + +### Helm + +Set `controller.replicaCount` and ensure `controller.leaderElection.enabled` +is `true` (the default): + +```yaml +controller: + replicaCount: 2 + leaderElection: + enabled: true +``` + +Apply with: + +```shell +helm upgrade --install ai-gateway-helm \ + oci://docker.io/envoyproxy/ai-gateway-helm \ + --namespace envoy-ai-gateway-system \ + --values values.yaml +``` + +### Kubernetes Deployment (without Helm) + +Edit the controller `Deployment` directly: + +```shell +kubectl -n envoy-ai-gateway-system scale deployment ai-gateway-controller \ + --replicas=2 +``` + +Or patch the `Deployment` manifest: + +```yaml +spec: + replicas: 2 +``` + +## Horizontal Pod Autoscaler + +For dynamic workloads you can pair the above with an HPA. The extension server +is CPU-bound, so CPU utilization is a reasonable metric: + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ai-gateway-controller + namespace: envoy-ai-gateway-system +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ai-gateway-controller + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 +``` + +## Resource Requests and Limits + +Size each replica according to the number of Envoy instances it will serve. +A reasonable starting point: + +```yaml +controller: + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" +``` + +Adjust based on observed usage in your environment. + +## See Also + +- [Gateway Configuration](./gateway-config.md) — per-gateway ext_proc settings +- [Observability](./observability/) — metrics to monitor controller load diff --git a/site/versioned_docs/version-0.7/capabilities/security/index.md b/site/versioned_docs/version-0.7/capabilities/security/index.md new file mode 100644 index 0000000000..8c5d0bdc0f --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/security/index.md @@ -0,0 +1,32 @@ +--- +id: security +title: Security +--- + +# Security + +As Envoy AI Gateway is built on Envoy Gateway, you can leverage the Envoy Gateway Security Policy by attaching them to the Gateway and/or generated HTTPRoutes. + +:::tip +View all **[Envoy Gateway Security Docs](https://gateway.envoyproxy.io/docs/tasks/security/)** to learn more what security configurations are available to you. +::: + +## Common Security Docs + +Below are a list of common security configurations that can be useful when securing your gateway leveraging Envoy Gateway configurations. + +### Access Control + +- [JWT Validation](https://gateway.envoyproxy.io/docs/tasks/security/jwt-authentication/) - _Validate signed JWT tokens_ +- [JWT Claim Based Authorization](https://gateway.envoyproxy.io/docs/tasks/security/jwt-claim-authorization/) - _Assert values of claims in JWT tokens_ +- [Mutual TLS](https://gateway.envoyproxy.io/docs/tasks/security/mutual-tls/) - _Certificate based access control_ +- [Integrate with External Authorization Service](https://gateway.envoyproxy.io/docs/tasks/security/ext-auth/) - _Useful for custom logic to your business_ +- [OIDC Provider Integration](https://gateway.envoyproxy.io/docs/tasks/security/oidc/) - _When you want to have integration with a login flow for an end-user_ +- [Require Basic Auth](https://gateway.envoyproxy.io/docs/tasks/security/basic-auth/) +- [Require API Key](https://gateway.envoyproxy.io/docs/tasks/security/apikey-auth/) +- [IP Allowlist/Denylist](https://gateway.envoyproxy.io/docs/tasks/security/restrict-ip-access/) + +### TLS + +- [Setup TLS Certificate](https://gateway.envoyproxy.io/docs/tasks/security/secure-gateways/) +- [Using TLS cert-manager](https://gateway.envoyproxy.io/docs/tasks/security/tls-cert-manager/) diff --git a/site/versioned_docs/version-0.7/capabilities/security/upstream-auth.mdx b/site/versioned_docs/version-0.7/capabilities/security/upstream-auth.mdx new file mode 100644 index 0000000000..bb21de9eec --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/security/upstream-auth.mdx @@ -0,0 +1,74 @@ +--- +id: upstream-auth +title: Upstream Authentication +sidebar_position: 8 +--- +# Upstream Authentication + +## Overview + +Upstream Authentication secures the connectivity between the Gateway and the LLM Providers. This layer ensures secure communication and proper access control when the Gateway interacts with various AI service providers like AWS Bedrock, Azure OpenAI, Open AI, Mistral, Gemini, and other platforms. + +## Why Upstream Authentication Matters + +Upstream Authentication is essential for several reasons: + +1. **Security Layer**: It provides a secure authentication mechanism between the Gateway and upstream providers, preventing credentials sprawl across teams and organizations. + +1. **Credential Management**: It allows AI platform teams to handle the secure storage and management of credentials and use them in one place. This makes the revocation and rotation of credentials easier. + +1. **Compliance**: It helps organizations meet security and compliance requirements by maintaining proper authentication protocols and enforcing controlled usage of AI resources. + +## Enterprise Security Architecture + +In enterprise environments, security is implemented in multiple layers: + +1. **Client to Gateway Authentication** + - Clients must authenticate to the Gateway using appropriate methods (API keys, OAuth, etc.) aligned with the organization's security standards. + - This ensures only authorized clients can access the Gateway's services, and the Gateway enforces the access to the upstream providers and models. + + :::tip + Check out [Envoy Gateway Security Documentation](https://gateway.envoyproxy.io/docs/tasks/security/) for Client to Gateway security configuration options. + ::: + +2. **Gateway to Upstream Authentication** + - The Gateway must authenticate to upstream providers. + - This layer is managed by Upstream Authentication. + - Ensures secure communication between the Gateway and AI service providers. + +
+ ![Upstream Authentication](/diagrams/upstream-auth.png) +
+ +## Credential Management + +**Where the providers support short lived access credentials**, the Envoy AI Gateway control plane supports automated credential management with the providers' identity system. This ensures a short-lived proof of authorization, such as an access token, is used when the request is sent to the upstream service. + +**For providers that support long lived access credentials**, the Envoy AI Gateway control plane supports a manual credential management process. In these cases the credentials, like API keys, are stored in Kubernetes secrets and managed by the administrator. + +### Automated Credential Management + +The Gateway integrates with each provider's identity system to ensure secure, short-lived authentication: + +- **AWS Bedrock**: Uses OIDC integration with AWS STS to generate temporary credentials for each request +- **Azure OpenAI**: Leverages Entra ID (formerly Azure AD) to provide short-lived access tokens +- **GCP VertexAI**: Uses GCP workload federation with Google STS to generate temporary credentials for each request + + +In both cases, the Gateway automatically manages these credentials, ensuring that each request to upstream providers is sent with short-lived credentials. This approach significantly reduces the risk of credential exposure and aligns with enterprise security best practices. + +:::info +Learn more about connecting to [AWS Bedrock](/docs/getting-started/connect-providers/aws-bedrock) and [Azure OpenAI](/docs/getting-started/connect-providers/azure-openai) in the provider specific documentation. +::: + +### Manual Credential Management + +For providers that support long lived access credentials, the Envoy AI Gateway control plane supports a manual credential management process. In these cases the credentials, like API keys, are stored in Kubernetes secrets and managed by the AI Gateway administrator. Envoy AI Gateway will use the credentials from the secret to authenticate with the upstream service, attaching them to each request by securely retrieving them from the secret and attaching them to the request. + +:::info +Learn more about connecting to [OpenAI](/docs/getting-started/connect-providers/openai) and adding your API key to the secret. You can use the same approach for other providers that support long lived credentials. +::: + +## Conclusion + +Upstream Authentication is a key component of the Envoy AI Gateway's security architecture. It ensures secure communication between the Gateway and upstream AI service providers while supporting modern authentication methods and enterprise security requirements. Leverage Envoy AI Gateway's Upstream Authentication to maintain a secure and compliant AI infrastructure in your enterprise environments. diff --git a/site/versioned_docs/version-0.7/capabilities/traffic/header-body-mutations.md b/site/versioned_docs/version-0.7/capabilities/traffic/header-body-mutations.md new file mode 100644 index 0000000000..bf05f4e6b8 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/traffic/header-body-mutations.md @@ -0,0 +1,187 @@ +--- +id: header-body-mutations +title: Header and Body Mutations +sidebar_position: 7 +--- + +# Header and Body Mutations + +Envoy AI Gateway allows you to mutate HTTP headers and JSON request body fields before requests are sent to backends. This is useful for adding provider-specific headers, setting service tiers, or removing internal fields before forwarding to upstream providers. + +## Use Cases + +- Add custom headers required by specific backends (e.g., organization IDs, API keys). +- Set JSON body fields like `service_tier`, `max_tokens`, or `temperature` per backend. +- Remove internal or sensitive fields before forwarding requests to upstream providers. +- Override default values on a per-backend or per-route basis. + +## Where Mutations Can Be Configured + +Mutations can be configured at two levels: + +- **AIServiceBackend** — Mutations defined here apply to **all requests** routed to this backend, regardless of which route matched. +- **AIGatewayRoute backendRef** — Mutations defined here apply **only when traffic matches a specific route rule** and is sent to the referenced backend. + +:::note Precedence +When both route-level and backend-level mutations are defined, **route-level takes precedence** over backend-level for conflicting operations. Non-conflicting operations from both levels are applied together. +::: + +## Header Mutations + +Use `headerMutation` to add, overwrite, or remove HTTP headers on the outgoing request. + +### Setting Headers + +The `set` field overwrites an existing header or adds it if not present. A maximum of 16 entries is allowed. + +```yaml +headerMutation: + set: + - name: "x-custom-header" + value: "my-value" +``` + +For example, given an incoming request with `my-header: foo`, the following configuration changes it to `my-header: bar`: + +```yaml +headerMutation: + set: + - name: "my-header" + value: "bar" +``` + +### Removing Headers + +The `remove` field removes the specified headers from the request. Header names are **case-insensitive** (per [RFC 2616](https://datatracker.ietf.org/doc/html/rfc2616#section-4.2)). A maximum of 16 entries is allowed. + +```yaml +headerMutation: + remove: + - "x-internal-header" + - "x-debug-header" +``` + +## Body Mutations + +Use `bodyMutation` to add, overwrite, or remove top-level JSON fields in the HTTP request body. + +:::warning +Only **top-level** fields are currently supported. Nested field paths are not available. +::: + +### Setting Body Fields + +The `set` field overwrites an existing JSON field or adds it if not present. A maximum of 16 entries is allowed. + +```yaml +bodyMutation: + set: + - path: "service_tier" + value: '"scale"' + - path: "max_tokens" + value: "4096" + - path: "temperature" + value: "0.7" +``` + +The `value` field is parsed as raw JSON. This means different value types require different formatting: + +| Type | Example `value` | Result in JSON body | +| ------- | -------------------- | ------------------- | +| String | `'"scale"'` | `"scale"` | +| Number | `"42"` | `42` | +| Boolean | `"true"` | `true` | +| Object | `'{"key": "value"}'` | `{"key": "value"}` | +| Array | `"[1, 2, 3]"` | `[1, 2, 3]` | +| Null | `"null"` | `null` | + +:::tip +String values require inner quotes. For example, to set a field to the string `"scale"`, use `'"scale"'` in YAML. Numeric and boolean values do not need inner quotes. +::: + +### Removing Body Fields + +The `remove` field removes the specified top-level fields from the request body. A maximum of 16 entries is allowed. + +```yaml +bodyMutation: + remove: + - "internal_flag" + - "debug_mode" +``` + +## Complete Examples + +### Example 1: AIServiceBackend with Mutations + +This example configures an `AIServiceBackend` that adds a custom organization header and sets `service_tier` while removing an internal tracking field for all requests routed to this backend: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIServiceBackend +metadata: + name: my-openai-backend +spec: + schema: + name: OpenAI + backendRef: + name: my-openai-backend + kind: Backend + group: gateway.envoyproxy.io + headerMutation: + set: + - name: "x-custom-org" + value: "my-org-id" + bodyMutation: + set: + - path: "service_tier" + value: '"scale"' + remove: + - "internal_tracking_id" +``` + +### Example 2: AIGatewayRoute backendRef with Mutations + +This example configures route-level mutations that apply only when requests match a specific route rule. Here, requests for `gpt-4` get a premium header and an increased `max_tokens` value: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIGatewayRoute +metadata: + name: my-route +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4 + backendRefs: + - name: my-openai-backend + headerMutation: + set: + - name: "x-route-specific" + value: "premium" + bodyMutation: + set: + - path: "max_tokens" + value: "8192" +``` + +## Precedence Rules + +:::note +When both route-level and backend-level mutations are defined for the same backend: + +- **Route-level mutations take precedence** over backend-level mutations for conflicting operations (e.g., both setting the same header name or body field path). +- **Non-conflicting operations from both levels are applied together.** For example, if the backend-level sets header `x-org` and the route-level sets header `x-tier`, both headers are added to the request. + ::: + +## References + +- [AIServiceBackend](../../api/api.mdx#aiservicebackend) +- [AIGatewayRoute](../../api/api.mdx#aigatewayroute) diff --git a/site/versioned_docs/version-0.7/capabilities/traffic/index.md b/site/versioned_docs/version-0.7/capabilities/traffic/index.md new file mode 100644 index 0000000000..5b2a7086f4 --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/traffic/index.md @@ -0,0 +1,6 @@ +--- +id: traffic +title: Traffic Handling +--- + +This section provides information about traffic routing related capabilities in Envoy AI Gateway. diff --git a/site/versioned_docs/version-0.7/capabilities/traffic/model-virtualization.md b/site/versioned_docs/version-0.7/capabilities/traffic/model-virtualization.md new file mode 100644 index 0000000000..53df0466ce --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/traffic/model-virtualization.md @@ -0,0 +1,156 @@ +--- +id: model-name-virtualization +title: Model Name Virtualization +sidebar_position: 7 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Envoy AI Gateway provides an advanced model name virtualization capability that allows you to manage and route requests to different AI models seamlessly. +This guide covers the key feature and configuration for model virtualization. + +## Motivation + +It is not uncommon for multiple AI providers to offer a similar or identical model, such as Llama-3-70b, etc. +However, each provider tends to have its own unique naming convention for the same model. +For example, `Claude 4 Sonnet` is hosted both on GCP and AWS Bedrock, but they have different model names: + +- GCP: `claude-sonnet-4@20250514`, etc. +- AWS Bedrock: `anthropic.claude-sonnet-4-20250514-v1:0` + +Even within the same provider, model names may vary based on deployment configurations or versions. +For example, an OpenAI Platform request to `gpt-5-nano` might result in a response from `gpt-5-nano-2025-08-07`. +Azure OpenAI uses deployment names in the URL path rather than model names in the request body, so you won't know the model in use until you get the response. + +From downstream GenAI applications' perspective, it is beneficial to have a unified model name that abstracts away these differences. We do this while still availing the authoritative model name that served a response. + +## Provider-specific Virtualization + +AI providers handle model naming and execution differently to balance flexibility, determinism, and optimization. This section categorizes these behaviors into virtualization types, enabling Envoy AI Gateway to abstract differences for unified downstream access while retaining executed model details where possible. + +- **Automatic Routing & Resolution**: Providers select and optimize models at runtime behind generic identifiers, returning the actual executed version for reproducibility and debugging. +- **Static Model Execution**: Uses direct mapping to immutable identifiers or fixed versions for deterministic execution, ensuring requested models run exactly as specified, ideal for consistent embeddings. +- **Deterministic Snapshot Mapping**: Resolves models via endpoint/URL with dated snapshots (no response model field), providing consistent cross-platform behavior through timestamped versions. +- **URI-Based Resolution**: Ignores request body model field, using URI path for deployment-based routing while returning the actual versioned model name in responses. +- **Third-Party Delegation**: Delegates embeddings to specialized third-party services with separate APIs, integrating ecosystem expertise while focusing on core capabilities. + +| Upstream | API Type | Virtualization Type | Request Model Example | Response Model Example | +| ---------------- | ---------------- | ------------------------------ | ----------------------------------------- | -------------------------- | +| **awsbedrock** | Chat Completions | Static Model Execution | `anthropic.claude-sonnet-4-20250514-v1:0` | N/A (no model field) | +| **awsbedrock** | Embeddings | Static Model Execution | `amazon.titan-embed-text-v2:0` | N/A (no model field) | +| **azureopenai** | Chat Completions | URI-Based Resolution | `{any-value}` (ignored) | `gpt-5-nano-2025-08-07` | +| **azureopenai** | Embeddings | URI-Based Resolution | `{any-value}` (ignored) | `text-embedding-ada-002-2` | +| **gcpanthropic** | Chat Completions | Deterministic Snapshot Mapping | `claude-sonnet-4@20250514` | N/A (no model field) | +| **gcpanthropic** | Embeddings | Third-Party Delegation | `voyage-3.5` | N/A (no model field) | +| **gcpvertexai** | Chat Completions | Deterministic Snapshot Mapping | `gemini-1.5-pro-002` | N/A (no model field) | +| **gcpvertexai** | Embeddings | Static Model Execution | `text-embedding-004` | N/A (no model field) | +| **openai** | Chat Completions | Automatic Routing & Resolution | `gpt-5-nano` | `gpt-5-nano-2025-08-07` | +| **openai** | Embeddings | Static Model Execution | `text-embedding-3-small` | `text-embedding-3-small` | + +### Azure OpenAI URI-Based Behavior + +Azure OpenAI has a unique approach where [the model field in the request JSON is completely ignored][azure-model-ignored]. The deployment name in the URI path determines which model is actually used, and the response returns the actual versioned model name: + +- **URI Format**: `https://{resource}.openai.azure.com/openai/deployments/{deployment-name}/chat/completions` +- **Request Body**: `{"model": "anything"}` ← This value is ignored +- **Response**: `{"model": "gpt-5-nano-2025-08-07"}` ← Returns the actual versioned model name + +This means [Azure OpenAI doesn't read the model field from the request JSON at all][azure-deployment-resolution] - it uses the deployment name from the URI path exclusively, but returns the real underlying versioned model identifier in the response. + +### AWS Bedrock Static Execution + +AWS Bedrock uses static model execution with immutable versioned identifiers. The Converse API does not return a model field in responses. Each model ID like `anthropic.claude-sonnet-4-20250514-v1:0` represents a specific, frozen model version with no automatic updates or routing. + +### GCP Anthropic Embeddings Delegation + +GCP Anthropic delegates embeddings to [Voyage AI's specialized embedding models][anthropic-embeddings-voyage] rather than providing Claude-native embeddings. Available models include: + +- `voyage-3.5` - Balanced performance embedding model +- `voyage-3-large` - Best quality general-purpose embeddings +- `voyage-code-3` - Optimized for code retrieval +- `voyage-law-2` - Legal domain-specific embeddings + +This delegation approach allows Anthropic to focus on language model capabilities while leveraging domain expertise for embeddings. + +### OpenAI Virtualization + +OpenAI performs automatic model routing where [generic identifiers like `gpt-5-nano` may route to different model versions][openai-model-routing] like `gpt-5-nano-2025-08-07` based on availability and optimization. + +### GCP Provider Behavior + +Both GCP Anthropic and GCP Vertex AI use endpoint-based model resolution where [the model is specified in the URI path rather than returned in the response body][gcp-endpoint-resolution]. This provides deterministic behavior with timestamped snapshots. + +## Virtualization with modelNameOverride API + +In our top level AIGatewayRoute configuration, you can specify a `modelNameOverride` inside [AIGatewayRouteBackendRef](/api/api.mdx#aigatewayrouterulebackendref) on each route rule to override the model name that is sent to the upstream AI provider. +This feature is primarily designed for scenarios where you want to dynamically change the model name based on the actual AI provider the request is being sent to. + +The example configuration looks like this: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: test-route +spec: + targetRefs: [...] + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: claude-4-sonnet + backendRefs: + - name: aws-backend + modelNameOverride: anthropic.claude-sonnet-4-20250514-v1:0 + weight: 50 + - name: gcp-backend + modelNameOverride: claude-sonnet-4@20250514 + weight: 50 +``` + +This configuration allows downstream applications to use a unified model name `claude-4-sonnet` while splitting traffic between the AWS Bedrock and GCP AI providers based on the specified `modelNameOverride`. +This is what the word "Virtualization" means in this context: abstracting away the differences in model names across different AI providers and providing a unified interface for downstream applications. +It also can be thought of as "one-to-many" aliasing of model names, where one unified model name can map to multiple different model names on different providers depending on the routing path. + +## Virtualization for fallback scenarios + +As we see in the [Provider Fallback](./provider-fallback) page, Envoy AI Gateway allows you to fallback to a different AI provider if the primary one fails. +However, sometimes we want to fallback to a different model on the same provider. +For example, it is natural to set up the Envoy AI Gateway in a way that if the primary expensive model fails (rate limit, etc), Envoy retries the request to a less expensive model on the same provider. +More concretely, if the request to `gpt-5-nano` fails, we want to retry it with `gpt-5-nano-mini` on the same OpenAI provider. + +`modelNameOverride` can also be used in this scenario to achieve the desired behavior. The configuration would look like this: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: test-route +spec: + targetRefs: [...] + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-5-nano + backendRefs: + - name: openai-backend + # This doesn't specify modelNameOverride, so it will use the default model name `gpt-5-nano` in the request. + priority: 0 + - name: openai-backend + modelNameOverride: gpt-5-nano-mini + priority: 1 +``` + +With this configuration, assuming the retry is properly configured as per the [Provider Fallback](./provider-fallback) page, if the request to `gpt-5-nano` fails, Envoy AI Gateway will automatically retry the request to `gpt-5-nano-mini` on the same OpenAI provider without requiring any changes to the downstream application. + +--- + +[azure-model-ignored]: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/chatgpt +[azure-deployment-resolution]: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/faq +[anthropic-embeddings-voyage]: https://docs.anthropic.com/en/docs/build-with-claude/embeddings +[openai-model-routing]: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/chatgpt +[gcp-endpoint-resolution]: https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude diff --git a/site/versioned_docs/version-0.7/capabilities/traffic/provider-fallback.md b/site/versioned_docs/version-0.7/capabilities/traffic/provider-fallback.md new file mode 100644 index 0000000000..7428b032ad --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/traffic/provider-fallback.md @@ -0,0 +1,112 @@ +--- +id: provider-fallback +title: Provider Fallback +sidebar_position: 6 +--- + +# Provider Fallback + +Envoy AI Gateway supports provider fallback to ensure high availability and reliability for AI/LLM workloads. With fallback, you can configure multiple upstream providers for a single route, so that if the primary provider fails (due to network errors, 5xx responses, or other health check failures), traffic is automatically routed to a healthy fallback provider. + +## When to Use Fallback + +- To ensure uninterrupted service when a primary AI/LLM provider is unavailable. +- To provide redundancy across multiple cloud or on-premise model providers. +- To implement active-active or active-passive failover strategies for critical AI workloads. + +## How Fallback Works + +- **Primary and Fallback Backends:** You can specify a prioritized list of backends in your `AIGatewayRoute` using `backendRefs`. The first backend is treated as primary, and subsequent backends are considered fallbacks. +- **Retry Policy:** Fallback is triggered based on retry policies, which can be configured using the [`BackendTrafficPolicy`](https://gateway.envoyproxy.io/contributions/design/backend-traffic-policy/) API. +- **Automatic Failover:** When the primary backend becomes unhealthy, Envoy AI Gateway automatically shifts traffic to the next healthy fallback backend. + +## Example + +Below is an example configuration that demonstrates provider fallback from a failing upstream to AWS Bedrock: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: provider-fallback + namespace: default +spec: + parentRefs: + - name: provider-fallback + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: us.meta.llama3-2-1b-instruct-v1:0 + backendRefs: + - name: provider-fallback-always-failing-upstream # Primary backend (expected to fail) + priority: 0 + - name: provider-fallback-aws # Fallback backend + priority: 1 +``` + +The corresponding `Backend` resources: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: provider-fallback-always-failing-upstream + namespace: default +spec: + endpoints: + - fqdn: + hostname: provider-fallback-always-failing-upstream.default.svc.cluster.local + port: 443 +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: provider-fallback-aws + namespace: default +spec: + endpoints: + - fqdn: + hostname: bedrock-runtime.us-east-1.amazonaws.com + port: 443 +``` + +## Configuring Fallback Behavior + +Attach a `BackendTrafficPolicy` to the generated `HTTPRoute` to control retry behavior: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: provider-fallback +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: provider-fallback # HTTPRoute is created with the same name as AIGatewayRoute + retry: + # This ensures that only one attempt is made per priority. + # For example, if the primary backend fails, it will not retry on the same backend. + numAttemptsPerPriority: 1 + numRetries: 5 + perRetry: + backOff: + baseInterval: 100ms + maxInterval: 10s + timeout: 30s + retryOn: + httpStatusCodes: + - 500 + triggers: + - connect-failure + - retriable-status-codes +``` + +## References + +- [Provider Fallback Example](https://github.com/envoyproxy/ai-gateway/tree/main/examples/provider_fallback) +- [`BackendTrafficPolicy` API Design](https://gateway.envoyproxy.io/contributions/design/backend-traffic-policy/) diff --git a/site/versioned_docs/version-0.7/capabilities/traffic/usage-based-ratelimiting.md b/site/versioned_docs/version-0.7/capabilities/traffic/usage-based-ratelimiting.md new file mode 100644 index 0000000000..13cdf155bf --- /dev/null +++ b/site/versioned_docs/version-0.7/capabilities/traffic/usage-based-ratelimiting.md @@ -0,0 +1,193 @@ +--- +id: usage-based-ratelimiting +title: Usage-based Rate Limiting +sidebar_position: 5 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +This guide focuses on AI Gateway's specific capabilities for token-based rate limiting in LLM requests. For general rate limiting concepts and configurations, refer to [Envoy Gateway's Rate Limiting documentation](https://gateway.envoyproxy.io/docs/tasks/traffic/global-rate-limit/). + +## Overview + +AI Gateway leverages Envoy Gateway's Global Rate Limit API to provide token-based rate limiting for LLM requests. Key features include: + +- Token usage tracking based on model and user identifiers +- Configuration for tracking input, output, and total token metadata from LLM responses +- Model-specific rate limiting using AI Gateway headers (`x-ai-eg-model`) which is inserted by the AI Gateway filter with the model name extracted from the request body. +- Support for custom token cost calculations using CEL expressions + +## Token Usage Behavior + +AI Gateway has specific behavior for token tracking and rate limiting: + +1. **Token Extraction**: AI Gateway automatically extracts token usage from LLM responses that follow the OpenAI schema format. The token counts are stored in the metadata specified in your `llmRequestCosts` configuration. + +2. **Rate Limit Timing**: The check for whether the total count has reached the limit happens during each request. When a request is received: + - AI Gateway checks if processing this request would exceed the configured token limit + - If the limit would be exceeded, the request is rejected with a 429 status code + - If within the limit, the request is processed and its token usage is counted towards the total + +3. **Token Types**: + - `InputToken`: Counts tokens in the request prompt + - `CachedInputToken`: Counts _cached_ input tokens in the request prompt + - `OutputToken`: Counts tokens in the model's response + - `TotalToken`: Combines both input and output tokens + - `CEL`: Allows custom token calculations using CEL expressions + +4. **Multiple Rate Limits**: You can configure multiple rate limit rules for the same user-model combination. For example: + - Limit total tokens per hour + - Separate limits for input and output tokens + - Custom limits using CEL expressions + +5. **Per Route Custom Token Calculation**: The `llmRequestCosts` defined in your `AIGatewayRoute` spec are scoped exclusively to that specific route. This means multiple `AIGatewayRoute` resources can define the exact same metadata keys, but the gateway will independently apply the correct calculations based on the route's name and namespace. + +To map the request to the correct calculation, the AI Gateway parses the route from the Envoy Gateway Metadata. If that metadata is not present, it falls back to parsing the route name from the route configuration, assuming Envoy Gateway has generated the name in the following format: `httproute///rule/`. + +:::note +For model providers with OpenAI schema transformations (like AWS Bedrock), AI Gateway automatically captures token usage through its request/response transformer. This enables consistent token tracking and rate limiting across different AI services using a unified OpenAI-compatible format. +::: + +## Configuration + +:::tip Prerequisites + +Rate limiting requires two components to be configured: + +1. **Redis Deployment**: A Redis instance must be running to store rate limit data. See the [redis.yaml example](https://github.com/envoyproxy/ai-gateway/blob/main/examples/token_ratelimit/redis.yaml) for a simple deployment. + +2. **Envoy Gateway Configuration**: Envoy Gateway must be configured at installation time to enable rate limiting and point to your Redis instance. See [Envoy Gateway Installation Guide](../../getting-started/prerequisites.md#additional-features-rate-limiting-inferencepool-etc) + +::: + +### 1. Configure Token Tracking + +AI Gateway automatically tracks token usage for each request. Configure which token counts you want to track in your `AIGatewayRoute`: + +```yaml +spec: + llmRequestCosts: + - metadataKey: llm_input_token + type: InputToken # Counts tokens in the request + - metadataKey: llm_cached_input_token + type: CachedInputToken # Counts cached input tokens in the request prompt + - metadataKey: llm_output_token + type: OutputToken # Counts tokens in the response + - metadataKey: llm_total_token + type: TotalToken # Tracks combined usage +``` + +For advanced token calculations specific to your use case: + +```yaml +spec: + llmRequestCosts: + - metadataKey: custom_cost + type: CEL + cel: "(input_tokens - cached_input_tokens) + (cached_input_tokens * 0.1) + output_tokens * 1.5" # Example: Weight cached tokens less and weight output tokens more heavily +``` + +LLMRequestCosts can be defined on a per-route level. + +### 2. Configure Rate Limits + +AI Gateway uses Envoy Gateway's Global Rate Limit API to configure rate limits. Rate limits should be defined using a combination of user and model identifiers to properly control costs at the model level. Configure this using a `BackendTrafficPolicy`: + +#### Example: Cost-Based Model Rate Limiting + +The following example demonstrates a common use case where different models have different token limits based on their costs. This is useful when: + +- You want to limit expensive models (like GPT-4) more strictly than cheaper ones +- You need to implement different quotas for different tiers of service +- You want to prevent cost overruns while still allowing flexibility with cheaper models + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: model-specific-token-limit-policy + namespace: default +spec: + targetRefs: + - name: envoy-ai-gateway-token-ratelimit + kind: Gateway + group: gateway.networking.k8s.io + rateLimit: + type: Global + global: + rules: + # Rate limit rule for GPT-4: 1000 total tokens per hour per user + # Stricter limit due to higher cost per token + - clientSelectors: + - headers: + - name: x-tenant-id + type: Distinct + - name: x-ai-eg-model + type: Exact + value: gpt-4 + limit: + requests: 1000 # 1000 total tokens per hour + unit: Hour + cost: + request: + from: Number + number: 0 # Set to 0 so only token usage counts + response: + from: Metadata + metadata: + namespace: io.envoy.ai_gateway + key: llm_total_token # Uses total tokens from the responses + # Rate limit rule for GPT-3.5: 5000 total tokens per hour per user + # Higher limit since the model is more cost-effective + - clientSelectors: + - headers: + - name: x-tenant-id + type: Distinct + - name: x-ai-eg-model + type: Exact + value: gpt-3.5-turbo + limit: + requests: 5000 # 5000 total tokens per hour (higher limit for less expensive model) + unit: Hour + cost: + request: + from: Number + number: 0 # Set to 0 so only token usage counts + response: + from: Metadata + metadata: + namespace: io.envoy.ai_gateway + key: llm_total_token # Uses total tokens from the response +``` + +:::warning +When configuring rate limits: + +1. Always set the request cost number to 0 to ensure only token usage counts towards the limit +2. Set appropriate limits for different models based on their costs and capabilities +3. Ensure both user and model identifiers are used in rate limiting rules + ::: + +## Making Requests + +For proper cost control and rate limiting, requests must include: + +- `x-tenant-id`: Identifies the user making the request + +Example request: + +```shell +curl -H "Content-Type: application/json" \ + -H "x-tenant-id: user123" \ + -d '{ + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` diff --git a/site/versioned_docs/version-0.7/cli/index.md b/site/versioned_docs/version-0.7/cli/index.md new file mode 100644 index 0000000000..29a6aaaf8c --- /dev/null +++ b/site/versioned_docs/version-0.7/cli/index.md @@ -0,0 +1,17 @@ +--- +id: cli +title: "Envoy AI Gateway CLI" +sidebar_position: 4 +--- + +# Envoy AI Gateway CLI (aigw) + +The Envoy AI Gateway CLI, `aigw`, is a command-line interface that provides a set of useful tools for using the gateway. + +:::warning +The CLI is experimental and currently under active development. +::: + +Currently, you can do the following with the `aigw` CLI: + +- **Run**: Run the Envoy AI Gateway locally as a standalone proxy with a given configuration file without any dependencies such as docker or Kubernetes. diff --git a/site/versioned_docs/version-0.7/cli/installation.md b/site/versioned_docs/version-0.7/cli/installation.md new file mode 100644 index 0000000000..d24dff1d86 --- /dev/null +++ b/site/versioned_docs/version-0.7/cli/installation.md @@ -0,0 +1,100 @@ +--- +id: aigwinstall +title: Installation +sidebar_position: 1 +--- + +## Official CLI binaries + +Each release includes the binaries for the `aigw` CLI build for different platforms.
+They can be downloaded directly from the corresponding release in the +[GitHub releases page](https://github.com/envoyproxy/ai-gateway/releases). + +## Using the Docker image + +You can also use the official Docker images to run the CLI without installing it locally. +The CLI images are available at: https://hub.docker.com/r/envoyproxy/ai-gateway-cli/tags + +To run the CLI using Docker, you only need to expose the port where the standalone `aigw` listens to +and configure the environment variables for the credentials. If you want to use a custom configuration file, +you can mount it as a volume. + +The following example runs the AI Gateway with the default configuration for the [OpenAI provider](../getting-started/connect-providers/openai.md): + +```shell +docker run --rm -p 1975:1975 -e OPENAI_API_KEY=OPENAI_API_KEY envoyproxy/ai-gateway-cli run +``` + +## Building the latest version + +To use the latest version, you can use the following commands to clone the repo and build the CLI: + +```shell +git clone https://github.com/envoyproxy/ai-gateway.git +cd ai-gateway +go install ./cmd/aigw +``` + +:::tip +`go install` command installs a binary in the `$(go env GOPATH)/bin` directory. +Make sure that the `$(go env GOPATH)/bin` directory is in your `PATH` environment variable. + +For example, you can add the following line to your shell profile (e.g., `~/.bashrc`, `~/.zshrc`, etc.): + +```sh +export PATH=$PATH:$(go env GOPATH)/bin +``` + +::: + +Now, you can check if the installation was successful by running the following command: + +```sh +aigw --help +``` + +This will display the help message for the `aigw` CLI like this: + +``` +Usage: aigw [flags] + +Envoy AI Gateway CLI + +Flags: + -h, --help Show context-sensitive help. + +Commands: + version [flags] + Show version. + + run [] [flags] + Run the AI Gateway locally for given configuration. + +Run "aigw --help" for more information on a command. +``` + +## Configuration + +The [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) defines standard locations for user-specific files: + +- **Config files**: User-specific configuration (persistent, shared) +- **Data files**: Downloaded binaries (persistent, shared) +- **State files**: Logs and configs per run (persistent, debugging) +- **Runtime files**: Ephemeral files like sockets (deleted on reboot) + +`aigw` adopts these conventions to separate configuration, downloaded Envoy binaries, logs, and ephemeral runtime files. + +| Environment Variable | Default Path | CLI Flag | +| -------------------- | --------------------- | --------------- | +| `AIGW_CONFIG_HOME` | `~/.config/aigw` | `--config-home` | +| `AIGW_DATA_HOME` | `~/.local/share/aigw` | `--data-home` | +| `AIGW_STATE_HOME` | `~/.local/state/aigw` | `--state-home` | +| `AIGW_RUNTIME_DIR` | `/tmp/aigw-${UID}` | `--runtime-dir` | + +**Priority**: CLI flags > Environment variables > Defaults + +## What's next? + +The following sections provide more information about each of the CLI commands: + +- [aigw run](./run.md): Run the AI Gateway locally for a given configuration. diff --git a/site/versioned_docs/version-0.7/cli/run.md b/site/versioned_docs/version-0.7/cli/run.md new file mode 100644 index 0000000000..9ea338e043 --- /dev/null +++ b/site/versioned_docs/version-0.7/cli/run.md @@ -0,0 +1,370 @@ +--- +id: aigwrun +title: aigw run +sidebar_position: 2 +--- + +# `aigw run` + +## Overview + +This command runs the Envoy AI Gateway locally as a standalone proxy with a given configuration file without any dependencies such as docker or Kubernetes. +Since the project is primarily focused on the Kubernetes environment, this command is useful for testing the configuration locally before deploying it to a Kubernetes cluster. +Not only does it help in testing the configuration, but it is also useful in a local development environment of the provider-agnostic AI applications. + +:::warning +Currently, `aigw run` supports Linux and macOS. +::: + +## Quick Start + +The simplest way to get started is to have `aigw` generate a configuration for +you using environment variables. `aigw` supports auto-configuration using the same +environment variables as the OpenAI SDK, making it easy to integrate with existing +tooling. Here are some examples: + +```bash +# OpenAI +OPENAI_API_KEY=sk-your-key aigw run +# Azure OpenAI +AZURE_OPENAI_ENDPOINT=https://example.openai.azure.com \ + AZURE_OPENAI_API_KEY=your-key \ + OPENAI_API_VERSION=2024-12-01-preview \ + aigw run +# Tetrate Agent Router Service (TARS) +OPENAI_BASE_URL=https://api.router.tetrate.ai/v1 OPENAI_API_KEY=sk-your-key aigw run +# Ollama running locally +OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=unused aigw run +``` + +Now, the AI Gateway is running locally with the default configuration serving at `localhost:1975`. + +Then, open a new terminal and run the following curl commands to test the AI Gateway. +For example, use `qwen2.5:0.5b` model to route to Ollama, assuming Ollama is running locally and `ollama pull qwen2.5:0.5b` is executed to pull the model. + +```shell +curl -H "Content-Type: application/json" -XPOST http://localhost:1975/v1/chat/completions \ + -d '{"model": "qwen2.5:0.5b","messages": [{"role": "user", "content": "Say this is a test!"}]}' +``` + +### Supported Environment Variables + +The following environment variables are compatible with the OpenAI SDK: + +**OpenAI / OpenAI-compatible backends:** + +When `OPENAI_API_KEY` is set, the following environment variables are read: + +| Variable | Required | Example | Description | +| ----------------- | -------- | --------------------------- | ---------------------------------------------------- | +| `OPENAI_API_KEY` | Yes | `sk-proj-...` | API key for authentication (use "unused" for Ollama) | +| `OPENAI_BASE_URL` | No | `https://api.openai.com/v1` | Base URL of your OpenAI-compatible backend | + +**Azure OpenAI:** + +When `AZURE_OPENAI_API_KEY` is set, the following environment variables are read: + +| Variable | Required | Example | Description | +| ----------------------- | -------- | ---------------------------------- | ------------------------------------------- | +| `AZURE_OPENAI_ENDPOINT` | Yes | `https://example.openai.azure.com` | Your Azure endpoint, including the resource | +| `AZURE_OPENAI_API_KEY` | Yes | `abc123...` | API key for authentication | +| `OPENAI_API_VERSION` | Yes | `2024-12-01-preview` | Azure OpenAI API version | + +**Optional headers (both OpenAI and Azure OpenAI):** + +| Variable | Example | Description | +| ------------------- | ---------- | ---------------------------------------------------------------------------------------------- | +| `OPENAI_ORG_ID` | `org-...` | Organization ID - adds `OpenAI-Organization` request header for billing and access control | +| `OPENAI_PROJECT_ID` | `proj_...` | Project ID - adds `OpenAI-Project` request header for project-level billing and access control | + +## Custom Configuration + +To run the AI Gateway with a custom configuration, provide the path to the configuration file as an argument to the `aigw run` command. +For example, to run the AI Gateway with a custom configuration file named `config.yaml`, run the following command: + +```shell +aigw run config.yaml +``` + +The configuration uses the same API as the Envoy AI Gateway custom resources definitions. See [API Reference](../api/) for more information. + +The best way to start customizing the configuration is to start with an [example configuration](https://github.com/envoyproxy/ai-gateway/tree/main/examples) and modify it as needed. + +### Modify an Example Configuration + +First, download an example configuration to use as a starting point: + +```shell +curl -o ollama.yaml https://raw.githubusercontent.com/envoyproxy/ai-gateway/refs/heads/{vars.aigwGitRef}/examples/aigw/ollama.yaml +``` + +Next, let's say change the model matcher from `.*` (match all) to specifically match `deepseek-r1:1.5b` and save the configuration to `custom.yaml`: + +```diff +--- ollama.yaml ++++ custom.yaml +@@ -88,7 +88,7 @@ + - headers: + - type: RegularExpression + name: x-ai-eg-model +- value: .* ++ value: deepseek-r1:1.5b +``` + +You can also use environment variable substitution (`envsubst`) to allow small +changes without needing to copy a file. For example, you could use this syntax +instead, to default the model to the `CHAT_MODEL` variable. + +```diff +--- ollama.yaml ++++ custom.yaml +@@ -88,7 +88,7 @@ + - headers: + - type: RegularExpression + name: x-ai-eg-model +- value: .* ++ value: ${CHAT_MODEL:=deepseek-r1:1.5b} +``` + +### Run with the Custom Configuration + +Now, run the AI Gateway with the custom configuration by running the following command: + +```shell +aigw run custom.yaml +``` + +Now, the AI Gateway is running locally with the custom configuration serving at `localhost:1975`. + +```shell +curl -H "Content-Type: application/json" -XPOST http://localhost:1975/v1/chat/completions \ + -d '{"model": "deepseek-r1:1.5b","messages": [{"role": "user", "content": "Say this is a test!"}]}' +``` + +## MCP Configuration + +`aigw run` supports running as an [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) Gateway, allowing AI agents to connect to multiple MCP servers through a unified endpoint. The gateway aggregates tools from multiple backends, applies security policies, and provides observability for MCP traffic. + +### Using MCP Servers Configuration File + +Use the `--mcp-config` flag to load MCP server configuration from a JSON file. The format follows the canonical [MCP servers configuration](https://modelcontextprotocol.io/docs/getting-started/intro#configuring-mcp-servers) used by clients like Claude Desktop, Cursor, and VS Code: + +```json +{ + "mcpServers": { + "context7": { + "type": "http", + "url": "https://mcp.context7.com/mcp" + }, + "github": { + "type": "http", + "url": "https://api.githubcopilot.com/mcp/readonly", + "headers": { + "Authorization": "Bearer ${GITHUB_ACCESS_TOKEN}" + }, + "includeTools": ["issue_read", "list_issues"] + } + } +} +``` + +Start the MCP Gateway: + +```shell +aigw run --mcp-config mcp-servers.json +``` + +The gateway will be available at `http://localhost:1975/mcp`. Configure your MCP client to connect to this URL. + +### Using Inline JSON Configuration + +For quick testing or when the configuration is provided programmatically, use the `--mcp-json` flag: + +```shell +aigw run --mcp-json '{"mcpServers":{"context7":{"type":"http","url":"https://mcp.context7.com/mcp"}}}' +``` + +### MCP Servers Configuration Format + +Each server configuration in the `mcpServers` object supports the following properties: + +**`type`** (string, required) + +The transport protocol for the MCP server. Supported values: `"http"`, `"streamable-http"`, `"streamable_http"`, or `"streamableHttp"`. + +**`url`** (string, required) + +The full endpoint URL for the MCP server, including protocol, hostname, and path. + +**`headers`** (object, optional) + +HTTP headers to send with requests to the MCP server. Authorization headers with Bearer tokens are automatically extracted and injected as API keys. + +Example: + +```json +"headers": { + "Authorization": "Bearer ${GITHUB_TOKEN}", + "X-Custom-Header": "value" +} +``` + +Header values support environment variable substitution using `${VAR}` syntax. For example, `${GITHUB_TOKEN}` will be replaced at runtime with the value of the `GITHUB_TOKEN` environment variable. + +**`includeTools`** (array of strings, optional) + +List of specific tool names to expose from this server. If not specified, all tools from the server are exposed. + +This follows the same convention as [Gemini CLI's tool filtering](https://google-gemini.github.io/gemini-cli/docs/tools/mcp-server.html#optional). + +Example: + +```json +"includeTools": ["issue_read", "list_issues", "search_issues"] +``` + +### Testing the MCP Gateway + +Use the [MCP Inspector](https://github.com/modelcontextprotocol/inspector) to test your gateway: + +```shell +# List available tools +npx @modelcontextprotocol/inspector@0.16.8 --cli http://localhost:1975/mcp --method tools/list + +# Call a tool +npx @modelcontextprotocol/inspector@0.16.8 \ + --cli http://localhost:1975/mcp \ + --method tools/call \ + --tool-name context7__resolve-library-id \ + --tool-arg libraryName="react" +``` + +:::note +Tool names are automatically prefixed with the server name (e.g., `github__issue_read`, `context7__resolve-library-id`) to route calls to the correct backend. +::: + +### Advanced MCP Configuration + +For production deployments with advanced features like OAuth authentication, fine-grained access control, and Kubernetes integration, use the [`MCPRoute` API](/docs/capabilities/mcp/). The same configuration can be used in both `aigw run` standalone mode and Kubernetes environments. + +### Admin Endpoints + +While running, `aigw` serves admin endpoints on port `1064` by default: + +- `/metrics`: Prometheus metrics endpoint for [LLM/AI metrics](../capabilities/observability/metrics.md) +- `/health`: Health check endpoint that verifies the external processor is healthy + +## OpenTelemetry + +Envoy AI Gateway's router joins and records distributed traces when supplied +with an [OpenTelemetry](https://opentelemetry.io/) collector endpoint. + +Requests to the OpenAI Chat Completions and Embeddings endpoints are recorded +as Spans which include typical timing and request details. In addition, there +are GenAI attributes representing the LLM or Embeddings call including full +request and response details, defined by [OpenInference semantic conventions][openinference]. + +OpenInference attributes default to include full request and response data for +both chat completions and embeddings. This can be toggled with configuration, +but when enabled allows systems like [Arize Phoenix][phoenix] to perform +evaluations of production requests captured in OpenTelemetry spans. + +For chat completions, this includes traditional LLM metrics such as correctness +and hallucination detection. For embeddings, it enables agentic RAG evaluations +focused on retrieval and semantic analysis. + +### OpenTelemetry configuration + +`aigw run` supports OpenTelemetry tracing via environment variables: + +- **[OTEL SDK][otel-env]**: OTLP exporter configuration that controls span + and metrics export such as: + - `OTEL_EXPORTER_OTLP_ENDPOINT`: Collector endpoint (e.g., `http://phoenix:6006`) + - `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`: Override traces endpoint separately + - `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`: Override metrics endpoint separately + - `OTEL_TRACES_EXPORTER`: Traces exporter type (e.g., `console`, `otlp`, `none`) + - `OTEL_METRICS_EXPORTER`: Metrics exporter type (e.g., `console`, `otlp`, `none`) + - `OTEL_BSP_SCHEDULE_DELAY`: Batch span processor delay (default: 5000ms) + - `OTEL_METRIC_EXPORT_INTERVAL`: Metrics export interval (default: 60000ms) + +- **[OpenInference][openinference-config]**: Control sensitive data redaction, + such as below. There is [similar config][openinference-embeddings] for embeddings: + - `OPENINFERENCE_HIDE_INPUTS`: Hide input messages/prompts (default: `false`) + - `OPENINFERENCE_HIDE_OUTPUTS`: Hide output messages/completions (default: `false`) + - `OPENINFERENCE_HIDE_EMBEDDINGS_TEXT`: Hide embeddings input (default: `false`) + - `OPENINFERENCE_HIDE_EMBEDDINGS_VECTORS`: Hide embeddings output (default: `false`) + +- **Header Mapping**: Map HTTP request headers to span attributes, metric labels, and access log attributes. See [Session Tracking][session-tracking] for more details. + - `OTEL_AIGW_REQUEST_HEADER_ATTRIBUTES`: Base mapping applied to metrics, spans, and access logs. Example: `x-tenant-id:tenant.id` + - `OTEL_AIGW_SPAN_REQUEST_HEADER_ATTRIBUTES`: Span-only mapping (merged with base). Default: `agent-session-id:session.id` when unset (set to empty to disable). Example: `x-forwarded-proto:url.scheme` + - `OTEL_AIGW_METRICS_REQUEST_HEADER_ATTRIBUTES`: Metrics-only mapping (merged with base). Example: `x-tenant-id:tenant.id` (metrics never default to `session.id`) + - `OTEL_AIGW_LOG_REQUEST_HEADER_ATTRIBUTES`: Access-log-only mapping (merged with base). Default: `agent-session-id:session.id` when unset (set to empty to disable). Example: `x-forwarded-proto:url.scheme` + +See [docker-compose-otel.yaml][docker-compose-otel.yaml] for a complete example configuration. + +## Configuration + +### File Locations + +`aigw run` uses the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) to organize files: + +| Environment Variable | Default Path | Purpose | +| -------------------- | --------------------- | --------------------------- | +| `AIGW_CONFIG_HOME` | `~/.config/aigw` | User configuration files | +| `AIGW_DATA_HOME` | `~/.local/share/aigw` | Downloaded Envoy binaries | +| `AIGW_STATE_HOME` | `~/.local/state/aigw` | Persistent logs and configs | +| `AIGW_RUNTIME_DIR` | `/tmp/aigw-${UID}` | Ephemeral runtime files | + +See [Installation - Configuration](./installation.md#configuration) for more details about XDG directories. + +### File Mappings + +Each invocation creates a unique run identifier (`runID`) in format `YYYYMMDD_HHMMSS_UUU` to isolate concurrent runs: + +| File Type | Purpose | Path | Type | +| ------------------------- | ---------------------------------------- | ---------------------------------------------------------------- | ------- | +| Default Config | Configuration file location | `${AIGW_CONFIG_HOME}/config.yaml` | CONFIG | +| Envoy Version Preference | Selected Envoy version (via func-e) | `${AIGW_CONFIG_HOME}/envoy-version` | CONFIG | +| Envoy Binaries | Downloaded executables (via func-e) | `${AIGW_DATA_HOME}/envoy-versions/{version}/bin/envoy` | DATA | +| AIGW Logs | Gateway logs and stderr output | `${AIGW_STATE_HOME}/runs/{runID}/aigw.log` | STATE | +| Envoy Gateway Config | Generated EG configuration | `${AIGW_STATE_HOME}/runs/{runID}/envoy-gateway-config.yaml` | STATE | +| Envoy Gateway Resources | Generated EG resources (Gateway, Routes) | `${AIGW_STATE_HOME}/runs/{runID}/envoy-ai-gateway-resources/...` | STATE | +| External Processor Config | Generated extproc configuration | `${AIGW_STATE_HOME}/runs/{runID}/extproc-config.yaml` | STATE | +| Envoy Run Logs (func-e) | Envoy stdout/stderr (via func-e) | `${AIGW_STATE_HOME}/envoy-runs/{runID}/stdout.log,stderr.log` | STATE | +| UDS Socket | Unix domain socket for extproc | `${AIGW_RUNTIME_DIR}/{runID}/uds.sock` | RUNTIME | +| Admin Address (func-e) | Envoy admin endpoint (via func-e) | `${AIGW_RUNTIME_DIR}/{runID}/admin-address.txt` | RUNTIME | + +**File Categories:** + +- **CONFIG**: User-specific configuration (persistent, shared across runs) +- **DATA**: Downloaded binaries (persistent, shared across runs) +- **STATE**: Per-run logs and configs (persistent for debugging) +- **RUNTIME**: Ephemeral files like sockets (cleaned on reboot) + +### `runID` + +By default, `aigw run` generates a timestamp-based `runID` for each invocation. You can customize this for predictable paths: + +```shell +# Use run ID "0" for Docker/Kubernetes deployments +aigw run --run-id=0 + +# Or via environment variable +AIGW_RUN_ID=production aigw run +``` + +Custom run IDs: + +- Enable predictable file paths in containers +- Allow correlation across multiple runs with the same ID +- Must not contain path separators (`/` or `\`) + +--- + +[openinference]: https://github.com/Arize-ai/openinference/tree/main/spec +[phoenix]: https://docs.arize.com/phoenix +[otel-env]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +[openinference-config]: https://github.com/Arize-ai/openinference/blob/main/spec/configuration.md +[openinference-embeddings]: https://github.com/Arize-ai/openinference/blob/main/spec/embedding_spans.md +[docker-compose-otel.yaml]: https://github.com/envoyproxy/ai-gateway/blob/main/cmd/aigw/docker-compose-otel.yaml +[session-tracking]: ../capabilities/observability/tracing.md#session-tracking diff --git a/site/versioned_docs/version-0.7/compatibility.md b/site/versioned_docs/version-0.7/compatibility.md new file mode 100644 index 0000000000..ea4cc4f378 --- /dev/null +++ b/site/versioned_docs/version-0.7/compatibility.md @@ -0,0 +1,75 @@ +--- +id: compatibility +title: Compatibility Matrix +sidebar_position: 6 +--- + +# Compatibility Matrix + +This document provides compatibility information for Envoy AI Gateway releases with their dependencies. + +| AI Gateway | Envoy Gateway | Kubernetes | Gateway API | Support Status | +| ---------- | ----------------------------- | ---------- | ----------- | -------------- | +| main | v1.7.x+ (Envoy Proxy v1.37.x) | v1.32+ | v1.4.x | Development | +| v0.6.x | v1.7.x+ (Envoy Proxy v1.37.x) | v1.32+ | v1.4.x | Supported | +| v0.5.x | v1.6.x+ (Envoy Proxy v1.35.x) | v1.32+ | v1.4.x | Supported | +| others | N/A | N/A | N/A | End of Life | + +Note that "compatibility" means that these specific combinations have been tested and verified to work together. +Other versions may work but are not officially supported. +Please refer to our [Support Policy](https://github.com/envoyproxy/ai-gateway/blob/main/RELEASES.md#support-policy) for more details +on how we manage releases and support for different versions. + +To upgrade to a new Envoy AI Gateway version, make sure upgrade your dependencies accordingly to maintain compatibility, especially make sure that +Envoy Gateway and Gateway API versions are up-to-date as per the compatibility matrix above. Then, upgrade the AI Gateway using the standard helm upgrade process. + +## Upgrading and Migration + +### Helm Upgrade Commands + +```bash +# 1. Upgrade CRDs +helm upgrade aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm \ + --version \ + envoy-ai-gateway-system < NEW_VERSION > -n + +# 2. Upgrade application +helm upgrade aieg oci://docker.io/envoyproxy/ai-gateway-helm \ + --version \ + envoy-ai-gateway-system < NEW_VERSION > -n +``` + +### Migrating from v1alpha1 to v1beta1 + +AIGatewayRoute, AIServiceBackend, BackendSecurityPolicy, GatewayConfig, and MCPRoute support both v1alpha1 (deprecated) and v1beta1. When you upgrade: + +- **Existing v1alpha1 resources** continue to work. The API server can serve them via both v1alpha1 and v1beta1 endpoints. +- **Storage version migration** is not automatic. To migrate existing resources to v1beta1 storage, you must manually re-apply them or use the storage migration API. +- **New resources** should use `apiVersion: aigateway.envoyproxy.io/v1beta1`. +- **QuotaPolicy** remains v1alpha1-only (no v1beta1 available). + +#### Migrating Storage Version + +Storage version migration requires manual action. After upgrading the CRDs, existing resources remain stored in v1alpha1 format until you explicitly trigger migration. The Kubernetes API server can serve these resources via both v1alpha1 and v1beta1 endpoints, but the underlying storage version in etcd won't change automatically. + +To migrate existing resources from v1alpha1 to v1beta1 storage version: + +**Re-apply resources** (recommended for small deployments) + +Update your manifests to use v1beta1 and re-apply them with `kubectl apply`. + +For more information about storage version migration in Kubernetes, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/storage-version/#migrating-to-a-different-storage-version). + +To adopt v1beta1 in your manifests, change the `apiVersion` field and re-apply: + +```yaml +# Before (v1alpha1, deprecated) +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIGatewayRoute +# ... + +# After (v1beta1, preferred) +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +# ... +``` diff --git a/site/versioned_docs/version-0.7/concepts/architecture/control-plane.md b/site/versioned_docs/version-0.7/concepts/architecture/control-plane.md new file mode 100644 index 0000000000..145e592e4e --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/architecture/control-plane.md @@ -0,0 +1,135 @@ +--- +id: control-plane +title: Control Plane Explained +sidebar_position: 3 +--- + +# Control Plane Explained + +The control plane is responsible for configuring and managing the system. It consists of several key components working together to manage the AI Gateway configuration. + +```mermaid +graph TB + %% Define styles for different component categories + classDef aiGatewayStyle fill:#f9f,stroke:#E74C3C,color:#000 + classDef envoyGatewayStyle fill:#ddf,stroke:#none,color:#000 + classDef aiGatewayResourceStyle fill:#f9f,stroke:#E74C3C,color:#000 + classDef envoyGatewayResourceStyle fill:#ddf,stroke:#none,color:#000 + classDef generatedResourceStyle fill:#ddf,stroke:#none,color:#000 + classDef envoyProxyStyle fill:#F5CBA7,stroke:#D35400,color:#000 + classDef aiGatewayExtProcStyle fill:#f9f,stroke:#E74C3C,color:#000 + + subgraph "UserAppliedResources"["User Applied Resources"] + subgraph "AI Gateway Resources" + AIGatewayRoute:::aiGatewayResourceStyle + AIServiceBackend:::aiGatewayResourceStyle + BackendSecurityPolicy:::aiGatewayResourceStyle + end + subgraph "Envoy Gateway resources" + Gateway:::envoyGatewayResourceStyle + Backend:::envoyGatewayResourceStyle + EGDots[...]:::envoyGatewayResourceStyle + end + end + style UserAppliedResources fill:none,stroke-dasharray:5 + + AIGateway((Envoy AI Gateway + Controller)):::aiGatewayStyle + EnvoyGateway((Envoy Gateway + Controller)):::envoyGatewayStyle + subgraph "Generated Resources" + HTTPRoute:::generatedResourceStyle + HTTPRouteFilter:::generatedResourceStyle + ExtProcConfigSecret[AI Gateway ExtProc Secret]:::aiGatewayExtProcStyle + end + + subgraph "EnvoyProxyPod"[Envoy Proxy Pod] + EnvoyProxy[Envoy Proxy]:::envoyProxyStyle + AIExtProc[AI Gateway ExtProc]:::aiGatewayExtProcStyle + end + style EnvoyProxyPod fill:none,stroke:5 + + AIGatewayRoute -.->AIGateway + AIServiceBackend -.->AIGateway + BackendSecurityPolicy -.->AIGateway + Gateway -.->|Watching|AIGateway + + Gateway -.->EnvoyGateway + Backend -.->EnvoyGateway + EGDots -.->EnvoyGateway + + AIGateway -.->HTTPRoute + AIGateway -.->HTTPRouteFilter + AIGateway -.->ExtProcConfigSecret + HTTPRoute -.->EnvoyGateway + HTTPRouteFilter -.->EnvoyGateway + ExtProcConfigSecret -.->AIExtProc + + EnvoyGateway -.->|xDS/Envoy config|EnvoyProxy + AIGateway <-.->|Extension Server protocol|EnvoyGateway + AIGateway -.->|Inserting ExtProc as Sidecar|AIExtProc +``` + +## How It Works + +The control plane operates through a chain of components that work together to manage the configuration: + +1. The Envoy AI Gateway controller watches AI Gateway Custom Resources (CRs) +2. When changes are detected, it updates/generates the Envoy Gateway configuration +3. The Envoy Gateway communicates with the Envoy AI Gateway controller via the [Envoy Gateway Extension server] protocol, and the Envoy AI Gateway controller then fine-tunes the xDS configuration before the Envoy Gateway applies it to the Envoy Proxy. +4. The data plane (Envoy Proxy) processes AI traffic based on this configuration where the AI Gateway ExtProc runs as a sidecar to handle AI-specific processing. The sidecar container is inserted by the AI Gateway controller into the Envoy Proxy Pod. + +This architecture ensures a clear separation of concerns, where the AI Gateway controller focuses on AI-specific configuration while leveraging Envoy Gateway for general proxy management. + +## Components + +### 1. Envoy AI Gateway Controller + +The AI Gateway Controller manages AI-specific components and configurations: + +#### ExtProc Management + +- Creates and updates ExtProc Secrets with processing rules as well as credentials +- Inserts the AI Gateway ExtProc as a [sidecar container](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) in the Envoy Proxy Pod via the [Kubernetes Admission Webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/). The container mounts the ExtProc config secret and communicates with the Envoy Proxy to process AI traffic. + +#### Resource Management + +- Watches AI Gateway Custom Resources (CRs) +- Creates and manages `HTTPRoute` and `HTTPRouteFilter` resources +- Manages backend security policies and authentication, including the credentials rotation + +#### Integration with Envoy Gateway + +- Works alongside Envoy Gateway Controller (not directly configuring Envoy) +- Creates resources, such as `HTTPRoute`, that Envoy Gateway translates into Envoy configuration with AI-specific processing rules +- Serves the [Envoy Gateway Extension Server] to fine-tune the Envoy configuration (xDS) +- Enables token-based rate limiting through metadata + +### 2. Envoy Gateway Controller + +- Manages the core Envoy configuration through xDS +- Handles service discovery and load balancing +- Manages TLS certificates +- Translates Gateway API resources into Envoy configuration + +## Notable Rationale + +- As explained, [Envoy Gateway Extension server] is used for fine-tuning the xDS configuration to implement our features. This allows us to leverage Envoy Gateway for core proxy management while still customizing the configuration for AI-specific needs. + - For example, upstream filters (HTTP filter attached per Backend) is not supported in Envoy Gateway, so the AI Gateway controller uses the extension server to achieve this, which is essential to implement per-model priority routing. + - Another example is that backendRef level priority configuration is not supported in Gateway API or Envoy Gateway, so the AI Gateway controller uses the extension server to insert this configuration into the xDS. +- Delegating core proxy management to Envoy Gateway allows us to focus on AI-specific features and configurations, reducing duplication of effort and leveraging the strengths of both controllers. + - For example, we don't need to re-implement Gateway API resource translation, service discovery, load balancing, and TLS management. +- Inserting the ExtProc as a sidecar container brings the following benefits: + - Performance: Running the ExtProc as a sidecar within the same Pod as Envoy Proxy reduces network latency and overhead, leading to faster processing of AI traffic. Notably, the communication between Envoy Proxy and the ExtProc happens over a local Unix Domain Socket (UDS), not over the network. + - Simplified Deployment: Managing the ExtProc as part of the Envoy Proxy Pod simplifies deployment and scaling, as both components can be managed together. + - Session Affinity: Running as a sidecar naturally ensures the session affinity between Envoy Proxy and the ExtProc which is crucial for implementing provider fallbacks. As we explain in [Data Plane and Traffic Flow], we have two phases in the request processing: + Router-level and Upstream-level processing to handle provider fallbacks. Since Envoy does not support session affinity for External Processor access as of this writing, without the session affinity, the Upstream-level ExtProc might end up talking to a different ExtProc instance which does not have the context of the router level processing (mainly the original request body and headers). + +## Next Steps + +To learn more: + +- Understand the [Data Plane and Traffic Flow] + +[Envoy Gateway Extension server]: https://gateway.envoyproxy.io/docs/tasks/extensibility/extension-server/ +[Data Plane and Traffic Flow]: ./data-plane.md diff --git a/site/versioned_docs/version-0.7/concepts/architecture/data-plane.md b/site/versioned_docs/version-0.7/concepts/architecture/data-plane.md new file mode 100644 index 0000000000..767633eb92 --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/architecture/data-plane.md @@ -0,0 +1,134 @@ +--- +id: data-plane +title: Data Plane and Traffic Flow +sidebar_position: 4 +--- + +# Data Plane and Traffic Flow + +The data plane handles the actual request traffic, with the External Processor (ExtProc) playing a central role in managing AI-specific processing. + +## Components + +The data plane consists of several key components: + +### 1. Envoy Proxy + +The core proxy that handles all incoming traffic and integrates with: + +- External Processor for AI-specific processing +- Rate Limit Service for token-based rate limiting +- Various AI providers as backends + +### 2. AI Gateway External Processor + +A specialized extension service of Envoy Proxy that handles AI-specific processing needs. It performs three main functions: + +1. **Request Processing** + - Routes requests to appropriate AI providers + - Handles model selection and validation + - Manages provider-specific authentication + - Supports different API formats (OpenAI, AWS Bedrock) + +2. **Token Management** + - Tracks token usage from AI providers + - Handles both streaming and non-streaming responses + - Provides usage data for rate limiting decisions + +3. **Provider Integration** + - Transforms requests between different AI provider formats + - Normalizes responses to a consistent format + - Manages provider-specific requirements + +### 3. Rate Limit Service + +Handles token-based rate limiting by: + +- Tracking token usage across requests +- Enforcing rate limits based on token consumption +- Managing rate limit budgets + +## Request Processing Flow + +```mermaid +sequenceDiagram + participant Client as Client (OpenAI SDK) + participant Envoy as Envoy Proxy + participant RLS as Rate Limit Service + participant Processor as AI Gateway External Processor + participant Provider as AI Provider / Upstream + + Client->>Envoy: Request + Envoy->>Processor: Router-level ExtProc Request + Note over Processor: Extract Model Name + Processor-->>Envoy: ClearRouteCache; + Envoy->>RLS: Check Rate Limit + RLS-->>Envoy: ; + loop Retry/Fallback loop + Note over Envoy: Select Upstream/Endpoint + Envoy->>Processor: Upstream level ExtProc Request + Note over Processor: Request-Transform & Upstream Authnz + Processor-->>Envoy: ; + Envoy->>Provider: Forward Request + Provider-->>Envoy: Response + end + Envoy->>Processor: Process Response + Note over Processor: Response Transform & Extract Token Usage + Processor-->>Envoy: Add Usage Metadata + Envoy->>RLS: Reduce Rate Limit budget + RLS-->>Envoy: ; + Envoy->>Client: Response +``` + +The data plane processes requests through several key steps: + +### 1. Request Path + +1. **Routing**: Calculates the destination AI provider based on: + - Request path + - Headers + - Model name extracted from the request body + +2. **Request Transformation**: Prepares the request for the provider: + - Request body transformation + - Request path modification + - Format adaptation + +3. **Upstream Authorization**: Handles provider authentication: + - API key management + - Header modifications + - Authentication token handling + +4. **Token Rate Limiting Check**: Checks the request against the Rate Limit Service: + - Validates token usage + - Enforces rate limits based on configured budgets + +### 2. Response Path + +1. **Response Transformation**: + - Transforms provider response for client compatibility + - Normalizes response format + - Handles streaming responses + +2. **Token Usage Management**: + - Extracts token usage from responses + - Calculates usage based on configuration + - Stores usage in per-request dynamic metadata + - Enables rate limiting based on token consumption + +## Notable Rationale + +- Why the External Processor is separated into two phases (Router-level and Upstream-level): + - In Envoy, retry/fallback happens after the router filter at the upstream level. For example, when the upstream server returns 5xx, Envoy does not invoke the router level filter again. + Instead, it invokes only the upstream level filters. In our case, retry/fallback will make the requests to totally different AI providers. For example, on the first try, it goes to OpenAI, and on the second try, it goes to AWS Bedrock. + In this case, we need to do different request transformations and upstream authorizations. So, this logic needs to be in the upstream level filter. +- Why the External Processor? + - The External Processor is the most powerful battle-tested and production-ready extension point in Envoy. It allows us to implement complex logic without modifying Envoy's core codebase. + - [Dynamic Modules](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/advanced/dynamic_modules) could be a future alternative as it offers better performance as well as less complexity in the overall architecture. The work is tracked in [envoyproxy/ai-gateway#90](https://github.com/envoyproxy/ai-gateway/issues/90). + +## Next Steps + +To learn more: + +- Explore the [System Architecture](./system-architecture.md) +- Check out our [Getting Started](../../getting-started/index.md) guide for hands-on experience diff --git a/site/versioned_docs/version-0.7/concepts/architecture/index.md b/site/versioned_docs/version-0.7/concepts/architecture/index.md new file mode 100644 index 0000000000..a70c89fa23 --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/architecture/index.md @@ -0,0 +1,47 @@ +--- +id: architecture +title: Architecture +sidebar_position: 2 +--- + +# Architecture + +This section provides a detailed look at the architectural components of Envoy AI Gateway. Understanding the architecture will help you better deploy, configure, and maintain your gateway installation. + +## Overview + +Envoy AI Gateway follows a modern cloud-native architecture with distinct control and data planes. This separation of concerns allows for better scalability, maintainability, and flexibility in deployment options. + +Envoy AI Gateway integrates with Envoy Gateway for the control plane and Envoy Proxy for the data plane. + +## Key Concepts + +### Control Plane + +A control plane is a component that manages the configuration of the data plane. We utilize Envoy Gateway as a central control plane, and Envoy AI Gateway works in conjunction with it to manage the data plane configuration. + +### Data Plane + +The data plane is the component that sits in the request path and processes the requests. In the context of Envoy AI Gateway, the data plane consists of Envoy Proxy and the AI Gateway external processor that processes the AI requests. + +## In This Section + +1. [System Architecture Overview](./system-architecture.md) + - High-level architecture overview + - Control and data plane separation + +2. [Control Plane Explained](./control-plane.md) + - AI Gateway Controller + - Envoy Gateway Controller + - Configuration management + - Resource orchestration + +3. [Data Plane and Traffic Flow](./data-plane.md) + - External Processor functionality + - Request processing flow + +## What's Next + +After understanding the architecture: + +- Check out our [Getting Started](../../getting-started/index.md) guide for hands-on experience diff --git a/site/versioned_docs/version-0.7/concepts/architecture/system-architecture.md b/site/versioned_docs/version-0.7/concepts/architecture/system-architecture.md new file mode 100644 index 0000000000..0fda8caa84 --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/architecture/system-architecture.md @@ -0,0 +1,66 @@ +--- +id: system-architecture +title: System Architecture Overview +sidebar_position: 1 +--- + +# System Architecture Overview + +Envoy AI Gateway follows a modern cloud-native architecture pattern with distinct control and data planes. Let's explore how these components work together to manage AI traffic effectively. + +The architecture is divided into two main planes: + +1. **Control Plane**: Responsible for configuring and managing the system + - Kubernetes API Server serves as the configuration interface + - AI Gateway Controller manages AI-specific resources and configurations + - Envoy Gateway Controller handles core proxy configuration and xDS + +2. **Data Plane**: Handles the actual request traffic + - Envoy Proxy and External Processor work together to handle AI traffic + - Envoy Proxy routes and manages all incoming requests + - External Processor handles AI-specific transformations and validations + - Rate Limit Service manages token-based rate limiting + +The Control Plane configures the Data Plane through several paths: + +- AI Gateway Controller configures the External Processor as well as fine-tuning xDS configurations via Envoy Gateway extension server mechanism. +- Envoy Gateway Controller configures Envoy Proxy through xDS as well as Rate Limit Service. + +```mermaid +graph TB + subgraph "Control Plane" + K8s[Kubernetes API Server] + Controller[AI Gateway Controller] + EGController[Envoy Gateway Controller] + + Controller --> |GWAPI/EG resources|K8s + K8s -->|AI Gateway resources| Controller + K8s -->|GWAPI/EG resources| EGController + Controller <-->|xDS fine-tune| EGController + + style Controller fill:#f9f,stroke:#333,stroke-width:2px + style EGController fill:#ddf,stroke:#333 + end + + subgraph "Data Plane" + Envoy[Envoy Proxy] + ExtProc[External Processor] + RateLimit[Rate Limit Service] + + Envoy --> RateLimit + Envoy --> ExtProc + + style Envoy fill:#f9f,stroke:#333,stroke-width:2px + style ExtProc fill:#f9f,stroke:#333,stroke-width:2px + style RateLimit fill:#dfd,stroke:#333 + end + + Controller -.->|Configures| ExtProc + EGController -.->|Configures| RateLimit + EGController -.->|Updates Config| Envoy +``` + +For detailed information about specific components: + +- Learn more about the [Control Plane](./control-plane.md) components +- Understand the [Data Plane](./data-plane.md) in detail diff --git a/site/versioned_docs/version-0.7/concepts/index.md b/site/versioned_docs/version-0.7/concepts/index.md new file mode 100644 index 0000000000..0ed4a0744e --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/index.md @@ -0,0 +1,19 @@ +--- +id: concepts +title: Concepts +sidebar_position: 3 +--- + +# Concepts + +Welcome to the Concepts section of Envoy AI Gateway documentation. This section will help you understand the core components and how they work together to manage AI traffic effectively. + +## Architecture + +- [System Architecture Overview](./architecture/system-architecture.md) - High-level overview of control and data planes +- [Control Plane](./architecture/control-plane.md) - How configuration and management works +- [Data Plane](./architecture/data-plane.md) - Request processing and traffic flow + +## Resources + +- [Resources Overview](./resources.md) - Understanding the AI Gateway resources and their relationships diff --git a/site/versioned_docs/version-0.7/concepts/resources.md b/site/versioned_docs/version-0.7/concepts/resources.md new file mode 100644 index 0000000000..540ebc1819 --- /dev/null +++ b/site/versioned_docs/version-0.7/concepts/resources.md @@ -0,0 +1,55 @@ +--- +id: resources +title: Resources +sidebar_position: 2 +--- + +# Resources + +The Envoy AI Gateway uses several custom resources to manage AI traffic. Here's an overview of the key resources and how they relate to each other: + +## Resource Reference + +| Resource | Purpose | API Reference | +| --------------------- | ---------------------------------------------------- | ------------------------------------------------------------- | +| AIGatewayRoute | Defines unified API and routing rules for AI traffic | [AIGatewayRoute](../api/api.mdx#aigatewayroute) | +| AIServiceBackend | Represents individual AI service backends | [AIServiceBackend](../api/api.mdx#aiservicebackend) | +| BackendSecurityPolicy | Configures authentication for backend access | [BackendSecurityPolicy](../api/api.mdx#backendsecuritypolicy) | + +## Core Resources + +### AIGatewayRoute + +A resource that defines a unified AI API for a Gateway, allowing clients to interact with multiple AI backends using a single schema. + +- Specifies the input API schema for client requests +- Contains routing rules to direct traffic to appropriate backends +- Manages request/response transformations between different API schemas +- Can track LLM request costs (like token usage) + +### AIServiceBackend + +Represents a single AI service backend that handles traffic with a specific API schema. + +- Defines the output API schema the backend expects +- References a Kubernetes Service or Envoy Gateway Backend +- Can reference a BackendSecurityPolicy for authentication + +### BackendSecurityPolicy + +Configures authentication and authorization rules for backend access. + +- API Key authentication +- AWS credentials authentication + +## Resource Relationships + +```mermaid +graph TD + A[AIGatewayRoute] -->|references| B[AIServiceBackend] + B -->|references| C[K8s Service/Backend] + B -->|references| D[BackendSecurityPolicy] + D -->|contains| E[API Key/AWS Credentials] +``` + +The AIGatewayRoute acts as the entry point, defining how client requests are processed and routed to one or more AIServiceBackends. Each AIServiceBackend can reference a BackendSecurityPolicy, which provides the necessary credentials for accessing the underlying AI service. diff --git a/site/versioned_docs/version-0.7/getting-started/basic-usage.md b/site/versioned_docs/version-0.7/getting-started/basic-usage.md new file mode 100644 index 0000000000..45287ee42e --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/basic-usage.md @@ -0,0 +1,302 @@ +--- +id: basic-usage +title: Basic Usage +sidebar_position: 4 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import vars from '../\_vars.json'; + +This guide will help you set up a basic AI Gateway configuration and make your first request. + +For Windows users, note that you are able to use Windows Subsystem for Linux (WSL) to run the commands below if they do not work on the Windows command prompt. + +## Setting Up Your Environment + +### Deploy Basic Configuration + +Let's start by deploying a basic AI Gateway setup that includes a test backend: + + +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/basic.yaml`} + + +Wait for the Gateway pod to be ready: + +```shell +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +:::tip + +In the `basic.yaml` file, you will find "ClientTrafficPolicy" resource that configures the buffer limit on the client to the gateway connection. +By default, Envoy Gateway configures 32KB as the buffer limit, which is not enough for most AI model responses, hence it's configured to 50MB in the example. +Envoy Gateway provides more advanced policies, not limited to the buffer limit. Please refer to the [Client Traffic Policies](https://gateway.envoyproxy.io/latest/tasks/traffic/client-traffic-policy/) documentation for more details. + +**HTTP/2 window size settings:** If you are sending requests over HTTP/2, Envoy imposes additional +flow-control limits via `initialStreamWindowSize` and `initialConnectionWindowSize`. +These act as a secondary cap on request body size, independent of `bufferLimit`, and can cause +`413 request_payload_too_large` errors even after the buffer limit is raised. +To avoid this, set both values to the Envoy defaults (`16Mi` and `24Mi`) in the `ClientTrafficPolicy`: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: client-buffer-limit + namespace: default +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: envoy-ai-gateway-basic + connection: + bufferLimit: 50Mi + http2: + initialStreamWindowSize: 16Mi + initialConnectionWindowSize: 24Mi +``` + +See the Envoy API reference for +[`initial_stream_window_size`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-stream-window-size) +and +[`initial_connection_window_size`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-connection-window-size) +for more details. + +::: + +## Configure `$GATEWAY_URL` + +First, check if your Gateway has an external IP address assigned: + +```shell +kubectl get svc -n envoy-gateway-system \ + --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic +``` + +You'll see output similar to this: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +eg-envoy-ai-gateway LoadBalancer 10.96.61.234 80:31234/TCP +``` + +Choose one of these options based on the EXTERNAL-IP status: + + + + +If the EXTERNAL-IP shows an actual IP address (not ``), you can access the gateway directly: + +First, save the external IP and set the gateway URL: + +```shell +export GATEWAY_URL=$(kubectl get gateway/envoy-ai-gateway-basic -o jsonpath='{.status.addresses[0].value}') +``` + +Verify the URL is available: + +```shell +echo $GATEWAY_URL +``` + + + + +If the EXTERNAL-IP shows `` or your cluster doesn't support LoadBalancer services, use port forwarding. + +First, set the gateway URL: + +```shell +export GATEWAY_URL="http://localhost:8080" +``` + +Then set up port forwarding (this will block the terminal): + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system \ + --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -o jsonpath='{.items[0].metadata.name}') + +kubectl port-forward -n envoy-gateway-system svc/$ENVOY_SERVICE 8080:80 +``` + + + + +## Testing the Gateway + +### Making Test Requests + +Open a new terminal and send test requests to the AI Gateway using the `GATEWAY_URL` we set up: + +#### Test Chat Completions + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "some-cool-self-hosted-model", + "messages": [ + { + "role": "system", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +#### Test Completions (Legacy) + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "some-cool-self-hosted-model", + "prompt": "def fib(n):\n if n <= 1:\n return n\n else:\n return fib(n-1) + fib(n-2)", + "max_tokens": 25, + "temperature": 0.4, + "top_p": 0.9 + }' \ + $GATEWAY_URL/v1/completions +``` + +#### Test Embeddings + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "some-cool-self-hosted-model", + "input": "Hello, world!" + }' \ + $GATEWAY_URL/v1/embeddings +``` + +#### Test Rerank + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "some-cool-self-hosted-model", + "query": "What is the capital of France?", + "documents": [ + "Paris is the capital of France.", + "Berlin is the capital of Germany." + ] + }' \ + $GATEWAY_URL/cohere/v2/rerank +``` + +:::tip + +If you're opening a new terminal, you'll need to set the `GATEWAY_URL` variable again. + +::: + +### Expected Responses + +#### Chat Completions Response + +You should receive a response like: + +```json +{ + "choices": [ + { + "message": { + "content": "I'll be back." + } + } + ] +} +``` + +#### Completions Response + +For legacy completions requests, you should receive a response like: + +```json +{ + "choices": [ + { + "text": " + fib(n-3) + fib(n-4)\n\ndef fib(n):\n if n <= 1:\n return", + "index": 0, + "finish_reason": "length" + } + ] +} +``` + +#### Embeddings Response + +For embeddings requests, you should receive a response like: + +```json +{ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3, 0.4, 0.5], + "index": 0 + } + ], + "model": "some-cool-self-hosted-model", + "usage": { + "prompt_tokens": 3, + "total_tokens": 3 + } +} +``` + +#### Rerank Response + +You should receive a response like: + +```json +{ + "results": [ + { "index": 0, "relevance_score": 0.98 }, + { "index": 1, "relevance_score": 0.12 } + ] +} +``` + +:::note + +These responses come from a mock backend. The model `some-cool-self-hosted-model` is configured to return test responses. +For real AI model responses, see the [Connect Providers](./connect-providers) guide. + +::: + +### Understanding the Response Formats + +The basic setup includes a mock backend that demonstrates the API structure but doesn't provide real AI responses. The response formats follow the standard OpenAI API formats: + +**Chat Completions:** + +- A `choices` array containing responses +- Each message having a `role` and `content` + +**Embeddings:** + +- A `data` array containing embedding objects +- Each embedding having a vector array and index +- Usage information showing token consumption + +**Rerank:** + +- A `results` array containing ranked results for the provided documents +- Each result includes the original document `index` and a `relevance_score` +- Higher `relevance_score` indicates higher relevance to the query + +## Next Steps + +Now that you've tested the basic setup, you can: + +- Configure [real AI model backends](./connect-providers) like OpenAI or AWS Bedrock +- Explore the [API Reference](../api/) for more details about available endpoints diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/anthropic.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/anthropic.md new file mode 100644 index 0000000000..c679581f5a --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/anthropic.md @@ -0,0 +1,149 @@ +--- +id: anthropic +title: Connect Anthropic +sidebar_position: 6 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect Anthropic + +This guide will help you configure Envoy AI Gateway to work with Anthropic's models. + +## Prerequisites + +Before you begin, you'll need: + +- An Anthropic API key from [Anthropic's console](https://console.anthropic.com) +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview + +## Configuration Steps + +:::info Ready to proceed? +Ensure you have followed the steps in [Connect Providers](../connect-providers/) +::: + +### 1. Download configuration template + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/anthropic.yaml`} + + +### 2. Configure Anthropic Credentials + +Edit the `anthropic.yaml` file to replace the Anthropic placeholder value: + +- Find the section containing `ANTHROPIC_API_KEY` +- Replace it with your actual Anthropic API key + +:::caution Security Note +Make sure to keep your API key secure and never commit it to version control. +The key will be stored in a Kubernetes secret. +::: + +### 3. Apply Configuration + +Apply the updated configuration and wait for the Gateway pod to be ready. If you already have a Gateway running, +then the secret credential update will be picked up automatically in a few seconds. + +```shell +kubectl apply -f anthropic.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +### 4. Test the Configuration + +You should have set `$GATEWAY_URL` as part of the basic setup before connecting to providers. +See the [Basic Usage](../basic-usage.md) page for instructions. + +#### Test Chat Completions + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +## Troubleshooting + +If you encounter issues: + +1. Verify your API key is correct and active + +2. Check pod status: + + ```shell + kubectl get pods -n envoy-gateway-system + ``` + +3. View controller logs: + + ```shell + kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller + ``` + +4. View External Processor Logs + + ```shell + kubectl logs -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic -c ai-gateway-extproc + ``` + +5. Common errors: + - 401: Invalid API key + - 429: Rate limit exceeded + - 503: Anthropic service unavailable + +## Configuring More Models + +To use more models, add more [AIGatewayRouteRule]s to the `anthropic.yaml` file with the model name in the `value` field. + +For example, let's add [claude-haiku-3-5](https://docs.anthropic.com/en/docs/about-claude/models#claude-3.5-haiku) as a chat completion model: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-anthropic + namespace: default +spec: + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: claude-sonnet-4-5 + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: claude-haiku-3-5 + backendRefs: + - name: envoy-ai-gateway-basic-anthropic +``` + +## Next Steps + +After configuring Anthropic: + +- [Connect OpenAI](./openai.md) to add another provider +- [Connect AWS Bedrock](./aws-bedrock.md) to add another provider + +[AIGatewayRouteRule]: ../../api/api.mdx#aigatewayrouterule diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/aws-bedrock.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/aws-bedrock.md new file mode 100644 index 0000000000..1e8ae3ea52 --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/aws-bedrock.md @@ -0,0 +1,385 @@ +--- +id: aws-bedrock +title: Connect AWS Bedrock +sidebar_position: 3 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect AWS Bedrock + +This guide will help you configure Envoy AI Gateway to work with AWS Bedrock's foundation models, including Llama, Anthropic Claude, and other models available on AWS Bedrock. + +## Prerequisites + +- AWS credentials with access to Bedrock +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview +- Enabled model access to "Llama 3.2 1B Instruct" in the `us-east-1` region (see [AWS Bedrock Model Access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html)) + +## Authentication Methods + +Envoy AI Gateway supports the AWS SDK default credential chain, which includes: + +1. **EKS Pod Identity** - Recommended for production on EKS (v1.24+) +2. **IRSA (IAM Roles for Service Accounts)** - Recommended for production on EKS (all versions) +3. **Static Credentials** - For development, testing, or non-EKS environments + +## Setup Instructions + +### Option 1: EKS Pod Identity (Recommended for EKS 1.24+) + +EKS Pod Identity is the simplest way to grant AWS permissions to your pods without managing static credentials. + +**Step 1: Set up AWS IAM** + +Follow the official AWS documentation to configure EKS Pod Identity: + +- [EKS Pod Identity Agent installation](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html) +- [Create IAM role and policy](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-minimum-sdk.html) + +Your IAM policy needs these permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + "bedrock:ListFoundationModels", + "aws-marketplace:ViewSubscriptions" + ], + "Resource": "*" + } + ] +} +``` + +The Pod Identity association should reference: + +- **Namespace**: `envoy-gateway-system` +- **ServiceAccount**: `ai-gateway-dataplane-aws` + +**Step 2: Apply AI Gateway configuration** + + +{`kubectl apply -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/aws-pod-identity.yaml + +kubectl wait pods --timeout=2m \\ +-l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \\ +-n envoy-gateway-system --for=condition=Ready`} + + +**Step 3: Test** + +```shell +export GATEWAY_URL=$(kubectl get gateway envoy-ai-gateway-basic -n default -o jsonpath='{.status.addresses[0].value}') + +curl -H "Content-Type: application/json" -d '{ + "model": "us.meta.llama3-2-1b-instruct-v1:0", + "messages": [{"role": "user", "content": "Hello!"}] +}' http://$GATEWAY_URL/v1/chat/completions +``` + +--- + +### Option 2: IRSA (IAM Roles for Service Accounts) + +IRSA works on all EKS versions and uses OIDC federation for authentication. + +**Step 1: Set up AWS IAM** + +Follow the official AWS documentation to configure IRSA: + +- [Enable OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +- [Create IAM role with trust policy](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html) + +Your IAM policy needs the same Bedrock permissions as Pod Identity above. + +The trust policy should allow the ServiceAccount `system:serviceaccount:envoy-gateway-system:ai-gateway-dataplane-aws`. + +**Step 2: Download and configure** + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/aws-irsa.yaml`} + + +Edit `aws-irsa.yaml` and replace `ACCOUNT_ID` with your AWS account ID in the ServiceAccount annotation: + +```yaml +eks.amazonaws.com/role-arn: arn:aws:iam::YOUR_ACCOUNT_ID:role/AIGatewayBedrockRole +``` + +**Step 3: Apply and test** + +```shell +kubectl apply -f aws-irsa.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system --for=condition=Ready + +# Test (use same curl command as Pod Identity above) +export GATEWAY_URL=$(kubectl get gateway envoy-ai-gateway-basic -n default -o jsonpath='{.status.addresses[0].value}') +curl -H "Content-Type: application/json" -d '{ + "model": "us.meta.llama3-2-1b-instruct-v1:0", + "messages": [{"role": "user", "content": "Hello!"}] +}' http://$GATEWAY_URL/v1/chat/completions +``` + +--- + +### Option 3: Static Credentials + +Use AWS access key ID and secret for authentication. Suitable for development, testing, or non-EKS environments. + +:::caution Production Warning +Static credentials are not recommended for production. Use EKS Pod Identity or IRSA instead. +::: + +**Step 1: Download and configure** + +```shell +curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/{vars.aigwGitRef}/examples/basic/aws.yaml +``` + +Edit `aws.yaml` and replace the credential placeholders: + +- `AWS_ACCESS_KEY_ID`: Your AWS access key ID +- `AWS_SECRET_ACCESS_KEY`: Your AWS secret access key + +**Step 2: Apply and test** + +```shell +kubectl apply -f aws.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system --for=condition=Ready + +# Test (use same curl command as above) +export GATEWAY_URL=$(kubectl get gateway envoy-ai-gateway-basic -n default -o jsonpath='{.status.addresses[0].value}') +curl -H "Content-Type: application/json" -d '{ + "model": "us.meta.llama3-2-1b-instruct-v1:0", + "messages": [{"role": "user", "content": "Hello!"}] +}' http://$GATEWAY_URL/v1/chat/completions +``` + +You can also access an Anthropic model with native Anthropic messages endpoint: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "messages": [ + { + "role": "user", + "content": "What is capital of France?" + } + ], + "max_tokens": 100 + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +Expected output: + +```json +{ + "id": "msg_01XFDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "The capital of France is Paris." + } + ], + "model": "claude-3-5-sonnet-20241022", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 13, + "output_tokens": 8 + } +} +``` + +## Troubleshooting + +If you encounter issues: + +1. **Verify authentication is configured correctly** + - For **EKS Pod Identity**: Check Pod Identity association exists + ```shell + aws eks list-pod-identity-associations --cluster-name YOUR_CLUSTER_NAME + ``` + - For **IRSA**: Check ServiceAccount annotation + ```shell + kubectl get sa ai-gateway-dataplane-aws -n envoy-gateway-system -o yaml + ``` + - For **Static Credentials**: Verify secret exists + ```shell + kubectl get secret -n default + ``` + +2. **Check pod logs** + + ```shell + POD_NAME=$(kubectl get pod -n envoy-gateway-system \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -o jsonpath='{.items[0].metadata.name}') + kubectl logs -n envoy-gateway-system $POD_NAME -c extproc + ``` + +3. **Common errors** + - **401/403**: Invalid credentials or insufficient IAM permissions + - **404**: Model not found or not available in the specified region + - **AssumeRole errors**: Check IAM role trust policy + +For more details on AWS authentication, see: + +- [AWS SDK credential chain](https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html) +- [EKS Pod Identity documentation](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) +- [IRSA documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) + +## Configuring More Models + +To use additional models, add more [AIGatewayRouteRule]s to your configuration with the [model ID] in the `value` field. For example, to use [Claude 3 Sonnet]: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-aws + namespace: default +spec: + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: anthropic.claude-3-sonnet-20240229-v1:0 + backendRefs: + - name: envoy-ai-gateway-basic-aws +``` + +## Using Anthropic Native API + +When using Anthropic models on AWS Bedrock, you have two options: + +1. **OpenAI-compatible format** (`/v1/chat/completions`) - Works with most models but may not support all Anthropic-specific features. Note that `max_tokens` is required by the Anthropic API — if your clients do not include it, you must inject it via a request body mutation. +2. **Native Anthropic API** (`/anthropic/v1/messages`) - Provides full access to Anthropic-specific features (only for Anthropic models) + +### Streaming with Native Anthropic API + +The native Anthropic API also supports streaming responses: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "messages": [ + { + "role": "user", + "content": "Count from 1 to 5." + } + ], + "max_tokens": 100, + "stream": true + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +## Advanced Features with Anthropic Models + +Since the gateway supports the native Anthropic API, you have full access to Anthropic-specific features: + +### Extended Thinking + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "messages": [ + { + "role": "user", + "content": "Solve this puzzle: A farmer needs to cross a river with a fox, chicken, and bag of grain. The boat can only hold the farmer and one item. How does the farmer get everything across safely?" + } + ], + "max_tokens": 1000, + "thinking": { + "type": "enabled", + "budget_tokens": 5000 + } + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +### Prompt Caching + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "system": [ + { + "type": "text", + "text": "You are an AI assistant specialized in Python programming. You help users write clean, efficient Python code.", + "cache_control": {"type": "ephemeral"} + } + ], + "messages": [ + { + "role": "user", + "content": "Write a function to calculate fibonacci numbers." + } + ], + "max_tokens": 500 + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +### Tool Use (Function Calling) + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "messages": [ + { + "role": "user", + "content": "What is the weather in San Francisco?" + } + ], + "max_tokens": 500, + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + ] + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +[AIGatewayRouteRule]: ../../api/api.mdx#aigatewayrouterule +[model ID]: https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html +[Claude 3 Sonnet]: https://docs.anthropic.com/en/docs/about-claude/models#model-comparison-table diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/azure-openai.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/azure-openai.md new file mode 100644 index 0000000000..bfd3527a37 --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/azure-openai.md @@ -0,0 +1,138 @@ +--- +id: azure-openai +title: Connect Azure OpenAI +sidebar_position: 3 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect Azure OpenAI + +This guide will help you configure Envoy AI Gateway to work with Azure OpenAI's foundation models. + +There are two ways to do the [Azure OpenAI authentication](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#authentication): Microsoft Entra ID and API Key. + +We will use Microsoft Entra ID to authenticate an application to use the Azure OpenAI service. You can obtain an access token using the OAuth 2.0 client credentials grant flow. This process involves registering the application in Microsoft Entra ID (formerly Azure Active Directory), configuring the appropriate permissions, and acquiring a token from the Microsoft identity platform. The access token is then used as proof of authorization in API requests to the Azure OpenAI endpoint. + +For detailed steps, refer to the official [Microsoft documentation](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#get-a-token). + +API Key authentication is not supported yet. + +## Prerequisites + +Before you begin, you'll need: + +- Azure credentials with access to OpenAI service. +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview + +## Azure Credential Setup + +1. An Azure account with OpenAI service access enabled +2. Your Azure tenant ID, client ID, and client secret key +3. Enabled model access to "GPT-4o" + +## Configuration Steps + +### 1. Download configuration template + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/azure_openai.yaml`} + + +### 2. Configure Azure Credentials + +Edit the `azure_openai.yaml` file to replace these placeholder values: + +- `AZURE_TENANT_ID`: Your Azure tenant ID +- `AZURE_CLIENT_ID`: Your Azure client ID +- `AZURE_CLIENT_SECRET`: Your Azure client secret + +:::caution Security Note +Keep your Azure credentials secure and never commit them to version control. +The credentials will be stored in Kubernetes secrets. +::: + +### 3. Apply Configuration + +Apply the updated configuration and wait for the Gateway pod to be ready. If you already have a Gateway running, then the secret credential update will be picked up automatically in a few seconds. + +```shell +kubectl apply -f azure_openai.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +### 4. Test the Configuration + +You should have set `$GATEWAY_URL` as part of the basic setup before connecting to providers. +See the [Basic Usage](../basic-usage.md) page for instructions. + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +## Troubleshoot + +If you encounter issues: + +1. Verify your Azure credentials are correct and active +2. Check pod status + +```shell +kubectl get pods -n envoy-gateway-system +``` + +3. View controller logs: + +```shell +kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller +``` + +4. Common errors: + - 401/403: Invalid credentials or insufficient permissions + - 404: Model not found or not available in region + - 429: Rate limit exceeded + +## Configuring More Models + +To use more models, add more [AIGatewayRouteRule]s to the `azure_openai.yaml` file with the [model ID] in the `value` field. For example, to use [GPT-4.5 Preview] + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-azure + namespace: default +spec: + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gpt-4.5-preview + backendRefs: + - name: envoy-ai-gateway-basic-azure +``` + +[AIGatewayRouteRule]: ../../api/api.mdx#aigatewayrouterule +[model ID]: https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models +[GPT-4.5 Preview]: https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#gpt-45-preview diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/gcp-vertexai.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/gcp-vertexai.md new file mode 100644 index 0000000000..0cefe8069a --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/gcp-vertexai.md @@ -0,0 +1,220 @@ +--- +id: gcp-vertexai +title: Connect GCP VertexAI +sidebar_position: 3 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect GCP VertexAI + +This guide will help you configure Envoy AI Gateway to work with GCP VertexAI's Gemini and Anthropic models. + +## Prerequisites + +Before you begin, you'll need: + +- GCP credentials with access to GCP VertexAI +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview + +## Authentication Options + +Envoy AI Gateway supports three authentication methods for GCP VertexAI: + +1. **Application Default Credentials (ADC)** - Recommended for GKE with Workload Identity +2. **Service Account Key Files** - For explicit JSON credentials +3. **Workload Identity Federation** - For cross-cloud authentication + +### Option 1: Application Default Credentials (Recommended for GKE) + +When running on GKE with Workload Identity, configure only `projectName` and `region`: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: BackendSecurityPolicy +metadata: + name: gcp-credentials +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: your-backend + type: GCPCredentials + gcpCredentials: + projectName: YOUR_PROJECT_NAME + region: us-central1 +``` + +ADC automatically handles credential rotation via GKE Workload Identity. + +### Option 2: Service Account Key Files + +For non-GKE environments, use a service account key file: + +1. Your GCP project id and name. +2. In your GCP project, enable VertexAI API access. +3. Create a GCP service account and generate the JSON key file. + +:::caution Security Note +Service account key files should be avoided in production when possible. Use ADC/Workload Identity instead. +::: + +## Configuration Steps + +### 1. Download configuration template + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/gcp_vertex.yaml`} + + +### 2. Configure GCP Credentials + +Edit the `gcp_vertex.yaml` file to replace these placeholder values: + +- `GCP_PROJECT_NAME`: Your GCP project name +- `GCP_REGION`: GCP region +- Update the generated service account key JSON string in the secret + +:::caution Security Note +Make sure to keep your GCP service account credentials secure and never commit them to version control. +The credentials will be stored in Kubernetes secrets. +::: + +### 3. Apply Configuration + +Apply the updated configuration and wait for the Gateway pod to be ready. If you already have a Gateway running, +then the secret credential update will be picked up automatically in a few seconds. + +```shell +kubectl apply -f gcp_vertex.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +### 4. Test the Configuration + +You should have set `$GATEWAY_URL` as part of the basic setup before connecting to providers. +See the [Basic Usage](../basic-usage.md) page for instructions. + +To access a Gemini model with chat completion endpoint: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [ + { + "role": "user", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +To access an Anthropic model with chat completion endpoint: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "claude-3-7-sonnet@20250219", + "messages": [ + { + "role": "user", + "content": "What is capital of France?" + } + ], + "max_completion_tokens": 100 + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +Expected output: + +```json +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "The capital of France is Paris. Paris is not only the capital city but also the largest city in France, known for its cultural significance, historic landmarks like the Eiffel Tower and the Louvre Museum, and its influence in fashion, art, and cuisine.", + "role": "assistant" + } + } + ], + "object": "chat.completion", + "usage": { "completion_tokens": 58, "prompt_tokens": 13, "total_tokens": 71 } +} +``` + +You can also access an Anthropic model with native Anthropic messages endpoint: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "claude-3-7-sonnet@20250219", + "messages": [ + { + "role": "user", + "content": "What is capital of France?" + } + ], + "max_tokens": 100 + }' \ + $GATEWAY_URL/anthropic/v1/messages +``` + +## Troubleshooting + +If you encounter issues: + +1. Verify your GCP credentials are correct and active +2. Check pod status: + ```shell + kubectl get pods -n envoy-gateway-system + ``` +3. View controller logs: + ```shell + kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller + ``` +4. Common errors: + - 401/403: Invalid credentials or insufficient permissions + - 404: Model not found or not available in a region + - 429: Rate limit exceeded + +## Configuring More Models + +To use more models, add more [AIGatewayRouteRule]s to the `gcp_vertex.yaml` file with the [model ID] in the `value` field. For example, to use [Claude 3 Sonnet] + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-gcp-gemini + namespace: default +spec: + schema: + name: OpenAI + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: gemini-2.5-flash-pro + backendRefs: + - name: envoy-ai-gateway-basic-gcp +``` + +[AIGatewayRouteRule]: ../../api/api.mdx#aigatewayrouterule +[model ID]: https://cloud.google.com/vertex-ai/generative-ai/docs/models +[Anthropic Claude]: https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/index.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/index.md new file mode 100644 index 0000000000..384e684207 --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/index.md @@ -0,0 +1,49 @@ +--- +id: connect-providers +title: Connect Providers +sidebar_position: 5 +--- + +# Connect Providers + +After setting up the basic AI Gateway with the mock backend, you can configure it to work with real AI model providers. This section will guide you through connecting different AI providers to your gateway. + +## Example Providers + +In this getting started guide you'll find quickstart setups to connect to the following providers: + +- [Tetrate Agent Router Service (TARS)](./tars.md) - Connect to Tetrate Agent Router Service's models +- [OpenAI](./openai.md) - Connect to OpenAI's GPT models +- [Anthropic](./anthropic.md) - Connect to Anthropic's Claude models +- [AWS Bedrock](./aws-bedrock.md) - Access AWS Bedrock's suite of foundation models +- [Azure OpenAI](./azure-openai.md) - Access Azure OpenAI's suite of foundation models +- [GCP VertexAI](./gcp-vertexai.md) - Access GCP Gemini and Anthropic models on VertexAI + +:::tip +To learn how to connect to providers see [Connecting to AI Providers](/docs/capabilities/llm-integrations/connect-providers) and you can view all [Supported Providers](/docs/capabilities/llm-integrations/supported-providers). +::: + +## Before You Begin + +Before configuring any provider, complete the [Basic Usage](../basic-usage.md) guide. + +## Security Best Practices + +When configuring AI providers, keep these security considerations in mind: + +- Store credentials securely using Kubernetes secrets +- Never commit API keys or credentials to version control +- Regularly rotate your credentials +- Use the principle of least privilege when setting up access +- Monitor usage and set up appropriate rate limits + +## Next Steps + +Choose your provider to get started: + +- [Connect Tetrate Agent Router Service (TARS)](./tars.md) +- [Connect OpenAI](./openai.md) +- [Connect Anthropic](./anthropic.md) +- [Connect AWS Bedrock](./aws-bedrock.md) +- [Connect Azure OpenAI](./azure-openai.md) +- [Connect GCP VertexAI](./gcp-vertexai.md) diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/openai.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/openai.md new file mode 100644 index 0000000000..fb303213bc --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/openai.md @@ -0,0 +1,166 @@ +--- +id: openai +title: Connect OpenAI +sidebar_position: 2 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect OpenAI + +This guide will help you configure Envoy AI Gateway to work with OpenAI's models. + +## Prerequisites + +Before you begin, you'll need: + +- An OpenAI API key from [OpenAI's platform](https://platform.openai.com) +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview + +## Configuration Steps + +:::info Ready to proceed? +Ensure you have followed the steps in [Connect Providers](../connect-providers/) +::: + +### 1. Download configuration template + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/openai.yaml`} + + +### 2. Configure OpenAI Credentials + +Edit the `openai.yaml` file to replace the OpenAI placeholder value: + +- Find the section containing `OPENAI_API_KEY` +- Replace it with your actual OpenAI API key + +:::caution Security Note +Make sure to keep your API key secure and never commit it to version control. +The key will be stored in a Kubernetes secret. +::: + +### 3. Apply Configuration + +Apply the updated configuration and wait for the Gateway pod to be ready. If you already have a Gateway running, +then the secret credential update will be picked up automatically in a few seconds. + +```shell +kubectl apply -f openai.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +### 4. Test the Configuration + +You should have set `$GATEWAY_URL` as part of the basic setup before connecting to providers. +See the [Basic Usage](../basic-usage.md) page for instructions. + +#### Test Chat Completions + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +#### Test Completions (Legacy) + +OpenAI supports the legacy completions endpoint with specific models: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "babbage-002", + "prompt": "def fib(n):\n if n <= 1:\n return n\n else:\n return fib(n-1) + fib(n-2)", + "max_tokens": 25, + "temperature": 0.4, + "top_p": 0.9 + }' \ + $GATEWAY_URL/v1/completions +``` + +## Troubleshooting + +If you encounter issues: + +1. Verify your API key is correct and active + +2. Check pod status: + + ```shell + kubectl get pods -n envoy-gateway-system + ``` + +3. View controller logs: + + ```shell + kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller + ``` + +4. View External Processor Logs + + ```shell + kubectl logs -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic -c ai-gateway-extproc + ``` + +5. Common errors: + - 401: Invalid API key + - 429: Rate limit exceeded + - 503: OpenAI service unavailable + +## Configuring More Models + +To use more models, add more [AIGatewayRouteRule]s to the `openai.yaml` file with the [model alias] in the `value` field. + +For example, let's add [o1] as a chat completion model, and [text-embedding-ada-002](https://platform.openai.com/docs/models/text-embedding-ada-002) as embedding models: + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-openai + namespace: default +spec: + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: o1 + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: text-embedding-ada-002 + backendRefs: + - name: envoy-ai-gateway-basic-openai +``` + +## Next Steps + +After configuring OpenAI: + +- [Connect AWS Bedrock](./aws-bedrock.md) to add another provider + +[AIGatewayRouteRule]: ../../api/api.mdx#aigatewayrouterule +[model alias]: https://platform.openai.com/docs/models#current-model-aliases +[o1]: https://platform.openai.com/docs/models#o1 diff --git a/site/versioned_docs/version-0.7/getting-started/connect-providers/tars.md b/site/versioned_docs/version-0.7/getting-started/connect-providers/tars.md new file mode 100644 index 0000000000..52909e18b2 --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/connect-providers/tars.md @@ -0,0 +1,124 @@ +--- +id: tars +title: Connect Tetrate Agent Router Service (TARS) +sidebar_position: 4 +--- + +import CodeBlock from '@theme/CodeBlock'; +import vars from '../../\_vars.json'; + +# Connect Tetrate Agent Router Service (TARS) + +This guide will help you configure Envoy AI Gateway to work with Tetrate Agent Router Service (TARS)'s models. + +## Prerequisites + +Before you begin, you'll need: + +- An API key from [Tetrate Agent Router Service's platform](https://router.tetrate.ai/) +- Basic setup completed from the [Basic Usage](../basic-usage.md) guide +- Basic configuration removed as described in the [Advanced Configuration](./index.md) overview + +## Configuration Steps + +:::info Ready to proceed? +Ensure you have followed the steps in [Connect Providers](../connect-providers/) +::: + +### 1. Download configuration template + + +{`curl -O https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/basic/tars.yaml`} + + +### 2. Configure Tetrate Agent Router Service (TARS) Credentials + +Edit the `tars.yaml` file to replace the TARS placeholder value: + +- Find the section containing `TARS_API_KEY` +- Replace it with your actual TARS API key + +:::caution Security Note +Make sure to keep your API key secure and never commit it to version control. +The key will be stored in a Kubernetes secret. +::: + +### 3. Apply Configuration + +Apply the updated configuration and wait for the Gateway pod to be ready. If you already have a Gateway running, +then the secret credential update will be picked up automatically in a few seconds. + +```shell +kubectl apply -f tars.yaml + +kubectl wait pods --timeout=2m \ + -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ + -n envoy-gateway-system \ + --for=condition=Ready +``` + +### 4. Test the Configuration + +You should have set `$GATEWAY_URL` as part of the basic setup before connecting to providers. +See the [Basic Usage](../basic-usage.md) page for instructions. + +#### Test Chat Completions + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Hi." + } + ] + }' \ + $GATEWAY_URL/v1/chat/completions +``` + +#### Test Completions (Legacy) + +TARS fully supports the legacy completions endpoint: + +```shell +curl -H "Content-Type: application/json" \ + -d '{ + "model": "babbage-002", + "prompt": "def fib(n):\n if n <= 1:\n return n\n else:\n return fib(n-1) + fib(n-2)", + "max_tokens": 25, + "temperature": 0.4, + "top_p": 0.9 + }' \ + $GATEWAY_URL/v1/completions +``` + +## Troubleshooting + +If you encounter issues: + +1. Verify your API key is correct and active + +2. Check pod status: + + ```shell + kubectl get pods -n envoy-gateway-system + ``` + +3. View controller logs: + + ```shell + kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller + ``` + +4. View External Processor Logs + + ```shell + kubectl logs -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic -c ai-gateway-extproc + ``` + +5. Common errors: + - 401: Invalid API key + - 429: Rate limit exceeded + - 503: TARS service unavailable diff --git a/site/versioned_docs/version-0.7/getting-started/index.md b/site/versioned_docs/version-0.7/getting-started/index.md new file mode 100644 index 0000000000..3823dfd36d --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/index.md @@ -0,0 +1,42 @@ +--- +id: getting-started +title: Getting Started +sidebar_position: 2 +--- + +# Getting Started with Envoy AI Gateway + +Welcome to the Envoy AI Gateway getting started guide! + +This guide will walk you through setting up and using Envoy AI Gateway, a tool for managing GenAI traffic using Envoy. + +## Guide Structure + +This getting started guide is organized into several sections: + +1. [Prerequisites](./prerequisites.md) + - Setting up your Kubernetes cluster + - Installing required tools + - Setting up Envoy Gateway + +2. [Installation](./installation.md) + - Installing Envoy AI Gateway + - Configuring the gateway + - Verifying the installation + +3. [Basic Usage](./basic-usage.md) + - Deploying a basic configuration + - Making your first request + - Understanding the response format + +4. [Connect Providers](./connect-providers) + - Setting up OpenAI integration + - Configuring AWS Bedrock + - Managing credentials securely + +## Need Help? + +If you run into any issues: + +- Join our [Community Slack](https://envoyproxy.slack.com/archives/C07Q4N24VAA) +- File an issue on [GitHub](https://github.com/envoyproxy/ai-gateway/issues) diff --git a/site/versioned_docs/version-0.7/getting-started/installation.md b/site/versioned_docs/version-0.7/getting-started/installation.md new file mode 100644 index 0000000000..172f3e105d --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/installation.md @@ -0,0 +1,78 @@ +--- +id: installation +title: Installation +sidebar_position: 3 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import vars from '../\_vars.json'; + +This guide will walk you through installing Envoy AI Gateway and its required components. + +## Installing Envoy AI Gateway + +The easiest way to install Envoy AI Gateway is using the Helm charts. You need to install the CRDs first, followed by the AI Gateway controller. + +### Step 1: Install AI Gateway CRDs + +First, install the CRD Helm chart (`ai-gateway-crds-helm`) which manages all Custom Resource Definitions: + + +{`helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm \\ + --version v${vars.aigwVersion} \\ + --namespace envoy-ai-gateway-system \\ + --create-namespace`} + + +### Step 2: Install AI Gateway Resources + +After the CRDs are installed, install the AI Gateway Helm chart: + + +{`helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm \\ + --version v${vars.aigwVersion} \\ + --namespace envoy-ai-gateway-system \\ + --create-namespace + +kubectl wait --timeout=2m -n envoy-ai-gateway-system deployment/ai-gateway-controller --for=condition=Available`} + + +:::tip +Note that you are browsing the documentation for the main branch version of Envoy AI Gateway, which is not a stable release. +We highly recommend you replace `v0.0.0-latest` with `v0.0.0-${commit hash of https://github.com/envoyproxy/ai-gateway}` to pin to a specific version. +Otherwise, the controller will be installed with the latest version at the time of installation, which can be unstable over time due to ongoing development (the latest container tags are overwritten). +::: + +> If you are experiencing network issues with `docker.io`, you can install the helm charts from the code repo [ai-gateway-crds-helm](https://github.com/envoyproxy/ai-gateway/tree/{vars.aigwGitRef}/manifests/charts/ai-gateway-crds-helm) and [ai-gateway-helm](https://github.com/envoyproxy/ai-gateway/tree/{vars.aigwGitRef}/manifests/charts/ai-gateway-helm) instead. + +:::tip Verify Installation + +Check the status of the pods. All pods should be in the `Running` state with `Ready` status. + +Check AI Gateway pods: + +```shell +kubectl get pods -n envoy-ai-gateway-system +``` + +::: + +:::note Upgrading from Previous Versions + +If you installed AI Gateway with only `ai-gateway-helm` previously, first install the CRD chart with `--take-ownership` to transfer CRD ownership, then upgrade the main chart: + + +{`helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v${vars.aigwVersion} --namespace envoy-ai-gateway-system --take-ownership +helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm --version v${vars.aigwVersion} --namespace envoy-ai-gateway-system`} + + +::: + +## Next Steps + +After completing the installation: + +- Continue to [Basic Usage](./basic-usage.md) to learn how to make your first request +- Or jump to [Connect Providers](./connect-providers) to set up OpenAI and AWS Bedrock integration diff --git a/site/versioned_docs/version-0.7/getting-started/prerequisites.md b/site/versioned_docs/version-0.7/getting-started/prerequisites.md new file mode 100644 index 0000000000..d98549946e --- /dev/null +++ b/site/versioned_docs/version-0.7/getting-started/prerequisites.md @@ -0,0 +1,196 @@ +--- +id: prerequisites +title: Prerequisites +sidebar_position: 2 +--- + +import Link from '@docusaurus/Link'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import vars from '../\_vars.json'; + +Before you begin using Envoy AI Gateway, you'll need to ensure you have the following prerequisites in place: + +## Required Tools + +Make sure you have the following tools installed: + +- `kubectl` - The Kubernetes command-line tool +- `helm` - The package manager for Kubernetes +- `curl` - For testing API endpoints (installed by default on most systems) + +:::tip Verify Installation +Run these commands to verify your tools are properly installed: + +Verify kubectl installation: + +```shell +kubectl version --client +``` + +Verify helm installation: + +```shell +helm version +``` + +Verify curl installation: + +```shell +curl --version +``` + +::: + +## Kubernetes Cluster + +:::info Version Requirements +Envoy AI Gateway requires Kubernetes version {vars.k8sMinVersion} or higher. We recommend using a recent stable version of Kubernetes for the best experience. +::: + +You need a running Kubernetes cluster with your kubeconfig properly configured. You have several options: + + + + +If you already have a Kubernetes cluster, ensure your kubeconfig is properly configured to access it. + +Verify your cluster meets the version requirements by running: + +```shell +kubectl version --output=json +``` + +The server version in the output should show version {vars.k8sMinVersion} or higher: + +```json +{ + "serverVersion": { + "major": "1", + "minor": "32+", + ... + } +} +``` + +:::caution + +If your cluster is running a version lower than {vars.k8sMinVersion}, our recommendation is to upgrade your cluster to meet the minimum requirements. +The recommended Kubernetes versions are those that are actively tested and supported by the Envoy AI Gateway team. + +::: + + + + +If you're using Docker Desktop, you can enable its built-in Kubernetes cluster: + +1. Open Docker Desktop +2. Click on the gear icon (Settings) +3. Select "Kubernetes" from the left sidebar +4. Check "Enable Kubernetes" +5. Click "Apply & Restart" + +Wait for Docker Desktop to restart and for the Kubernetes cluster to be ready. You can verify the setup by running: + +```shell +kubectl config use-context docker-desktop +kubectl cluster-info +``` + +The output should show that the Kubernetes control plane is running. + +:::tip + +Docker Desktop's Kubernetes is a great choice for local development as it: + +- Comes pre-installed with Docker Desktop +- Runs locally on your machine +- Integrates well with Docker Desktop's UI +- Requires minimal setup + +::: + + + + +If you don't have a Kubernetes cluster, you can quickly create a local one using [kind](https://kind.sigs.k8s.io/). + +First, install kind if you haven't already (on macOS with Homebrew): + +```shell +brew install kind +``` + +Then create a cluster: + +```shell +kind create cluster +``` + + + + +## Envoy Gateway + +:::warning Important + +Ensure you're using a clean Envoy Gateway deployment. If you have an existing Envoy Gateway installation with custom configurations, it may conflict with AI Gateway's requirements. We recommend: + +- Using a fresh Kubernetes cluster, or +- Uninstalling any existing Envoy Gateway deployments before proceeding: + ```shell + helm uninstall eg -n envoy-gateway-system + kubectl delete namespace envoy-gateway-system + ``` + +::: + +:::info Version Requirements + +Envoy AI Gateway requires Envoy Gateway version {vars.egMinVersion} or higher. For the best experience while trying out AI Gateway, we recommend using the latest version as shown in the commands below. + +::: + +Envoy AI Gateway is built on top of Envoy Gateway. Install it using Helm and wait for the deployment to be ready. + + +{`helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm \\ + --version v${vars.egVersion} \\ + --namespace envoy-gateway-system \\ + --create-namespace \\ + -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/manifests/envoy-gateway-values.yaml + +kubectl wait --timeout=2m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available`} + + +> If you are experiencing network issues with `docker.io` , you can follow the [Install from Source Code](https://github.com/envoyproxy/gateway/tree/main/charts/gateway-helm#install-from-source-code) instructions. + +Note that we have included the AI Gateway-specific helm values file via the `-f` flag. This file contains the necessary configuration for AI Gateway integration. + +### Additional Features (Rate Limiting, InferencePool, etc.) + +Depending on the additional features you want (like rate limiting or InferencePool), you need to pass additional addon values files to modify the Envoy Gateway installation. +Currently, supported addons are: + +- [**Rate Limiting**](../capabilities/traffic/usage-based-ratelimiting.md): + + {`https://github.com/envoyproxy/ai-gateway/blob/${vars.aigwGitRef}/examples/token_ratelimit/envoy-gateway-values-addon.yaml`} + +- [**InferencePool**](../capabilities/inference/index.md): + + {`https://github.com/envoyproxy/ai-gateway/blob/${vars.aigwGitRef}/examples/inference-pool/envoy-gateway-values-addon.yaml`} + + +For example, to install with all addons enabled, run: + + +{`helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm \\ + --version v${vars.egVersion} \\ + --namespace envoy-gateway-system \\ + --create-namespace \\ + -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/manifests/envoy-gateway-values.yaml \\ + -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/token_ratelimit/envoy-gateway-values-addon.yaml \\ + -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/${vars.aigwGitRef}/examples/inference-pool/envoy-gateway-values-addon.yaml`} + diff --git a/site/versioned_docs/version-0.7/index.md b/site/versioned_docs/version-0.7/index.md new file mode 100644 index 0000000000..c582685ea7 --- /dev/null +++ b/site/versioned_docs/version-0.7/index.md @@ -0,0 +1,69 @@ +--- +id: home +title: Home +sidebar_position: 1 +--- + +# Envoy AI Gateway Overview + +Welcome to the **Envoy AI Gateway** documentation! This open-source project, built on **Envoy +Proxy**, aims to simplify how application clients interact with **Generative AI (GenAI)** services. +It provides a secure, scalable, and efficient way to manage LLM/AI traffic, with backend rate +limiting and policy control. + +## **Project Overview** + +The **Envoy AI Gateway** was created to address the complexity of connecting applications to GenAI services by leveraging Envoy's flexibility and Kubernetes-native features. The project has evolved through contributions from the Envoy community, fostering a collaborative approach to solving real-world challenges. + +### **Key Objectives** + +- Provide a unified layer for routing and managing LLM/AI traffic. +- Support automatic failover mechanisms to ensure service reliability. +- Ensure end-to-end security, including upstream authorization for LLM/AI traffic. +- Implement a policy framework to support usage limiting use cases. +- Foster an open-source community to address GenAI-specific routing and quality of service needs. + +## **Project Goals** + +The Envoy AI Gateway project is designed to address the critical challenges of AI/LLM integration in enterprise environments through the following core goals: + +- **Resilient Connectivity Across Providers and Self-Hosted Models**: Create robust, fault-tolerant connections that integrate with LLM providers (such as OpenAI, Anthropic, AWS Bedrock, etc.) and self-hosted models, ensuring high availability through intelligent routing and automatic failover. + +- **Comprehensive Observability for Performance and Cost Management**: Provide visibility into traffic performance, usage patterns, and cost analytics, enabling organizations to optimize their GenAI usage and monitor service quality. + +- **Enterprise-Grade Security Features**: Implement security controls including fine-grained access control, authorization policies, rate limiting, to access services, as well as ensure secure egress connection to external providers from the gateway via [Upstream Authentication](capabilities/security/upstream-auth.mdx). + +- **Extensible Architecture**: Leverage Envoy's proven extensibility framework to enable rapid development of custom features, allowing organizations to adapt the gateway to their specific AI infrastructure needs. + +- **Reliable Foundation with Envoy Proxy**: Build upon Envoy's battle-tested proxy technology to provide a stable, performant, and production-ready foundation that enterprises can rely on for their traffic handling needs. + +These goals guide the development of features and capabilities that make AI/LLM integration secure, scalable, and operationally excellent for enterprise environments. + +Documentation for installation, setup, and contribution guidelines is included to help new users and contributors get started easily. + +## **Community Collaboration** + +[Weekly community meetings][meeting-notes] are held every Monday to discuss updates, address issues, and review contributions. + +## **Architecture Overview** + +## **Get Involved** + +We welcome community contributions! Here's how you can participate: + +- Attend the [weekly community meetings][meeting-notes] to stay updated and share ideas. +- Submit feature requests and pull requests via the GitHub repository. +- Join discussions in the [#envoy-ai-gateway] Slack channel. + +Refer to [this contributing guide][contributing.md] for detailed instructions on setting up your +environment and contributing. + +--- + +The **Envoy AI Gateway** addresses the growing demand for secure, scalable, and efficient AI/LLM +traffic management. Your contributions and feedback are key to its success and to advancing the +future of AI service integration. + +[meeting-notes]: https://docs.google.com/document/d/10e1sfsF-3G3Du5nBHGmLjXw5GVMqqCvFDqp_O65B0_w +[#envoy-ai-gateway]: https://envoyproxy.slack.com/archives/C07Q4N24VAA +[contributing.md]: https://github.com/envoyproxy/ai-gateway/blob/main/CONTRIBUTING.md diff --git a/site/versioned_docs/version-0.7/terminology.md b/site/versioned_docs/version-0.7/terminology.md new file mode 100644 index 0000000000..e23112fb22 --- /dev/null +++ b/site/versioned_docs/version-0.7/terminology.md @@ -0,0 +1,160 @@ +--- +id: terminology +title: Terminology +sidebar_position: 6 +--- + +# AI Gateway Glossary + +This glossary provides definitions for key terms and concepts used in AI Gateway and GenAI traffic handling. + +## Quick Reference + +| Term | Quick Definition | +| ---------------- | -------------------------------------------------- | +| GenAI Gateway | Gateway for managing AI model traffic | +| Foundation Model | Base pre-trained AI model | +| Token | Basic unit of text in LLM processing | +| Token Usage | Monitoring and limiting model resource consumption | +| Model Routing | Directing requests to appropriate models | +| Prompt | Input text guiding AI model response | +| Temperature | Control for model output randomness | + +## Categories + +- [AI/ML Fundamentals](#ai-ml-fundamentals): Token, Prompt, Context Window, Temperature +- [Inference Infrastructure](#inference-infrastructure): Inference Instance, Service, Model Provider +- [Gateway Components](#gateway-components): GenAI Gateway, Gateway API Inference Extension +- [Usage & Analytics](#usage--analytics): Usage Monitoring, Token Usage Limiting +- [Model Types & Management](#model-types--management): Foundation Model, Fine-Tuned Model +- [Content & Safety](#content--safety): Content Filtering + +## AI/ML Fundamentals {#ai-ml-fundamentals} + +### Context Window + +The maximum amount of text (in tokens) that a model can process in a single request. + +**Related**: [Token](#token) + +### Prompt + +The input text that guides the AI model's response, including instructions, context, and specific queries. + +### Temperature + +A parameter that controls the randomness/creativity of model outputs, typically ranging from 0 (deterministic) to 1 (more creative). + +### Token + +The basic unit of text processing in LLMs, representing parts of words or characters. + +**Related**: [Context Window](#context-window) · [Token Cost](#token-cost) + +### Token Cost + +The financial or resource cost associated with token usage in model requests. + +**Related**: [Token](#token) · [Rate of LLM Token Consumption](#rate-of-llm-token-consumption) + +## Content & Safety {#content--safety} + +### Content Filtering + +A mechanism to screen and moderate AI-generated content to ensure compliance with ethical standards, company policies, or regulatory requirements. + +## Gateway Components {#gateway-components} + +### Gateway API Inference Extension + +A Kubernetes SIG Network extension for Gateway API that provides specialized routing and load balancing capabilities for AI/ML workloads, handling traffic management at the level of inference instances. + +**Related**: [Inference Instance](#inference-instance) + +### GenAI Gateway + +A specialized gateway solution designed to manage, monitor, and route traffic to Generative AI models. It provides capabilities such as load balancing, authorization, token usage monitoring, and integration with multiple model providers. + +**Related**: [Token](#token) · [Model Provider](#model-provider) + +### Hybrid GenAI Gateway + +A GenAI Gateway configuration that supports both local inference instances and external cloud-based AI models, providing flexibility in deployment and cost management. + +**Related**: [GenAI Gateway](#genai-gateway) · [Inference Instance](#inference-instance) · [Model Provider](#model-provider) + +## Inference Infrastructure {#inference-infrastructure} + +### Inference Instance + +An individual compute resource or container used to run a machine learning model for generating AI outputs (inference). + +### Inference Service + +A service that provides model inference capabilities, including model loading, input processing, inference execution, and output formatting. + +**Related**: [Inference Instance](#inference-instance) + +### Model Endpoint + +The API endpoint provided by a specific AI model, whether hosted by a cloud provider, open-source solution, or private deployment. + +### Model Provider + +Services providing AI model capabilities through APIs, which can be either first-party providers who develop their own models (like OpenAI, Anthropic) or third-party providers who host other companies' models (like AWS Bedrock, Azure OpenAI Service). + +## Model Types & Management {#model-types--management} + +### Fine-Tuned Model + +A version of a base Generative AI model that has been customized for specific tasks or domains using additional training data. + +**Related**: [Foundation Model](#foundation-model) + +### Foundation Model + +Foundation models are large-scale, pre-trained AI models designed to handle a broad range of tasks. They are trained on extensive datasets and can be fine-tuned or adapted to specific use cases. + +**Related**: [Fine-Tuned Model](#fine-tuned-model) + +### Model Routing + +A feature in GenAI Gateways that dynamically routes requests to specific models or model versions based on client configuration, use case requirements, or service level agreements. + +**Related**: [GenAI Gateway](#genai-gateway) + +## Usage & Analytics {#usage--analytics} + +### GenAI Usage Analytics + +The collection and analysis of data regarding how users interact with AI models via the GenAI Gateway, including token usage, request patterns, and latency metrics. + +**Related**: [GenAI Gateway](#genai-gateway) · [Token](#token) + +### GenAI Usage Monitoring + +The tracking of resource consumption across different types of models, including token-based monitoring for LLMs, image resolution and compute resources for LVMs, and combined metrics for multimodal models. + +**Related**: [Token](#token) + +### LLM Token Usage Limiting + +A mechanism to monitor and control the number of tokens processed by an LLM GenAI model, including input, output, and total token limits. + +**Related**: [Token](#token) · [GenAI Gateway](#genai-gateway) + +### Rate of LLM Token Consumption + +The speed at which tokens are consumed by an AI model during processing. This metric is crucial for cost estimation and performance optimization. + +**Related**: [Token](#token) + +:::note +This glossary is continuously evolving as the field of GenAI traffic handling develops. If you'd like to contribute or suggest changes, please visit our [GitHub repository](https://github.com/envoyproxy/ai-gateway). +::: + +:::tip See Also + +- Check our [Getting Started](./getting-started/index.md) guide for practical examples +- Join our [Community Slack](https://envoyproxy.slack.com/archives/C07Q4N24VAA) for discussions + ::: diff --git a/site/versioned_sidebars/version-0.7-sidebars.json b/site/versioned_sidebars/version-0.7-sidebars.json new file mode 100644 index 0000000000..caea0c03ba --- /dev/null +++ b/site/versioned_sidebars/version-0.7-sidebars.json @@ -0,0 +1,8 @@ +{ + "tutorialSidebar": [ + { + "type": "autogenerated", + "dirName": "." + } + ] +} diff --git a/site/versions.json b/site/versions.json index 95661d4822..3354250430 100644 --- a/site/versions.json +++ b/site/versions.json @@ -1,4 +1,5 @@ [ + "0.7", "0.6", "0.5", "0.4", diff --git a/tests/internal/e2elib/e2elib.go b/tests/internal/e2elib/e2elib.go index 5e9a627edd..8a96bc2e33 100644 --- a/tests/internal/e2elib/e2elib.go +++ b/tests/internal/e2elib/e2elib.go @@ -425,6 +425,44 @@ func initEnvoyGateway(ctx context.Context, namespace string, inferenceExtension return } + // EG v1.8.1+ gates readiness on cache sync (cacheReadyCheck), which requires all + // informer caches—including those for backendResources—to complete an initial List. + // The EG Helm chart RBAC doesn't cover extension-managed CRDs like InferencePool, + // so the informer gets 403s and the pod never becomes ready. Grant access explicitly. + if inferenceExtension { + initLog("\tGranting Envoy Gateway RBAC for InferencePool resources") + if err = KubectlApplyManifestStdin(ctx, ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: eg-inference-pool-access +rules: +- apiGroups: + - inference.networking.k8s.io + resources: + - inferencepools + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: eg-inference-pool-access +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eg-inference-pool-access +subjects: +- kind: ServiceAccount + name: envoy-gateway + namespace: envoy-gateway-system +`); err != nil { + return fmt.Errorf("failed to apply InferencePool RBAC: %w", err) + } + } + initLog("\tWaiting for Envoy Gateway deployment to be ready") return kubectlWaitForDeploymentReady(ctx, "envoy-gateway-system", "envoy-gateway") }