Skip to content

Commit b873335

Browse files
jrhee17jrhee17
authored
Introduce the xds-api module (#6403)
Motivation: Envoy utilizes pgv to validate the schema of a resource. ref: https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/config_validation/config_validation Given that upstream `libs.controlplane.api` doesn't support this, I propose that we publish our own jar containing APIs along with validators. See jrhee17#43 for a sample run. Going forward, this can be further developed so that we can add our own annotations. This may be useful for marking which fields are supported by Armeria - this can be further exposed to users via metrics/logs (in case a control plane specifies fields not supported) and in the UI later on. Modifications: - Workflows are added to periodically check if upstream envoy version is updated - `xds-apply-updates.yml`: Runs the update script and creates a pull request - `xds-compare-versions.yml`: Compares envoy versions with the current version and determines whether an update is needed - `xds-sync-apis.yml`: An umbrella task which invokes the above two workflows periodically - Scripts are added under `/xds-api/tools` - `upstream-patch.sh`: Creates two worktrees corresponding to each envoy version, and computes a patch. The patch is applied to the working directory. - `update-sha.sh`: Updates the sha of each version (envoy and dependencies) and writes `API_SHAS` and `envoy_version` files. This has been forked from upstream (which is also apache 2.0 licensed) - `update-api.sh`: Updates the proto directory according to the `API_SHAS` and `envoy_version` files. This has also been forked. - Added dependencies to `pgv-java-stub` and `protoc-gen-validate`, each of which generates validator files and applies validation at runtime. Result: - The `xds` module now relies on the `xds-api` module. <!-- Visit this URL to learn more about how to write a pull request description: https://armeria.dev/community/developer-guide#how-to-write-pull-request-description -->
1 parent abc51b0 commit b873335

File tree

628 files changed

+65953
-4
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

628 files changed

+65953
-4
lines changed

‎.github/workflows/xds-apply-updates.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Apply xDS API updates from upstream
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
workflow_call:
8+
inputs:
9+
target_version:
10+
description: 'Envoy version to update to'
11+
required: true
12+
type: string
13+
14+
jobs:
15+
update-protobuf:
16+
permissions:
17+
contents: write
18+
pull-requests: write
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v4
22+
- name: Configure git
23+
run: |
24+
git config user.email "[email protected]"
25+
git config user.name "Meri Kim"
26+
- name: Run scripts
27+
working-directory: ./xds-api/tools/
28+
run: |
29+
if ! ./upstream-patch.sh --target ${{ inputs.target_version }}; then
30+
echo "❌ Automatic update failed due to conflicts."
31+
echo ""
32+
echo "To resolve manually:"
33+
echo "1. Run locally: ./xds-api/tools/upstream-patch.sh --target ${{ inputs.target_version }}"
34+
echo "2. Resolve any conflicts shown in the output"
35+
echo "3. Commit and push the changes"
36+
exit 1
37+
fi
38+
- name: Create Pull Request
39+
uses: peter-evans/create-pull-request@v7
40+
with:
41+
branch: update-protobuf-to-${{ inputs.target_version }}
42+
base: main
43+
author: Meri Kim <[email protected]>
44+
committer: Meri Kim <[email protected]>
45+
signoff: true
46+
delete-branch: true
47+
title: '[xds] Update protobuf definitions to ${{ inputs.target_version }}'
48+
commit-message: |
49+
[xds] Update protobuf definitions to ${{ inputs.target_version }}
50+
body: |
51+
This is an automatic PR created by github action workflow:
52+
- Updated protobuf files

‎.github/workflows/xds-compare-versions.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Compare upstream envoy versions
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
workflow_call:
8+
outputs:
9+
target_version:
10+
description: "Envoy version we need to update to"
11+
value: ${{ jobs.compare-envoy-versions.outputs.target_version }}
12+
should_update:
13+
description: "Whether the apis should be updated"
14+
value: ${{ jobs.compare-envoy-versions.outputs.should_update }}
15+
16+
jobs:
17+
compare-envoy-versions:
18+
runs-on: ubuntu-latest
19+
outputs:
20+
target_version: ${{ steps.latest-envoy-version.outputs.version }}
21+
should_update: ${{ steps.compare.outputs.should_update }}
22+
steps:
23+
- uses: actions/checkout@v4
24+
- name: Fetch latest Envoy version
25+
id: latest-envoy-version
26+
run: |
27+
version=$(curl -s https://api.github.com/repos/envoyproxy/envoy/releases/latest | jq -r '.tag_name')
28+
echo "version=$version" >> $GITHUB_OUTPUT
29+
- name: Read current Envoy version
30+
id: current-envoy-version
31+
run: |
32+
version=$(cat ./xds-api/tools/envoy_release)
33+
echo "version=$version" >> $GITHUB_OUTPUT
34+
- name: Compare latest to current
35+
id: compare
36+
run: |
37+
latest="${{ steps.latest-envoy-version.outputs.version }}"
38+
current="${{ steps.current-envoy-version.outputs.version }}"
39+
40+
# Remove 'v' prefix if present
41+
latest_clean=${latest#v}
42+
current_clean=${current#v}
43+
44+
# Function to compare semantic versions
45+
version_greater() {
46+
printf '%s\n%s\n' "$1" "$2" | sort -V | head -n1 | grep -q "^$2$"
47+
}
48+
49+
if version_greater "$latest_clean" "$current_clean"; then
50+
echo "should_update=true" >> $GITHUB_OUTPUT
51+
else
52+
echo "should_update=false" >> $GITHUB_OUTPUT
53+
fi

‎.github/workflows/xds-sync-apis.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Sync xDS APIs with upstream
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
schedule:
8+
# every day at 10
9+
- cron: '0 10 * * *'
10+
11+
jobs:
12+
envoy-versions:
13+
uses: ./.github/workflows/xds-compare-versions.yml
14+
15+
call-update-protobuf:
16+
permissions:
17+
contents: write
18+
pull-requests: write
19+
needs: envoy-versions
20+
if: ${{ needs.envoy-versions.outputs.should_update == 'true' }}
21+
uses: ./.github/workflows/xds-apply-updates.yml
22+
with:
23+
target_version: ${{ needs.envoy-versions.outputs.target_version }}

‎dependencies.toml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ opensaml = "3.4.6"
109109
osdetector = "1.7.3"
110110
# Used for kubernetes-chaos-tests
111111
picocli = "4.7.7"
112+
protoc-gen-validate = "1.2.1"
112113
proguard = "7.5.0"
113114
prometheus = "1.3.10"
114115
prometheus-legacy = "0.16.0"
@@ -344,9 +345,6 @@ version.ref = "checkstyle"
344345
module = "io.micrometer:context-propagation"
345346
version.ref = "context-propagation"
346347

347-
[libraries.controlplane-api]
348-
module = "io.envoyproxy.controlplane:api"
349-
version.ref = "controlplane"
350348
[libraries.controlplane-cache]
351349
module = "io.envoyproxy.controlplane:cache"
352350
version.ref = "controlplane"
@@ -994,6 +992,13 @@ javadocs = "https://prometheus.github.io/client_java/"
994992
module = "io.prometheus:simpleclient_common"
995993
version.ref = "prometheus-legacy"
996994

995+
[libraries.protoc-pgv-java-stub]
996+
module = "build.buf.protoc-gen-validate:pgv-java-stub"
997+
version.ref = "protoc-gen-validate"
998+
[libraries.protoc-gen-validate]
999+
module = "build.buf.protoc-gen-validate:protoc-gen-validate"
1000+
version.ref = "protoc-gen-validate"
1001+
9971002
[libraries.protobuf-java]
9981003
module = "com.google.protobuf:protobuf-java"
9991004
version.ref = "protobuf"

‎gradle/scripts/lib/java-rpc-proto.gradle‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,16 @@ configure(projectsWithFlags('java')) {
8888
}
8989
}
9090
}
91+
92+
def validateVersion = managedVersions['build.buf.protoc-gen-validate:protoc-gen-validate']
93+
if (validateVersion != null) {
94+
plugins {
95+
javapgv {
96+
artifact = "build.buf.protoc-gen-validate:protoc-gen-validate:${validateVersion}"
97+
}
98+
}
99+
}
100+
91101
generateProtoTasks {
92102
all()*.plugins {
93103
if (project.ext.hasFlag('scala-grpc_2.13') && managedVersions.containsKey('com.thesamet.scalapb:scalapb-runtime_2.13')) {
@@ -114,6 +124,11 @@ configure(projectsWithFlags('java')) {
114124
akkaGrpc {}
115125
}
116126
}
127+
if (project.ext.hasFlag('javapgv')) {
128+
javapgv {
129+
option "lang=java"
130+
}
131+
}
117132
}
118133

119134
all().each { task ->

‎settings.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ includeWithFlags ':tomcat8', 'java', 'publish', 'rel
197197
includeWithFlags ':tomcat9', 'java', 'publish', 'relocate', 'no_aggregation'
198198
includeWithFlags ':tomcat10', 'java11', 'publish', 'relocate'
199199
includeWithFlags ':xds', 'java', 'publish', 'relocate'
200+
includeWithFlags ':xds-api', 'java', 'publish', 'relocate', 'javapgv'
200201
includeWithFlags ':zookeeper3', 'java', 'publish', 'relocate', 'native'
201202
includeWithFlags ':saml', 'java', 'publish', 'relocate', 'native'
202203
includeWithFlags ':bucket4j', 'java', 'publish', 'relocate', 'native'

‎xds-api/build.gradle‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
dependencies {
2+
configurations.configureEach {
3+
resolutionStrategy {
4+
force "com.google.protobuf:protobuf-java:${libs.versions.protobuf.asProvider().get()}"
5+
}
6+
}
7+
8+
compileOnly libs.protobuf.java
9+
compileOnly libs.protobuf.java.util
10+
compileOnly project(":grpc")
11+
compileOnly libs.protoc.gen.validate
12+
compileOnly libs.protoc.pgv.java.stub
13+
14+
testImplementation libs.protobuf.java
15+
testImplementation libs.protobuf.java.util
16+
testImplementation libs.protoc.pgv.java.stub
17+
}
18+
19+
tasks.withType(JavaCompile).configureEach {
20+
options.forkOptions.memoryMaximumSize = '1g'
21+
}

0 commit comments

Comments
 (0)