Feat: rework sdk

.github/workflows/java-sdk.yml

@@ -56,9 +56,21 @@ jobs:
         if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
         run: ./gradlew check
 
+      - name: GCP - Auth with github service account
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: configure docker access to GCP registry for EE image
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Build local SDK and test it in an example project
         if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
-        run: ./run-tests.sh
+        env:
+          APPLICATION_SECRETS: ${{ secrets.APPLICATION_SECRETS }}
+        run: |
+          echo $APPLICATION_SECRETS | base64 -d > ../test-utils/docker-setup/application-secrets.yml
+          ./run-tests.sh
 
       - name: Test - Publish Test Results
         uses: dorny/test-reporter@v2

.github/workflows/javascript-sdk.yml

@@ -43,8 +43,20 @@ jobs:
       - name: build
         run: npm run build
 
+      - name: GCP - Auth with github service account
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: configure docker access to GCP registry for EE image
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Test with vitest
-        run: ./run-tests.sh
+        env:
+          APPLICATION_SECRETS: ${{ secrets.APPLICATION_SECRETS }}
+        run: |
+          echo $APPLICATION_SECRETS | base64 -d > ../test-utils/docker-setup/application-secrets.yml
+          ./run-tests.sh
 
       - name: Publish to npm
         run: npm publish --access public

.github/workflows/python-sdk.yml

@@ -41,8 +41,20 @@ jobs:
       - name: Install tests deps
         run: pip install -r test-requirements.txt
 
+      - name: GCP - Auth with github service account
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: configure docker access to GCP registry for EE image
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Test with pytest
-        run: ./run-tests.sh
+        env:
+          APPLICATION_SECRETS: ${{ secrets.APPLICATION_SECRETS }}
+        run: |
+          echo $APPLICATION_SECRETS | base64 -d > ../test-utils/docker-setup/application-secrets.yml
+          ./run-tests.sh
 
       - name: Build a binary wheel and a source tarball
         env:

README_JAVA_SDK.md

@@ -42,6 +42,14 @@
       }
       ```
 
+### More informations
+
+- SSE methods are injected from templates
+- Openapi spec is modified during generation through a custom TS script
+- KestraClient is manually written to gather all API clients in one client
+- ApiClient & build.gradle are fixed through the templates
+- Method with Multipart form need an annotation on Kestra side to generate properly the SDK method
+
 ## Step to use
 
 The openapi generator will generate 1 Api per controller, so we create a custom Kestra Client that need to be instantiated once for every API.

README_PYTHON_SDK.md

@@ -21,14 +21,15 @@
      ```python
      from kestrapy.kestra_client import KestraClient as KestraClient
      ```
-  - In the `task.py`, delete the following part, from the type validator:
-      ```python
-              if not re.match(r"\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*(\.\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)*", value):
-            raise ValueError(r"must validate the regular expression /\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*(\.\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)*/")
-      ```
- - In the `executions_api.py` inject the SSE method, done through the inject_sse_method.sh script
-
- - In the core, "public HttpResponse<Flow> updateFlowFromJson" had to be commented to avoid a conflict on the path, see the issue: https://github.com/kestra-io/kestra/issues/11928
+
+### More informations
+
+- SSE methods are injected from templates
+- Openapi spec is modified during generation through a custom TS script
+- KestraClient is manually written to gather all API clients in one client
+- Method with Multipart form need an annotation on Kestra side to generate properly the SDK method
+
+
 ## Step to use
 
 The openapi generator will generate 1 Api per controller, so we create a custom Kestra Client that need to be instantiated once for every API.

configurations/go-config.yml

@@ -3,8 +3,12 @@ inputSpec: /local/kestra-ee.yml
 outputDir: /local/go-sdk
 
 globalProperties:
-  apiTests: false
+  apiTests: true
   modelTests: false
+  apiDocs: true
+  modelDocs: true
+  skipFormModel: false
+
 
 additionalProperties:
   packageName: kestra_api_client
@@ -24,3 +28,6 @@ typeMappings:
   PropertyDouble: string
   PropertyListString: string
   PropertyObject: string
+
+openapiNormalizer:
+  FILTER: 'tag:Executions|Flows|Namespaces|Triggers|Users|Groups|KV|Roles|ServiceAccount'

configurations/java-config.yml

@@ -5,6 +5,9 @@ outputDir: /local/java-sdk
 globalProperties:
   apiTests: false
   modelTests: false
+  apiDocs: true
+  modelDocs: true
+  skipFormModel: false
 
 additionalProperties:
   disallowAdditionalPropertiesIfNotPresent: false
@@ -15,9 +18,13 @@ modelPackage: io.kestra.sdk.model
 apiPackage: io.kestra.sdk.api
 groupId: io.kestra
 artifactId: kestra-api-client
+enumUnknownDefaultCase: true
+useOneOfInterfaces: true
+useJakartaEe: true
 
 hideGenerationTimestamp: true
 supportUrlQuery: false
+useRuntimeException: true
 
 typeMappings:
   PropertyDuration: String
@@ -26,3 +33,6 @@ typeMappings:
   PropertyDouble: String
   PropertyListString: String
   PropertyObject: String
+
+openapiNormalizer:
+  FILTER: 'tag:Executions|Flows|Namespaces|Triggers|Users|Groups|KV|Roles|ServiceAccount'

configurations/javascript-config.yml

@@ -3,12 +3,20 @@ inputSpec: /local/kestra-ee.yml
 outputDir: /local/javascript-sdk
 
 globalProperties:
-  apiTests: false
+  apiTests: true
   modelTests: false
+  apiDocs: true
+  modelDocs: true
+  skipFormModel: false
+
+usePromises: true
 
 hideGenerationTimestamp: true
 
 additionalProperties:
   packageName: "@kestra-io/kestra-sdk"
   projectName: "@kestra-io/kestra-sdk"
   disallowAdditionalPropertiesIfNotPresent: false
+
+openapiNormalizer:
+  FILTER: 'tag:Executions|Flows|Namespaces|Triggers|Users|Groups|KV|Roles|ServiceAccount'

configurations/kestra-openapi-sdk-customizer.json

@@ -0,0 +1,18 @@
+{
+  "removeDeprecatedParameters": true,
+  "removeDeprecatedOperations": true,
+  "operationIdsToSkip": [
+    "triggerExecutionByPostWebhook",
+    "validateNewExecutionInputs",
+    "validateResumeExecutionInputs",
+    "triggerExecutionByPutWebhook",
+    "triggerExecution",
+    "searchTaskRun",
+    "resumeExecutionFromBreakpoint",
+    "previewFileFromExecution",
+    "listFlowExecutionsByNamespace",
+    "listExecutableDistinctNamespaces",
+    "evalTaskRunExpression",
+    "updateTaskRunState"
+  ]
+}

configurations/python-config.yml

@@ -3,9 +3,11 @@ inputSpec: /local/kestra-ee.yml
 outputDir: /local/python-sdk
 
 globalProperties:
-  apiTests: false
+  apiTests: true
   modelTests: false
-
+  apiDocs: true
+  modelDocs: true
+  skipFormModel: false
 hideGenerationTimestamp: true
 
 additionalProperties:
@@ -16,3 +18,6 @@ additionalProperties:
   customEmail: "contact@kestra.io"
   customAuthor: "Kestra.io"
   customDescription: "Kestra Python SDK"
+
+openapiNormalizer:
+  FILTER: 'tag:Executions|Flows|Namespaces|Triggers|Users|Groups|KV|Roles|ServiceAccount'

generate-sdks.sh

@@ -25,17 +25,22 @@ fi
 
 BASE_PKG=io.kestra.sdk
 
-OPENAPI_GITHUB_LOCATION="/repos/kestra-io/kestra-ee/contents/kestra-ee.yml"
-OPENAPI_LOCATION_BRANCH="v1.0.7-openapi-spec"
-echo "download openapi spec from github at '$OPENAPI_GITHUB_LOCATION' in branch '$OPENAPI_LOCATION_BRANCH'"
-gh api -H "Accept: application/vnd.github.raw" "$OPENAPI_GITHUB_LOCATION?ref=$OPENAPI_LOCATION_BRANCH" > kestra-ee.yml
+# Disabled for now
+#OPENAPI_GITHUB_LOCATION="/repos/kestra-io/kestra-ee/contents/kestra-ee.yml"
+#OPENAPI_LOCATION_BRANCH="v1.0.7-openapi-spec"
+#echo "download openapi spec from github at '$OPENAPI_GITHUB_LOCATION' in branch '$OPENAPI_LOCATION_BRANCH'"
+#gh api -H "Accept: application/vnd.github.raw" "$OPENAPI_GITHUB_LOCATION?ref=$OPENAPI_LOCATION_BRANCH" > kestra-ee.yml
 
 if [ -n "$TEMPLATE_FLAG" ]; then
   echo "Generating templates"
-  docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/openapi-generator-cli:latest-release author template -g "$LANGUAGES" -o /local/templates/python
+  docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/openapi-generator-cli:latest-release author template -g "$LANGUAGES" -o /local/templates/$LANGUAGES
   exit 0
 fi
 
+KESTRA_OPENAPI_SDK_CUSTOMIZER_CONF=$(readlink -f ./configurations/kestra-openapi-sdk-customizer.json)
+KESTRA_OPENAPI=$(readlink -f ./kestra-ee.yml)
+sh -c "cd ./generation-helpers/kestra-openapi-sdk-customizer && npm i && npm run build && npm start $KESTRA_OPENAPI_SDK_CUSTOMIZER_CONF $KESTRA_OPENAPI"
+
 # Generate Java SDK
 if [[ ",$LANGUAGES," == *",java,"* ]]; then
 rm -rf ./java-sdk/docs
@@ -45,10 +50,18 @@ rm -rf ./java-sdk/src/main/java/io/kestra/sdk/model
 
 docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/openapi-generator-cli:latest-release generate \
      -c /local/configurations/java-config.yml --artifact-version $VERSION \
-      --skip-validate-spec
+      --skip-validate-spec \
+      --template-dir=/local/templates/java
 
 find ./java-sdk/src/main/java -type f -name "*.java" -exec sed -i.bak 's/Map<Task>/List<Task>/g' {} + && find ./java-sdk/src/main/java -name "*.bak" -delete
 echo "version=$VERSION" > ./java-sdk/gradle.properties
+
+
+# Replace wrong prop in docs for each api
+for f in java-sdk/docs/*.md; do
+  [ -f "$f" ] || continue
+  sed -E -i 's/\b([A-Za-z]+)Api\b/\L\1/g' "$f"
+done
 fi
 
 # Generate Python SDK
@@ -59,28 +72,33 @@ docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/ope
     --additional-properties=packageVersion=$VERSION \
     --template-dir=/local/templates/python
 
-sed $SED_INPLACE -E 's/^license = .*/license = "Apache-2.0"/' python-sdk/pyproject.toml
-sed $SED_INPLACE -E 's/^requires-python = .*/requires-python = ">=3.9"/' python-sdk/pyproject.toml
 sed $SED_INPLACE -E '/from kestrapy\.models\.list\[label\] import List\[Label\]/d' python-sdk/kestrapy/api/executions_api.py
-grep -vF '{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}' python-sdk/kestrapy/models/task.py > temp_file && mv temp_file python-sdk/kestrapy/models/task.py
+sed $SED_INPLACE -E 's/value: Optional\[Dict\[str, Any\]\] = None/value: Optional[Any] = None/' python-sdk/kestrapy/models/kv_controller_typed_value.py
 echo "from kestrapy.kestra_client import KestraClient as KestraClient" >> python-sdk/kestrapy/__init__.py
-sh generation-helpers/python/inject_sse_method.sh # Inject SSE method in the client for follow execution endpoint
+
+# Replace wrong prop in docs for each api
+for f in python-sdk/docs/*.md; do
+  [ -f "$f" ] || continue
+  sed -E -i 's/\b([A-Za-z]+)Api\b/\L\1/g' "$f"
+done
 fi
 
 # Generate Javascript SDK
 if [[ ",$LANGUAGES," == *",javascript,"* ]]; then
 docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/openapi-generator-cli:latest-release generate \
     -c /local/configurations/javascript-config.yml \
     --skip-validate-spec \
-    --additional-properties=projectVersion=$VERSION
+    --additional-properties=projectVersion=$VERSION \
+    --template-dir=/local/templates/javascript
 fi
 
 # Generate GoLang SDK
 if [[ ",$LANGUAGES," == *",go,"* ]]; then
 docker run --rm -v ${PWD}:/local --user ${HOST_UID}:${HOST_GID} openapitools/openapi-generator-cli:latest-release generate \
       -c /local/configurations/go-config.yml \
       --skip-validate-spec \
-      --additional-properties=packageVersion=$VERSION
+      --additional-properties=packageVersion=$VERSION \
+    --template-dir=/local/templates/go
 # these generated structs collides between api_cluster.go and api_maintenance.go, needs to be improved TODO
 sed $SED_INPLACE.bak -e 's/ApiEnterMaintenanceRequest/ApiClusterEnterMaintenanceRequest/g' ./go-sdk/api_cluster.go && rm ./go-sdk/api_cluster.go.bak
 sed $SED_INPLACE.bak -e 's/ApiExitMaintenanceRequest/ApiClusterExitMaintenanceRequest/g' ./go-sdk/api_cluster.go && rm ./go-sdk/api_cluster.go.bak

generation-helpers/kestra-openapi-sdk-customizer/.gitignore

@@ -0,0 +1,40 @@
+### Java
+Thumbs.db
+.DS_Store
+.gradle
+build/
+target/
+out/
+.idea
+.vscode
+prettierrc.js
+*.iml
+*.ipr
+*.iws
+.project
+.settings
+.classpath
+.attach*
+**/*.class
+**/bin/*
+
+### Configurations
+docker-compose.override.yml
+cli/src/main/resources/application-*.yml
+/local
+
+### Javascript
+node
+node_modules
+coverage
+
+### Build
+core/src/main/resources/gradle.properties
+.plugins.override
+
+# Allure Reports
+**/allure-results/*
+
+*storybook.log
+storybook-static
+/jmh-benchmarks/src/main/resources/gradle.properties

generation-helpers/kestra-openapi-sdk-customizer/.prettierrc.json

@@ -0,0 +1,6 @@
+{
+  "semi": true,
+  "singleQuote": false,
+  "trailingComma": "all",
+  "printWidth": 100
+}

generation-helpers/kestra-openapi-sdk-customizer/README.md

@@ -0,0 +1,16 @@
+# kestra-openapi-sdk-customizer
+
+this is a cli to transform our kestra-ee.yml to better fit our SDK needs:
+- remove all deprecated parameters and operations (example: old query filter params)
+- exclude a list of operationId we don't want to handle yet
+
+## run it
+```
+npm run build && npm start <your-conf.json> <your-kestra-openapi-spec.yml>
+```
+
+
+Example in  kestra-openapi-sdk-customizer dir:
+```
+ npm run build && npm start $(readlink -f ../../configurations/kestra-openapi-sdk-customizer.json) $(readlink -
+f ../../kestra-ee.yml)```

generation-helpers/kestra-openapi-sdk-customizer/eslint.config.mjs

@@ -0,0 +1,12 @@
+// @ts-check
+import eslint from "@eslint/js";
+import { defineConfig } from "eslint/config";
+import tseslint from "typescript-eslint";
+
+export default defineConfig(
+    {
+        ignores: ["dist/**", "coverage/**", "node_modules/**"],
+    },
+    eslint.configs.recommended,
+    tseslint.configs.recommended,
+);