feat: Support orchestration prompt module fallback (with stream)

.changeset/curvy-mice-hear.md

@@ -0,0 +1,5 @@
+---
+'@sap-ai-sdk/orchestration': minor
+---
+
+[feat] Support streaming with orchestration prompt module fallback.

packages/orchestration/src/index.ts

@@ -11,6 +11,7 @@ export type {
   ChatCompletionRequest,
   RequestOptions,
   StreamOptions,
+  StreamOptionsArray,
   DocumentGroundingServiceConfig,
   DocumentGroundingServiceFilter,
   DpiMaskingConfig,

packages/orchestration/src/orchestration-client.test.ts

@@ -28,7 +28,8 @@ import type { CompletionPostResponse } from './client/api/schema/index.js';
 import type {
   OrchestrationModuleConfig,
   OrchestrationConfigRef,
-  ChatCompletionRequest
+  ChatCompletionRequest,
+  StreamOptionsArray
 } from './orchestration-types.js';
 
 const defaultJsonConfig = `{
@@ -1318,4 +1319,265 @@ describe('orchestration service client', () => {
       spy.mockRestore();
     });
   });
+
+  describe('Orchestration Client with module fallback configs', () => {
+    it('calls chatCompletion with module fallback configuration array', async () => {
+      const primaryConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-4o', timeout: 5, params: { max_tokens: 100 } },
+          prompt: {
+            template: [{ role: 'user', content: 'Try primary model' }]
+          }
+        }
+      };
+
+      const fallbackConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-5-mini', params: { max_tokens: 50 } },
+          prompt: {
+            template: [{ role: 'user', content: 'Fallback model' }]
+          }
+        }
+      };
+
+      const mockResponse = await parseMockResponse<CompletionPostResponse>(
+        'orchestration',
+        'orchestration-chat-completion-success-response.json'
+      );
+
+      mockInference(
+        {
+          data: constructCompletionPostRequest([primaryConfig, fallbackConfig])
+        },
+        {
+          data: mockResponse,
+          status: 200
+        },
+        {
+          url: 'inference/deployments/1234/v2/completion'
+        }
+      );
+
+      const response = await new OrchestrationClient([
+        primaryConfig,
+        fallbackConfig
+      ]).chatCompletion();
+
+      expect(response).toBeInstanceOf(OrchestrationResponse);
+      expect(response.getContent()).toBe('Hello! How can I assist you today?');
+    });
+
+    it('calls streaming with module fallback configuration array', async () => {
+      const primaryConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-4o' },
+          prompt: {
+            template: [{ role: 'user', content: 'Primary model' }]
+          }
+        }
+      };
+
+      const fallbackConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-5-mini' },
+          prompt: {
+            template: [{ role: 'user', content: 'Fallback model' }]
+          }
+        }
+      };
+
+      mockInference(
+        {
+          data: constructCompletionPostRequest(
+            [primaryConfig, fallbackConfig],
+            undefined,
+            true
+          )
+        },
+        {
+          data: streamMockResponse,
+          status: 200
+        },
+        {
+          url: 'inference/deployments/1234/v2/completion'
+        }
+      );
+
+      const response = await new OrchestrationClient([
+        primaryConfig,
+        fallbackConfig
+      ]).stream();
+
+      const chunks: string[] = [];
+      for await (const chunk of response.stream) {
+        const content = chunk.getDeltaContent();
+        if (content) {
+          chunks.push(content);
+        }
+      }
+
+      // Verify that streaming completed successfully and returned content
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks.join('')).toContain('SAP Cloud SDK');
+    });
+
+    it('returns intermediate failures from fallback attempts', async () => {
+      const primaryConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'non-existing-model' },
+          prompt: {
+            template: [{ role: 'user', content: 'Test' }]
+          }
+        }
+      };
+
+      const fallbackConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-5-mini' },
+          prompt: {
+            template: [{ role: 'user', content: 'Test' }]
+          }
+        }
+      };
+
+      const mockResponseWithErrors = {
+        ...(await parseMockResponse<CompletionPostResponse>(
+          'orchestration',
+          'orchestration-chat-completion-success-response.json'
+        )),
+        intermediate_failures: [
+          {
+            message: 'Model non-existing-model not found',
+            code: 'MODEL_NOT_FOUND',
+            location: 'config[0]'
+          }
+        ]
+      };
+
+      mockInference(
+        {
+          data: constructCompletionPostRequest([primaryConfig, fallbackConfig])
+        },
+        {
+          data: mockResponseWithErrors,
+          status: 200
+        },
+        {
+          url: 'inference/deployments/1234/v2/completion'
+        }
+      );
+
+      const response = await new OrchestrationClient([
+        primaryConfig,
+        fallbackConfig
+      ]).chatCompletion();
+
+      const failures = response.getIntermediateFailures();
+      expect(failures).toBeDefined();
+      expect(failures).toHaveLength(1);
+      expect(failures![0].message).toBe('Model non-existing-model not found');
+    });
+
+    it('should throw when provided empty config array', () => {
+      expect(() => new OrchestrationClient([])).toThrow(
+        'Configuration array must not be empty.'
+      );
+    });
+  });
+
+  describe('Orchestration Client with per-config stream options', () => {
+    it('calls streaming with array of stream options matching module fallback configs', async () => {
+      const primaryConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-4o' },
+          prompt: {
+            template: [{ role: 'user', content: 'Primary' }]
+          }
+        }
+      };
+
+      const fallbackConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-5-mini' },
+          prompt: {
+            template: [{ role: 'user', content: 'Fallback' }]
+          }
+        }
+      };
+
+      const streamOptions: StreamOptionsArray = [
+        { promptTemplating: { include_usage: true } },
+        { promptTemplating: { include_usage: false } }
+      ];
+
+      mockInference(
+        {
+          data: constructCompletionPostRequest(
+            [primaryConfig, fallbackConfig],
+            undefined,
+            true,
+            streamOptions
+          )
+        },
+        {
+          data: streamMockResponse,
+          status: 200
+        },
+        {
+          url: 'inference/deployments/1234/v2/completion'
+        }
+      );
+
+      const response = await new OrchestrationClient([
+        primaryConfig,
+        fallbackConfig
+      ]).stream({}, undefined, streamOptions);
+
+      const chunks: string[] = [];
+      for await (const chunk of response.stream) {
+        const content = chunk.getDeltaContent();
+        if (content) {
+          chunks.push(content);
+        }
+      }
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks.join('')).toContain('SAP Cloud SDK');
+    });
+
+    it('throws error when stream options array length does not match configs array', async () => {
+      const primaryConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-4o' },
+          prompt: {
+            template: [{ role: 'user', content: 'Primary' }]
+          }
+        }
+      };
+
+      const fallbackConfig: OrchestrationModuleConfig = {
+        promptTemplating: {
+          model: { name: 'gpt-5-mini' },
+          prompt: {
+            template: [{ role: 'user', content: 'Fallback' }]
+          }
+        }
+      };
+
+      const streamOptions = [
+        { promptTemplating: { include_usage: true } }
+        // Missing second element
+      ];
+
+      await expect(
+        new OrchestrationClient([primaryConfig, fallbackConfig]).stream(
+          {},
+          undefined,
+          streamOptions
+        )
+      ).rejects.toThrow(
+        'StreamOptions array length (1) must match moduleConfigs array length (2)'
+      );
+    });
+  });
 });

packages/orchestration/src/orchestration-client.ts

@@ -31,7 +31,8 @@ import type {
   OrchestrationConfigRef,
   ChatCompletionRequest,
   RequestOptions,
-  StreamOptions
+  StreamOptions,
+  StreamOptionsArray
 } from './orchestration-types.js';
 import type { OrchestrationStreamChunkResponse } from './orchestration-stream-chunk-response.js';
 import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity';
@@ -94,15 +95,9 @@ export class OrchestrationClient {
   async stream(
     request?: ChatCompletionRequest,
     signal?: AbortSignal,
-    options?: StreamOptions,
+    options?: StreamOptions | StreamOptionsArray,
     requestConfig?: CustomRequestConfig
   ): Promise<OrchestrationStreamResponse<OrchestrationStreamChunkResponse>> {
-    if (isOrchestrationModuleConfigList(this.config)) {
-      throw new Error(
-        'Streaming is not supported when using multiple orchestration module configurations for fallback. Please use a single configuration.'
-      );
-    }
-
     const controller = new AbortController();
     if (signal) {
       signal.throwIfAborted();
@@ -160,12 +155,19 @@ export class OrchestrationClient {
               this.config,
               request
             )
-          : constructCompletionPostRequest(
-              this.config,
-              request,
-              stream,
-              streamOptions
-            );
+          : isOrchestrationModuleConfigList(this.config)
+            ? constructCompletionPostRequest(
+                this.config,
+                request,
+                stream,
+                streamOptions
+              )
+            : constructCompletionPostRequest(
+                this.config,
+                request,
+                stream,
+                streamOptions as StreamOptions | undefined
+              );
 
     const deploymentId = await getOrchestrationDeploymentId(
       this.deploymentConfig || {},
@@ -187,7 +189,7 @@ export class OrchestrationClient {
     );
 
     // Log summary when fallbacks were used
-    if (!stream && response.data?.intermediate_failures) {
+    if (response.data?.intermediate_failures) {
       const failureCount = response.data.intermediate_failures.length;
       const successModel = response.data.final_result?.model;
       logger.info(

packages/orchestration/src/orchestration-stream-response.ts

@@ -8,7 +8,8 @@ import type {
   LlmChoice,
   MessageToolCalls,
   ModuleResults,
-  TokenUsage
+  TokenUsage,
+  Error as OrchestrationError
 } from './client/api/schema/index.js';
 import type { OrchestrationStream } from './orchestration-stream.js';
 
@@ -172,6 +173,19 @@ export class OrchestrationStreamResponse<T> {
     return this._data.intermediate_results;
   }
 
+  /**
+   * Gets the intermediate failures from the orchestration response.
+   * When using module fallback, this contains errors from module configurations
+   * that failed before a successful one was found.
+   * @returns The intermediate failures, or undefined if there were none.
+   */
+  getIntermediateFailures(): OrchestrationError[] | undefined {
+    if (this.isStreamOpen()) {
+      return;
+    }
+    return this._data.intermediate_failures;
+  }
+
   /**
    * Parses the response and returns the choice by index.
    * @param index - The index of the choice to find.