Skip to content

Commit 9a63879

Browse files
feat(speech): expose DashScope voice-enrollment (voice cloning) API
Adds POST /api/v1/services/audio/tts/customization as a fork-only, native DashScope endpoint so voice-cloning management (create_voice, list_voice, query_voice, update_voice, delete_voice) flows through the gateway with the usual auth / logging / RBAC surface attached. Design: this is a pass-through, not a translation. There is no OpenAI-shaped counterpart to voice-cloning; the caller speaks DashScope's native schema and the gateway forwards the body unchanged. Only body.model is parsed (to populate x-ai-eg-model=voice-enrollment for route matching) and body.input.action is exposed on the request struct so operators can tag metrics per action. Layout mirrors the existing GeminiCachedContentsEndpointSpec / cachedContents translator pair: - internal/apischema/dashscope: VoiceEnrollmentRequest + VoiceEnrollmentInput minimal parsing types (Model + Action). Rest of the payload stays as raw JSON because it varies per action and modelling it here would just track upstream drift. - internal/translator/voice_enrollment_dashscope.go: pass-through translator that rewrites `:path` to /api/v1/services/audio/tts/customization and only replays the body when Envoy's upstream stage forces a mutation (retry / streaming). No response reshaping — DashScope's action-specific envelope is the caller's contract. - internal/endpointspec/endpointspec.go: DashScopeVoiceEnrollmentEndpointSpec with ParseBody / GetTranslator / RedactSensitiveInfoFromRequest. Only AlibabaDashScope schema is supported. - cmd/extproc/mainlib/main.go: server.Register for the native path with a noop tracer (no response object to record) and speech metrics factory. Not covered by this PR (defer): - gitops backend + route rule (workspace endpoint + AIGatewayRoute) — separate follow-up so the gateway image can roll independently of the route change. - OpenAI-shaped surface (/v1/audio/voices etc.). If demand shows up, layer it on top; DashScope's action-set doesn't map cleanly to any OpenAI shape today. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent b6e8a4e commit 9a63879

5 files changed

Lines changed: 256 additions & 0 deletions

File tree

cmd/extproc/mainlib/main.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"google.golang.org/grpc"
2727
"google.golang.org/grpc/health/grpc_health_v1"
2828

29+
dashscopeschema "github.com/envoyproxy/ai-gateway/internal/apischema/dashscope"
2930
gcpschema "github.com/envoyproxy/ai-gateway/internal/apischema/gcp"
3031
"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
3132
"github.com/envoyproxy/ai-gateway/internal/endpointspec"
@@ -344,6 +345,20 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) {
344345
extractGeminiPathInfo,
345346
)
346347

348+
// Alibaba DashScope voice-enrollment (Qwen-TTS voice cloning management). Fork-only, non-OpenAI
349+
// native endpoint. A single URL hosts create_voice / list_voice / query_voice / update_voice /
350+
// delete_voice via `input.action`. Body flows through the translator unchanged; only x-ai-eg-model
351+
// is populated from body.model for route matching.
352+
voiceEnrollmentMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationSpeech)
353+
server.Register(
354+
path.Join(flags.rootPrefix, "/api/v1/services/audio/tts/customization"),
355+
extproc.NewFactory(
356+
voiceEnrollmentMetricsFactory,
357+
tracingapi.NoopTracer[dashscopeschema.VoiceEnrollmentRequest, struct{}, struct{}]{},
358+
endpointspec.DashScopeVoiceEnrollmentEndpointSpec{},
359+
),
360+
)
361+
347362
// Gemini cachedContents API for explicit context caching (POST/GET/PATCH/DELETE).
348363
// Shares the /v1/projects/ prefix with generateContent above; server.go falls through
349364
// from generateContent's pathExtract (which returns nil for non-/models/ paths) to this entry.

internal/apischema/dashscope/dashscope.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,34 @@ type SpeechOutput struct {
6262
Audio SpeechOutputAudio `json:"audio"`
6363
}
6464

65+
// VoiceEnrollmentRequest is the request body for POST /api/v1/services/audio/tts/customization —
66+
// the DashScope voice-cloning management endpoint. A single URL hosts all five actions
67+
// (create_voice, list_voice, query_voice, update_voice, delete_voice); which one is executed
68+
// is decided by `input.action` in the body.
69+
//
70+
// The translator only needs `model` (routing) and `input.action` (observability/metrics); every
71+
// other field flows through as raw JSON. Callers speak DashScope's native schema — this
72+
// endpoint has no OpenAI counterpart to translate against.
73+
//
74+
// https://www.alibabacloud.com/help/en/model-studio/voice-clone-design-http-api
75+
type VoiceEnrollmentRequest struct {
76+
// Model is always "voice-enrollment" for this endpoint; kept in the parsed request so the
77+
// gateway can populate x-ai-eg-model for route matching.
78+
Model string `json:"model"`
79+
// Input carries the per-action payload. Only `action` is inspected here for logging /
80+
// metrics tagging; the remaining fields (target_model, prefix, url, voice_id, …) are left
81+
// to flow through as raw JSON.
82+
Input VoiceEnrollmentInput `json:"input"`
83+
}
84+
85+
// VoiceEnrollmentInput mirrors the `input` object of a voice-enrollment request. Only the
86+
// action tag is modelled — everything else is per-action noise that the gateway forwards
87+
// unchanged.
88+
type VoiceEnrollmentInput struct {
89+
// Action selects the operation: create_voice | list_voice | query_voice | update_voice | delete_voice.
90+
Action string `json:"action"`
91+
}
92+
6593
// SpeechOutputAudio is the audio pointer within a DashScope response.
6694
//
6795
// Note: DashScope's non-streaming reply also carries an `expires_at` field encoded as a Unix

internal/endpointspec/endpointspec.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/envoyproxy/ai-gateway/internal/apischema/anthropic"
2323
cohereschema "github.com/envoyproxy/ai-gateway/internal/apischema/cohere"
24+
dashscopeschema "github.com/envoyproxy/ai-gateway/internal/apischema/dashscope"
2425
gcpschema "github.com/envoyproxy/ai-gateway/internal/apischema/gcp"
2526
"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
2627
"github.com/envoyproxy/ai-gateway/internal/filterapi"
@@ -124,6 +125,13 @@ type (
124125
// Used for explicit context cache CRUD (POST/GET/PATCH/DELETE). The request body is parsed
125126
// best-effort: POST/PATCH carry a CachedContentRequest, GET/DELETE carry no body.
126127
GeminiCachedContentsEndpointSpec struct{}
128+
// DashScopeVoiceEnrollmentEndpointSpec implements EndpointSpec for Alibaba DashScope's
129+
// voice-cloning management endpoint (POST /api/v1/services/audio/tts/customization). This
130+
// is a fork-only, non-OpenAI native endpoint; a single URL hosts five actions
131+
// (create_voice, list_voice, query_voice, update_voice, delete_voice) chosen via
132+
// input.action. The gateway forwards the body unchanged; only body.model is parsed so route
133+
// matching can key off x-ai-eg-model=voice-enrollment.
134+
DashScopeVoiceEnrollmentEndpointSpec struct{}
127135
// TranscriptionEndpointSpec implements EndpointSpec for /v1/audio/transcriptions.
128136
TranscriptionEndpointSpec struct{}
129137
// TranslationEndpointSpec implements EndpointSpec for /v1/audio/translations.
@@ -599,6 +607,46 @@ func (GeminiCachedContentsEndpointSpec) RedactSensitiveInfoFromRequest(req *gcps
599607
return req, nil
600608
}
601609

610+
// ParseBody implements [Spec.ParseBody] for DashScope voice-enrollment. Only body.model and
611+
// body.input.action are pulled out (used for x-ai-eg-model routing and metrics tagging); the
612+
// rest of the payload varies per action and flows through as raw JSON.
613+
func (DashScopeVoiceEnrollmentEndpointSpec) ParseBody(
614+
body []byte,
615+
_ bool,
616+
) (internalapi.OriginalModel, *dashscopeschema.VoiceEnrollmentRequest, bool, []byte, error) {
617+
var req dashscopeschema.VoiceEnrollmentRequest
618+
if len(body) > 0 {
619+
if err := json.Unmarshal(body, &req); err != nil {
620+
return "", nil, false, nil, fmt.Errorf("%w: failed to parse JSON for voice enrollment: %w", internalapi.ErrMalformedRequest, err)
621+
}
622+
}
623+
// voice-enrollment is never streamed.
624+
return req.Model, &req, false, nil, nil
625+
}
626+
627+
// ParseMultipartBody implements [Spec.ParseMultipartBody]. voice-enrollment is JSON-only.
628+
func (DashScopeVoiceEnrollmentEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi.OriginalModel, *dashscopeschema.VoiceEnrollmentRequest, bool, []byte, error) {
629+
return "", nil, false, nil, errMultipartNotSupported
630+
}
631+
632+
// GetTranslator implements [EndpointSpec.GetTranslator]. Only AlibabaDashScope is supported;
633+
// there is no OpenAI-shaped counterpart for voice-cloning management, so this is fork-only.
634+
func (DashScopeVoiceEnrollmentEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, _ string) (translator.DashScopeVoiceEnrollmentTranslator, error) {
635+
switch schema.Name {
636+
case filterapi.APISchemaAlibabaDashScope:
637+
return translator.NewDashScopeVoiceEnrollmentTranslator(), nil
638+
default:
639+
return nil, fmt.Errorf("unsupported API schema for voice enrollment: backend=%s", schema)
640+
}
641+
}
642+
643+
// RedactSensitiveInfoFromRequest implements [EndpointSpec.RedactSensitiveInfoFromRequest].
644+
// voice-enrollment carries only opaque control fields (voice_id, prefix, action, remote audio
645+
// URL); nothing needs redaction for debug logs.
646+
func (DashScopeVoiceEnrollmentEndpointSpec) RedactSensitiveInfoFromRequest(req *dashscopeschema.VoiceEnrollmentRequest) (redactedReq *dashscopeschema.VoiceEnrollmentRequest, err error) {
647+
return req, nil
648+
}
649+
602650
// redactMessage redacts sensitive content from a chat message while preserving its type and structure.
603651
// This dispatches to role-specific redaction functions based on the message type.
604652
func redactMessage(msg openai.ChatCompletionMessageParamUnion) openai.ChatCompletionMessageParamUnion {
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Copyright Envoy AI Gateway Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
// The full text of the Apache license is available in the LICENSE file at
4+
// the root of the repo.
5+
6+
package translator
7+
8+
import (
9+
"io"
10+
"strconv"
11+
12+
"github.com/envoyproxy/ai-gateway/internal/apischema/dashscope"
13+
"github.com/envoyproxy/ai-gateway/internal/internalapi"
14+
"github.com/envoyproxy/ai-gateway/internal/metrics"
15+
"github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi"
16+
)
17+
18+
// dashScopeVoiceEnrollmentPath is DashScope's native voice-cloning endpoint. A single URL
19+
// hosts every action (create_voice / list_voice / query_voice / update_voice / delete_voice);
20+
// the action is chosen in the request body.
21+
//
22+
// https://www.alibabacloud.com/help/en/model-studio/voice-clone-design-http-api
23+
const dashScopeVoiceEnrollmentPath = "/api/v1/services/audio/tts/customization"
24+
25+
// DashScopeVoiceEnrollmentSpan is a zero-value span for voice-enrollment. There is no
26+
// per-request response object to record — the endpoint returns short JSON envelopes
27+
// (voice_id, status, page_index, …) that we forward untouched.
28+
type DashScopeVoiceEnrollmentSpan = tracingapi.Span[struct{}, struct{}]
29+
30+
// DashScopeVoiceEnrollmentTranslator translates DashScope voice-enrollment requests as a
31+
// pass-through. It exists so the gateway can attach auth/logging/RBAC to the DashScope
32+
// management surface without reshaping the payload.
33+
type DashScopeVoiceEnrollmentTranslator = Translator[dashscope.VoiceEnrollmentRequest, DashScopeVoiceEnrollmentSpan]
34+
35+
// dashScopeVoiceEnrollmentTranslator forwards voice-enrollment CRUD calls to DashScope
36+
// unchanged. The body varies per action (create takes `url`+`prefix`, delete takes only
37+
// `voice_id`, list is paginated, etc.) so any translator-side reshaping would just add
38+
// friction; we forward the caller's payload verbatim and only rewrite `:path`.
39+
type dashScopeVoiceEnrollmentTranslator struct{}
40+
41+
// NewDashScopeVoiceEnrollmentTranslator creates a pass-through translator for voice-enrollment.
42+
func NewDashScopeVoiceEnrollmentTranslator() DashScopeVoiceEnrollmentTranslator {
43+
return &dashScopeVoiceEnrollmentTranslator{}
44+
}
45+
46+
// RequestBody implements [DashScopeVoiceEnrollmentTranslator.RequestBody].
47+
//
48+
// Body flows through untouched — DashScope owns the schema per action, and modelling every
49+
// variant here would just track upstream drift. The router already parsed body.model /
50+
// body.input.action for logging + routing, so nothing else is required. We only need to
51+
// re-emit the original bytes when Envoy's upstream stage forces a body replacement (retry /
52+
// streaming), and to rewrite `:path` to the DashScope native URL because Envoy is otherwise
53+
// still sitting on the client's inbound path.
54+
func (dashScopeVoiceEnrollmentTranslator) RequestBody(original []byte, _ *dashscope.VoiceEnrollmentRequest, forceBodyMutation bool) (
55+
newHeaders []internalapi.Header, newBody []byte, err error,
56+
) {
57+
newHeaders = []internalapi.Header{{pathHeaderName, dashScopeVoiceEnrollmentPath}}
58+
if forceBodyMutation && len(original) > 0 {
59+
newBody = original
60+
newHeaders = append(newHeaders, internalapi.Header{contentLengthHeaderName, strconv.Itoa(len(newBody))})
61+
}
62+
return
63+
}
64+
65+
// ResponseHeaders implements [DashScopeVoiceEnrollmentTranslator.ResponseHeaders].
66+
// DashScope returns application/json envelopes and the caller expects that shape, so we do
67+
// not touch response headers.
68+
func (dashScopeVoiceEnrollmentTranslator) ResponseHeaders(_ map[string]string) (newHeaders []internalapi.Header, err error) {
69+
return nil, nil
70+
}
71+
72+
// ResponseBody implements [DashScopeVoiceEnrollmentTranslator.ResponseBody]. Passes DashScope's
73+
// JSON response through unchanged; the action-specific envelope (`voice_id`, `voice_list`,
74+
// resource_link, status, …) is the caller's contract, not the gateway's to reshape.
75+
func (dashScopeVoiceEnrollmentTranslator) ResponseBody(_ map[string]string, _ io.Reader, _ bool, _ DashScopeVoiceEnrollmentSpan) (
76+
newHeaders []internalapi.Header, newBody []byte, tokenUsage metrics.TokenUsage, responseModel internalapi.ResponseModel, err error,
77+
) {
78+
return nil, nil, metrics.TokenUsage{}, "voice-enrollment", nil
79+
}
80+
81+
// ResponseError implements [DashScopeVoiceEnrollmentTranslator.ResponseError]. DashScope's
82+
// error envelope (`code`, `message`) is passed through as-is — no OpenAI-shaped rewriting is
83+
// meaningful for a fork-only native endpoint.
84+
func (dashScopeVoiceEnrollmentTranslator) ResponseError(_ map[string]string, _ io.Reader) (newHeaders []internalapi.Header, newBody []byte, err error) {
85+
return nil, nil, nil
86+
}
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Copyright Envoy AI Gateway Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
// The full text of the Apache license is available in the LICENSE file at
4+
// the root of the repo.
5+
6+
package translator
7+
8+
import (
9+
"strings"
10+
"testing"
11+
12+
"github.com/stretchr/testify/require"
13+
14+
"github.com/envoyproxy/ai-gateway/internal/apischema/dashscope"
15+
)
16+
17+
func TestDashScopeVoiceEnrollment_RequestBody(t *testing.T) {
18+
tr := NewDashScopeVoiceEnrollmentTranslator()
19+
req := &dashscope.VoiceEnrollmentRequest{
20+
Model: "voice-enrollment",
21+
Input: dashscope.VoiceEnrollmentInput{Action: "create_voice"},
22+
}
23+
original := []byte(`{"model":"voice-enrollment","input":{"action":"create_voice","target_model":"qwen3-tts-flash","prefix":"pk","url":"https://example.com/a.wav"}}`)
24+
25+
t.Run("path is rewritten, body is not replayed by default", func(t *testing.T) {
26+
hm, body, err := tr.RequestBody(original, req, false)
27+
require.NoError(t, err)
28+
require.Empty(t, body, "no forced mutation → translator should not carry the body itself")
29+
30+
var pathVal string
31+
for _, h := range hm {
32+
if h.Key() == ":path" {
33+
pathVal = h.Value()
34+
}
35+
}
36+
require.Equal(t, dashScopeVoiceEnrollmentPath, pathVal)
37+
})
38+
39+
t.Run("forceBodyMutation replays the original body verbatim", func(t *testing.T) {
40+
hm, body, err := tr.RequestBody(original, req, true)
41+
require.NoError(t, err)
42+
require.Equal(t, original, body, "forced replay must preserve caller's payload byte-for-byte")
43+
44+
var cl string
45+
for _, h := range hm {
46+
if h.Key() == "content-length" {
47+
cl = h.Value()
48+
}
49+
}
50+
require.NotEmpty(t, cl)
51+
})
52+
}
53+
54+
func TestDashScopeVoiceEnrollment_ResponseHeaders(t *testing.T) {
55+
tr := NewDashScopeVoiceEnrollmentTranslator()
56+
hm, err := tr.ResponseHeaders(nil)
57+
require.NoError(t, err)
58+
require.Empty(t, hm, "response headers pass through untouched")
59+
}
60+
61+
func TestDashScopeVoiceEnrollment_ResponseBody(t *testing.T) {
62+
tr := NewDashScopeVoiceEnrollmentTranslator()
63+
envelope := `{"request_id":"rid","output":{"voice_id":"custom-abc"},"usage":{"count":1}}`
64+
hm, body, tokens, respModel, err := tr.ResponseBody(nil, strings.NewReader(envelope), true, nil)
65+
require.NoError(t, err)
66+
require.Empty(t, hm)
67+
require.Empty(t, body, "pass-through: envoy delivers upstream body directly, no mutation")
68+
require.Equal(t, "voice-enrollment", respModel)
69+
_, set := tokens.OutputTokens()
70+
require.False(t, set, "voice-enrollment carries no token accounting")
71+
}
72+
73+
func TestDashScopeVoiceEnrollment_ResponseError(t *testing.T) {
74+
tr := NewDashScopeVoiceEnrollmentTranslator()
75+
hm, body, err := tr.ResponseError(nil, strings.NewReader(`{"code":"InvalidParameter","message":"missing url"}`))
76+
require.NoError(t, err)
77+
require.Empty(t, hm)
78+
require.Empty(t, body, "error envelope passes through unchanged")
79+
}

0 commit comments

Comments
 (0)