feat(ai): add instance AI providers and transcription (#5829)

Co-authored-by: memoclaw <265580040+memoclaw@users.noreply.github.com>

Author: memoclaw

internal/ai/ai.go

@@ -0,0 +1,26 @@
+package ai
+
+// ProviderType identifies an AI provider implementation.
+type ProviderType string
+
+const (
+	// ProviderOpenAI is OpenAI's hosted API.
+	ProviderOpenAI ProviderType = "OPENAI"
+	// ProviderOpenAICompatible is an OpenAI-compatible API endpoint.
+	ProviderOpenAICompatible ProviderType = "OPENAI_COMPATIBLE"
+	// ProviderAnthropic is Anthropic's API.
+	ProviderAnthropic ProviderType = "ANTHROPIC"
+	// ProviderGemini is Google's Gemini API.
+	ProviderGemini ProviderType = "GEMINI"
+)
+
+// ProviderConfig configures a callable AI provider connection.
+type ProviderConfig struct {
+	ID           string
+	Title        string
+	Type         ProviderType
+	Endpoint     string
+	APIKey       string
+	Models       []string
+	DefaultModel string
+}

internal/ai/errors.go

@@ -0,0 +1,10 @@
+package ai
+
+import "github.com/pkg/errors"
+
+var (
+	// ErrProviderNotFound indicates that a requested provider ID does not exist.
+	ErrProviderNotFound = errors.New("AI provider not found")
+	// ErrCapabilityUnsupported indicates that the provider does not support the requested capability.
+	ErrCapabilityUnsupported = errors.New("AI provider capability unsupported")
+)

internal/ai/openai/client.go

@@ -0,0 +1,59 @@
+package openai
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/usememos/memos/internal/ai"
+)
+
+const defaultEndpoint = "https://api.openai.com/v1"
+
+// Transcriber transcribes audio with OpenAI-compatible transcription APIs.
+type Transcriber struct {
+	endpoint   string
+	apiKey     string
+	httpClient *http.Client
+}
+
+// NewTranscriber creates a new OpenAI-compatible transcriber.
+func NewTranscriber(config ai.ProviderConfig, options ...Option) (*Transcriber, error) {
+	endpoint := strings.TrimSpace(config.Endpoint)
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	if _, err := url.ParseRequestURI(endpoint); err != nil {
+		return nil, errors.Wrap(err, "invalid OpenAI endpoint")
+	}
+	if config.APIKey == "" {
+		return nil, errors.New("OpenAI API key is required")
+	}
+
+	transcriber := &Transcriber{
+		endpoint: endpoint,
+		apiKey:   config.APIKey,
+		httpClient: &http.Client{
+			Timeout: 2 * time.Minute,
+		},
+	}
+	for _, option := range options {
+		option(transcriber)
+	}
+	return transcriber, nil
+}
+
+// Option configures a Transcriber.
+type Option func(*Transcriber)
+
+// WithHTTPClient sets the HTTP client used by the transcriber.
+func WithHTTPClient(client *http.Client) Option {
+	return func(t *Transcriber) {
+		if client != nil {
+			t.httpClient = client
+		}
+	}
+}

internal/ai/openai/transcription.go

@@ -0,0 +1,145 @@
+package openai
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/usememos/memos/internal/ai"
+)
+
+type transcriptionResponse struct {
+	Text     string  `json:"text"`
+	Language string  `json:"language"`
+	Duration float64 `json:"duration"`
+}
+
+type errorResponse struct {
+	Error struct {
+		Message string `json:"message"`
+		Type    string `json:"type"`
+		Code    string `json:"code"`
+	} `json:"error"`
+}
+
+// Transcribe transcribes audio with the /audio/transcriptions endpoint.
+func (t *Transcriber) Transcribe(ctx context.Context, request ai.TranscribeRequest) (*ai.TranscribeResponse, error) {
+	if strings.TrimSpace(request.Model) == "" {
+		return nil, errors.New("model is required")
+	}
+	if request.Audio == nil {
+		return nil, errors.New("audio is required")
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	if err := writeAudioFilePart(writer, request); err != nil {
+		return nil, err
+	}
+	if err := writer.WriteField("model", request.Model); err != nil {
+		return nil, errors.Wrap(err, "failed to write model field")
+	}
+	if err := writer.WriteField("response_format", "json"); err != nil {
+		return nil, errors.Wrap(err, "failed to write response format field")
+	}
+	if request.Prompt != "" {
+		if err := writer.WriteField("prompt", request.Prompt); err != nil {
+			return nil, errors.Wrap(err, "failed to write prompt field")
+		}
+	}
+	if request.Language != "" {
+		if err := writer.WriteField("language", request.Language); err != nil {
+			return nil, errors.Wrap(err, "failed to write language field")
+		}
+	}
+	if err := writer.Close(); err != nil {
+		return nil, errors.Wrap(err, "failed to close multipart writer")
+	}
+
+	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(t.endpoint, "/")+"/audio/transcriptions", body)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create transcription request")
+	}
+	httpRequest.Header.Set("Authorization", "Bearer "+t.apiKey)
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	httpResponse, err := t.httpClient.Do(httpRequest)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to send transcription request")
+	}
+	defer httpResponse.Body.Close()
+
+	responseBody, err := io.ReadAll(httpResponse.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read transcription response")
+	}
+	if httpResponse.StatusCode < http.StatusOK || httpResponse.StatusCode >= http.StatusMultipleChoices {
+		return nil, errors.Errorf("transcription request failed with status %d: %s", httpResponse.StatusCode, extractErrorMessage(responseBody))
+	}
+
+	var response transcriptionResponse
+	if err := json.Unmarshal(responseBody, &response); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal transcription response")
+	}
+	return &ai.TranscribeResponse{
+		Text:     response.Text,
+		Language: response.Language,
+		Duration: response.Duration,
+	}, nil
+}
+
+func writeAudioFilePart(writer *multipart.Writer, request ai.TranscribeRequest) error {
+	filename := strings.TrimSpace(request.Filename)
+	if filename == "" {
+		filename = "audio"
+	}
+	contentType := strings.TrimSpace(request.ContentType)
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	} else {
+		mediaType, _, err := mime.ParseMediaType(contentType)
+		if err != nil {
+			return errors.Wrap(err, "invalid audio content type")
+		}
+		contentType = mediaType
+	}
+
+	header := make(textproto.MIMEHeader)
+	header.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{
+		"name":     "file",
+		"filename": sanitizeFilename(filename),
+	}))
+	header.Set("Content-Type", contentType)
+	part, err := writer.CreatePart(header)
+	if err != nil {
+		return errors.Wrap(err, "failed to create audio file part")
+	}
+	if _, err := io.Copy(part, request.Audio); err != nil {
+		return errors.Wrap(err, "failed to write audio file part")
+	}
+	return nil
+}
+
+func extractErrorMessage(responseBody []byte) string {
+	var response errorResponse
+	if err := json.Unmarshal(responseBody, &response); err == nil && response.Error.Message != "" {
+		return response.Error.Message
+	}
+	return string(responseBody)
+}
+
+func sanitizeFilename(filename string) string {
+	filename = strings.NewReplacer("\r", "_", "\n", "_").Replace(filename)
+	if strings.TrimSpace(filename) == "" {
+		return "audio"
+	}
+	return filename
+}

internal/ai/openai/transcription_test.go

@@ -0,0 +1,65 @@
+package openai
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/internal/ai"
+)
+
+func TestTranscribe(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, http.MethodPost, r.Method)
+		require.Equal(t, "/audio/transcriptions", r.URL.Path)
+		require.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
+		require.NoError(t, r.ParseMultipartForm(10<<20))
+		require.Equal(t, "gpt-4o-transcribe", r.FormValue("model"))
+		require.Equal(t, "json", r.FormValue("response_format"))
+		require.Equal(t, "domain words", r.FormValue("prompt"))
+		require.Equal(t, "en", r.FormValue("language"))
+
+		file, header, err := r.FormFile("file")
+		require.NoError(t, err)
+		defer file.Close()
+		require.Equal(t, "voice.wav", header.Filename)
+		require.Equal(t, "audio/wav", header.Header.Get("Content-Type"))
+
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"text":     "hello world",
+			"language": "en",
+			"duration": 1.5,
+		}))
+	}))
+	defer server.Close()
+
+	transcriber, err := NewTranscriber(ai.ProviderConfig{
+		Endpoint: server.URL,
+		APIKey:   "test-key",
+	})
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	response, err := transcriber.Transcribe(ctx, ai.TranscribeRequest{
+		Model:       "gpt-4o-transcribe",
+		Filename:    "voice.wav",
+		ContentType: "audio/wav",
+		Audio:       strings.NewReader("RIFF"),
+		Prompt:      "domain words",
+		Language:    "en",
+	})
+	require.NoError(t, err)
+	require.Equal(t, "hello world", response.Text)
+	require.Equal(t, "en", response.Language)
+	require.Equal(t, 1.5, response.Duration)
+}

internal/ai/resolver.go

@@ -0,0 +1,16 @@
+package ai
+
+import "github.com/pkg/errors"
+
+// FindProvider returns the provider with the given ID.
+func FindProvider(providers []ProviderConfig, providerID string) (*ProviderConfig, error) {
+	if providerID == "" {
+		return nil, errors.Wrap(ErrProviderNotFound, "provider ID is required")
+	}
+	for _, provider := range providers {
+		if provider.ID == providerID {
+			return &provider, nil
+		}
+	}
+	return nil, errors.Wrapf(ErrProviderNotFound, "provider ID %q", providerID)
+}

internal/ai/transcription.go

@@ -0,0 +1,29 @@
+package ai
+
+import (
+	"context"
+	"io"
+)
+
+// Transcriber transcribes audio into text.
+type Transcriber interface {
+	Transcribe(ctx context.Context, request TranscribeRequest) (*TranscribeResponse, error)
+}
+
+// TranscribeRequest contains an audio transcription request.
+type TranscribeRequest struct {
+	Model       string
+	Filename    string
+	ContentType string
+	Audio       io.Reader
+	Size        int64
+	Prompt      string
+	Language    string
+}
+
+// TranscribeResponse contains an audio transcription response.
+type TranscribeResponse struct {
+	Text     string
+	Language string
+	Duration float64
+}