Add audio support.

- Add a new task "audio scribe".
- Allow users to record audio clips or pick wav files to interact with model.
- Add support for importing models with audio capability.
- Fix a typo in Settings dialog (Thanks https://github.com/rhnvrm!)

PiperOrigin-RevId: 774832681

Author: Google AI Edge Gallery

Android/src/app/src/main/AndroidManifest.xml

@@ -29,6 +29,7 @@
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC"/>
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
     <uses-permission android:name="android.permission.WAKE_LOCK"/>
 
     <uses-feature

Android/src/app/src/main/java/com/google/ai/edge/gallery/common/Types.kt

@@ -25,3 +25,5 @@ interface LatencyProvider {
 data class Classification(val label: String, val score: Float, val color: Color)
 
 data class JsonObjAndTextContent<T>(val jsonObj: T, val textContent: String)
+
+class AudioClip(val audioData: ByteArray, val sampleRate: Int)

Android/src/app/src/main/java/com/google/ai/edge/gallery/common/Utils.kt

@@ -17,12 +17,17 @@
 package com.google.ai.edge.gallery.common
 
 import android.content.Context
+import android.net.Uri
 import android.util.Log
+import com.google.ai.edge.gallery.data.SAMPLE_RATE
 import com.google.gson.Gson
 import com.google.gson.reflect.TypeToken
 import java.io.File
 import java.net.HttpURLConnection
 import java.net.URL
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import kotlin.math.floor
 
 data class LaunchInfo(val ts: Long)
 
@@ -112,3 +117,135 @@ inline fun <reified T> getJsonResponse(url: String): JsonObjAndTextContent<T>? {
 
   return null
 }
+
+fun convertWavToMonoWithMaxSeconds(
+  context: Context,
+  stereoUri: Uri,
+  maxSeconds: Int = 30,
+): AudioClip? {
+  Log.d(TAG, "Start to convert wav file to mono channel")
+
+  try {
+    val inputStream = context.contentResolver.openInputStream(stereoUri) ?: return null
+    val originalBytes = inputStream.readBytes()
+    inputStream.close()
+
+    // Read WAV header
+    if (originalBytes.size < 44) {
+      // Not a valid WAV file
+      Log.e(TAG, "Not a valid wav file")
+      return null
+    }
+
+    val headerBuffer = ByteBuffer.wrap(originalBytes, 0, 44).order(ByteOrder.LITTLE_ENDIAN)
+    val channels = headerBuffer.getShort(22)
+    var sampleRate = headerBuffer.getInt(24)
+    val bitDepth = headerBuffer.getShort(34)
+    Log.d(TAG, "File metadata: channels: $channels, sampleRate: $sampleRate, bitDepth: $bitDepth")
+
+    // Normalize audio to 16-bit.
+    val audioDataBytes = originalBytes.copyOfRange(fromIndex = 44, toIndex = originalBytes.size)
+    var sixteenBitBytes: ByteArray =
+      if (bitDepth.toInt() == 8) {
+        Log.d(TAG, "Converting 8-bit audio to 16-bit.")
+        convert8BitTo16Bit(audioDataBytes)
+      } else {
+        // Assume 16-bit or other format that can be handled directly
+        audioDataBytes
+      }
+
+    // Convert byte array to short array for processing
+    val shortBuffer =
+      ByteBuffer.wrap(sixteenBitBytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()
+    var pcmSamples = ShortArray(shortBuffer.remaining())
+    shortBuffer.get(pcmSamples)
+
+    // Resample if sample rate is less than 16000 Hz ---
+    if (sampleRate < SAMPLE_RATE) {
+      Log.d(TAG, "Resampling from $sampleRate Hz to $SAMPLE_RATE Hz.")
+      pcmSamples = resample(pcmSamples, sampleRate, SAMPLE_RATE, channels.toInt())
+      sampleRate = SAMPLE_RATE
+      Log.d(TAG, "Resampling complete. New sample count: ${pcmSamples.size}")
+    }
+
+    // Convert stereo to mono if necessary
+    var monoSamples =
+      if (channels.toInt() == 2) {
+        Log.d(TAG, "Converting stereo to mono.")
+        val mono = ShortArray(pcmSamples.size / 2)
+        for (i in mono.indices) {
+          val left = pcmSamples[i * 2]
+          val right = pcmSamples[i * 2 + 1]
+          mono[i] = ((left + right) / 2).toShort()
+        }
+        mono
+      } else {
+        Log.d(TAG, "Audio is already mono. No channel conversion needed.")
+        pcmSamples
+      }
+
+    // Trim the audio to maxSeconds ---
+    val maxSamples = maxSeconds * sampleRate
+    if (monoSamples.size > maxSamples) {
+      Log.d(TAG, "Trimming clip from ${monoSamples.size} samples to $maxSamples samples.")
+      monoSamples = monoSamples.copyOfRange(0, maxSamples)
+    }
+
+    val monoByteBuffer = ByteBuffer.allocate(monoSamples.size * 2).order(ByteOrder.LITTLE_ENDIAN)
+    monoByteBuffer.asShortBuffer().put(monoSamples)
+    return AudioClip(audioData = monoByteBuffer.array(), sampleRate = sampleRate)
+  } catch (e: Exception) {
+    Log.e(TAG, "Failed to convert wav to mono", e)
+    return null
+  }
+}
+
+/** Converts 8-bit unsigned PCM audio data to 16-bit signed PCM. */
+private fun convert8BitTo16Bit(eightBitData: ByteArray): ByteArray {
+  // The new 16-bit data will be twice the size
+  val sixteenBitData = ByteArray(eightBitData.size * 2)
+  val buffer = ByteBuffer.wrap(sixteenBitData).order(ByteOrder.LITTLE_ENDIAN)
+
+  for (byte in eightBitData) {
+    // Convert the unsigned 8-bit byte (0-255) to a signed 16-bit short (-32768 to 32767)
+    // 1. Get the unsigned value by masking with 0xFF
+    // 2. Subtract 128 to center the waveform around 0 (range becomes -128 to 127)
+    // 3. Scale by 256 to expand to the 16-bit range
+    val unsignedByte = byte.toInt() and 0xFF
+    val sixteenBitSample = ((unsignedByte - 128) * 256).toShort()
+    buffer.putShort(sixteenBitSample)
+  }
+  return sixteenBitData
+}
+
+/** Resamples PCM audio data from an original sample rate to a target sample rate. */
+private fun resample(
+  inputSamples: ShortArray,
+  originalSampleRate: Int,
+  targetSampleRate: Int,
+  channels: Int,
+): ShortArray {
+  if (originalSampleRate == targetSampleRate) {
+    return inputSamples
+  }
+
+  val ratio = targetSampleRate.toDouble() / originalSampleRate
+  val outputLength = (inputSamples.size * ratio).toInt()
+  val resampledData = ShortArray(outputLength)
+
+  if (channels == 1) { // Mono
+    for (i in resampledData.indices) {
+      val position = i / ratio
+      val index1 = floor(position).toInt()
+      val index2 = index1 + 1
+      val fraction = position - index1
+
+      val sample1 = if (index1 < inputSamples.size) inputSamples[index1].toDouble() else 0.0
+      val sample2 = if (index2 < inputSamples.size) inputSamples[index2].toDouble() else 0.0
+
+      resampledData[i] = (sample1 * (1 - fraction) + sample2 * fraction).toInt().toShort()
+    }
+  }
+
+  return resampledData
+}

Android/src/app/src/main/java/com/google/ai/edge/gallery/data/Config.kt

@@ -50,6 +50,7 @@ enum class ConfigKey(val label: String) {
   DEFAULT_TOPP("Default TopP"),
   DEFAULT_TEMPERATURE("Default temperature"),
   SUPPORT_IMAGE("Support image"),
+  SUPPORT_AUDIO("Support audio"),
   MAX_RESULT_COUNT("Max result count"),
   USE_GPU("Use GPU"),
   ACCELERATOR("Choose accelerator"),

Android/src/app/src/main/java/com/google/ai/edge/gallery/data/Consts.kt

@@ -44,3 +44,12 @@ val DEFAULT_ACCELERATORS = listOf(Accelerator.GPU)
 
 // Max number of images allowed in a "ask image" session.
 const val MAX_IMAGE_COUNT = 10
+
+// Max number of audio clip in an "ask audio" session.
+const val MAX_AUDIO_CLIP_COUNT = 10
+
+// Max audio clip duration in seconds.
+const val MAX_AUDIO_CLIP_DURATION_SEC = 30
+
+// Audio-recording related consts.
+const val SAMPLE_RATE = 16000

Android/src/app/src/main/java/com/google/ai/edge/gallery/data/Model.kt

@@ -87,6 +87,9 @@ data class Model(
   /** Whether the LLM model supports image input. */
   val llmSupportImage: Boolean = false,
 
+  /** Whether the LLM model supports audio input. */
+  val llmSupportAudio: Boolean = false,
+
   /** Whether the model is imported or not. */
   val imported: Boolean = false,
 

Android/src/app/src/main/java/com/google/ai/edge/gallery/data/ModelAllowlist.kt

@@ -38,6 +38,7 @@ data class AllowedModel(
   val taskTypes: List<String>,
   val disabled: Boolean? = null,
   val llmSupportImage: Boolean? = null,
+  val llmSupportAudio: Boolean? = null,
   val estimatedPeakMemoryInBytes: Long? = null,
 ) {
   fun toModel(): Model {
@@ -96,6 +97,7 @@ data class AllowedModel(
       showRunAgainButton = showRunAgainButton,
       learnMoreUrl = "https://huggingface.co/${modelId}",
       llmSupportImage = llmSupportImage == true,
+      llmSupportAudio = llmSupportAudio == true,
     )
   }
 

Android/src/app/src/main/java/com/google/ai/edge/gallery/data/Tasks.kt

@@ -17,19 +17,22 @@
 package com.google.ai.edge.gallery.data
 
 import androidx.annotation.StringRes
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.outlined.Forum
+import androidx.compose.material.icons.outlined.Mic
+import androidx.compose.material.icons.outlined.Mms
+import androidx.compose.material.icons.outlined.Widgets
 import androidx.compose.runtime.MutableState
 import androidx.compose.runtime.mutableLongStateOf
 import androidx.compose.ui.graphics.vector.ImageVector
 import com.google.ai.edge.gallery.R
-import com.google.ai.edge.gallery.ui.icon.Forum
-import com.google.ai.edge.gallery.ui.icon.Mms
-import com.google.ai.edge.gallery.ui.icon.Widgets
 
 /** Type of task. */
 enum class TaskType(val label: String, val id: String) {
   LLM_CHAT(label = "AI Chat", id = "llm_chat"),
   LLM_PROMPT_LAB(label = "Prompt Lab", id = "llm_prompt_lab"),
   LLM_ASK_IMAGE(label = "Ask Image", id = "llm_ask_image"),
+  LLM_ASK_AUDIO(label = "Audio Scribe", id = "llm_ask_audio"),
   TEST_TASK_1(label = "Test task 1", id = "test_task_1"),
   TEST_TASK_2(label = "Test task 2", id = "test_task_2"),
 }
@@ -71,7 +74,7 @@ data class Task(
 val TASK_LLM_CHAT =
   Task(
     type = TaskType.LLM_CHAT,
-    icon = Forum,
+    icon = Icons.Outlined.Forum,
     models = mutableListOf(),
     description = "Chat with on-device large language models",
     docUrl = "https://ai.google.dev/edge/mediapipe/solutions/genai/llm_inference/android",
@@ -83,7 +86,7 @@ val TASK_LLM_CHAT =
 val TASK_LLM_PROMPT_LAB =
   Task(
     type = TaskType.LLM_PROMPT_LAB,
-    icon = Widgets,
+    icon = Icons.Outlined.Widgets,
     models = mutableListOf(),
     description = "Single turn use cases with on-device large language model",
     docUrl = "https://ai.google.dev/edge/mediapipe/solutions/genai/llm_inference/android",
@@ -95,7 +98,7 @@ val TASK_LLM_PROMPT_LAB =
 val TASK_LLM_ASK_IMAGE =
   Task(
     type = TaskType.LLM_ASK_IMAGE,
-    icon = Mms,
+    icon = Icons.Outlined.Mms,
     models = mutableListOf(),
     description = "Ask questions about images with on-device large language models",
     docUrl = "https://ai.google.dev/edge/mediapipe/solutions/genai/llm_inference/android",
@@ -104,8 +107,23 @@ val TASK_LLM_ASK_IMAGE =
     textInputPlaceHolderRes = R.string.text_input_placeholder_llm_chat,
   )
 
+val TASK_LLM_ASK_AUDIO =
+  Task(
+    type = TaskType.LLM_ASK_AUDIO,
+    icon = Icons.Outlined.Mic,
+    models = mutableListOf(),
+    // TODO(do not submit)
+    description =
+      "Instantly transcribe and/or translate audio clips using on-device large language models",
+    docUrl = "https://ai.google.dev/edge/mediapipe/solutions/genai/llm_inference/android",
+    sourceCodeUrl =
+      "https://github.com/google-ai-edge/gallery/blob/main/Android/src/app/src/main/java/com/google/ai/edge/gallery/ui/llmchat/LlmChatModelHelper.kt",
+    textInputPlaceHolderRes = R.string.text_input_placeholder_llm_chat,
+  )
+
 /** All tasks. */
-val TASKS: List<Task> = listOf(TASK_LLM_ASK_IMAGE, TASK_LLM_PROMPT_LAB, TASK_LLM_CHAT)
+val TASKS: List<Task> =
+  listOf(TASK_LLM_ASK_IMAGE, TASK_LLM_ASK_AUDIO, TASK_LLM_PROMPT_LAB, TASK_LLM_CHAT)
 
 fun getModelByName(name: String): Model? {
   for (task in TASKS) {

Android/src/app/src/main/java/com/google/ai/edge/gallery/ui/ViewModelProvider.kt

@@ -21,6 +21,7 @@ import androidx.lifecycle.viewmodel.CreationExtras
 import androidx.lifecycle.viewmodel.initializer
 import androidx.lifecycle.viewmodel.viewModelFactory
 import com.google.ai.edge.gallery.GalleryApplication
+import com.google.ai.edge.gallery.ui.llmchat.LlmAskAudioViewModel
 import com.google.ai.edge.gallery.ui.llmchat.LlmAskImageViewModel
 import com.google.ai.edge.gallery.ui.llmchat.LlmChatViewModel
 import com.google.ai.edge.gallery.ui.llmsingleturn.LlmSingleTurnViewModel
@@ -49,6 +50,9 @@ object ViewModelProvider {
 
     // Initializer for LlmAskImageViewModel.
     initializer { LlmAskImageViewModel() }
+
+    // Initializer for LlmAskAudioViewModel.
+    initializer { LlmAskAudioViewModel() }
   }
 }