Muted talker detection

Author: hiroshihorie

modules/audio_device/audio_engine_device.h

@@ -126,6 +126,7 @@ class AudioEngineDevice : public AudioDeviceGeneric,
 
   bool IsInterrupted();
 
+  int32_t SetObserver(AudioDeviceObserver* observer) override;
 
  private:
   struct EngineState {
@@ -184,6 +185,8 @@ class AudioEngineDevice : public AudioDeviceGeneric,
   // Set to true after successful call to Init(), false otherwise.
   bool initialized_ RTC_GUARDED_BY(thread_);
 
+  AudioDeviceObserver* observer_ RTC_GUARDED_BY(thread_);
+
   // Audio interruption observer instance.
   RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter) * audio_session_observer_
       RTC_GUARDED_BY(thread_);

modules/audio_device/audio_engine_device.mm

@@ -673,6 +673,14 @@
   return 0;
 }
 
+int32_t AudioEngineDevice::SetObserver(AudioDeviceObserver* observer) {
+  LOGI() << "SetObserver";
+  RTC_DCHECK_RUN_ON(thread_);
+
+  observer_ = observer;
+  return 0;
+}
+
 // ----------------------------------------------------------------------------------------------------
 // Private - Engine Related
 
@@ -862,6 +870,32 @@
       }
       LOGI() << "setVoiceProcessingEnabled (input) result: " << set_input_vp_result ? "YES" : "NO";
 
+      // Muted talker detection.
+      if (@available(iOS 17.0, macCatalyst 17.0, macOS 14.0, tvOS 17.0, visionOS 1.0, *)) {
+        auto listener_block = ^(AVAudioVoiceProcessingSpeechActivityEvent event) {
+          LOGI() << "AVAudioVoiceProcessingSpeechActivityEvent: " << event;
+          RTC_DCHECK(event == AVAudioVoiceProcessingSpeechActivityStarted ||
+                     event == AVAudioVoiceProcessingSpeechActivityEnded);
+          AudioDeviceModule::SpeechActivityEvent rtc_event =
+              (event == AVAudioVoiceProcessingSpeechActivityStarted
+                   ? AudioDeviceModule::SpeechActivityEvent::kStarted
+                   : AudioDeviceModule::SpeechActivityEvent::kEnded);
+
+          thread_->PostTask(SafeTask(safety_, [this, rtc_event] {
+            RTC_DCHECK_RUN_ON(thread_);  // Silence warning.
+            if (this->observer_ != nullptr) {
+              this->observer_->OnSpeechActivityEvent(rtc_event);
+            }
+          }));
+        };
+
+        BOOL set_listener_result =
+            [audio_engine_.inputNode setMutedSpeechActivityEventListener:listener_block];
+        LOGI() << "setMutedSpeechActivityEventListener result: " << set_listener_result ? "YES"
+                                                                                        : "NO";
+        RTC_DCHECK(set_listener_result);
+      }
+
       // Other audio ducking.
       // iOS 17.0+, iPadOS 17.0+, Mac Catalyst 17.0+, macOS 14.0+, visionOS 1.0+
       if (@available(iOS 17.0, macCatalyst 17.0, macOS 14.0, visionOS 1.0, *)) {

modules/audio_device/include/audio_device.h

@@ -20,14 +20,7 @@
 namespace webrtc {
 
 class AudioDeviceModuleForTest;
-
-class AudioDeviceObserver {
- public:
-  virtual ~AudioDeviceObserver() = default;
-
-  // input/output devices updated or default device changed
-  virtual void OnDevicesUpdated() = 0;
-};
+class AudioDeviceObserver;
 
 class AudioDeviceModule : public rtc::RefCountInterface {
  public:
@@ -50,6 +43,11 @@ class AudioDeviceModule : public rtc::RefCountInterface {
     kDefaultDevice = -2
   };
 
+  enum SpeechActivityEvent {
+    kStarted = 0,
+    kEnded,
+  };
+
   struct Stats {
     // The fields below correspond to similarly-named fields in the WebRTC stats
     // spec. https://w3c.github.io/webrtc-stats/#playoutstats-dict*
@@ -203,6 +201,16 @@ class AudioDeviceModuleForTest : public AudioDeviceModule {
   virtual int SetRecordingSampleRate(uint32_t sample_rate) = 0;
 };
 
+class AudioDeviceObserver {
+ public:
+  virtual ~AudioDeviceObserver() = default;
+
+  // input/output devices updated or default device changed
+  virtual void OnDevicesUpdated() {}
+  virtual void OnSpeechActivityEvent(
+      AudioDeviceModule::SpeechActivityEvent event) {}
+};
+
 }  // namespace webrtc
 
 #endif  // MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_

sdk/objc/api/peerconnection/RTCAudioDeviceModule.h

@@ -22,7 +22,13 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
-typedef void (^RTCOnAudioDevicesDidUpdate)();
+typedef NS_ENUM(NSInteger, RTCSpeechActivityEvent) {
+  RTCSpeechActivityEventStarted,
+  RTCSpeechActivityEventEnded,
+};
+
+typedef void (^RTCDevicesDidUpdateCallback)();
+typedef void (^RTCSpeechActivityCallback)(RTCSpeechActivityEvent);
 
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCAudioDeviceModule) : NSObject
@@ -42,7 +48,8 @@ RTC_OBJC_EXPORT
 - (BOOL)trySetOutputDevice:(nullable RTC_OBJC_TYPE(RTCIODevice) *)device;
 - (BOOL)trySetInputDevice:(nullable RTC_OBJC_TYPE(RTCIODevice) *)device;
 
-- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler;
+- (BOOL)setDevicesDidUpdateCallback:(nullable RTCDevicesDidUpdateCallback)callback;
+- (BOOL)setSpeechActivityCallback:(nullable RTCSpeechActivityCallback)callback;
 
 - (BOOL)startPlayout;
 - (BOOL)stopPlayout;

sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include <AudioUnit/AudioUnit.h>
+#include <os/lock.h>
 
 #import "RTCAudioDeviceModule.h"
 #import "RTCAudioDeviceModule+Private.h"
@@ -25,19 +25,60 @@
 
 class AudioDeviceObserver : public webrtc::AudioDeviceObserver {
  public:
-  AudioDeviceObserver() {}
+  AudioDeviceObserver() : lock_(OS_UNFAIR_LOCK_INIT) {}
 
   void OnDevicesUpdated() override {
+    os_unfair_lock_lock(&lock_);
+    if (on_devices_did_update_callback_) {
+      on_devices_did_update_callback_();
+    }
+    os_unfair_lock_unlock(&lock_);
+  }
 
-    RTCLogInfo(@"AudioDeviceObserver OnDevicesUpdated");
-
-    if (callback_handler_) {
-      callback_handler_();
+  void OnSpeechActivityEvent(webrtc::AudioDeviceModule::SpeechActivityEvent event) override {
+    os_unfair_lock_lock(&lock_);
+    if (on_speech_activity_callback_) {
+      on_speech_activity_callback_(ConvertSpeechActivityEvent(event));
     }
+    os_unfair_lock_unlock(&lock_);
+  }
+
+ void SetDevicesUpdatedCallBack(RTCDevicesDidUpdateCallback cb) {
+    os_unfair_lock_lock(&lock_);
+    on_devices_did_update_callback_ = cb;
+    os_unfair_lock_unlock(&lock_);
+  }
+
+  void SetOnSpeechActivityCallBack(RTCSpeechActivityCallback cb) {
+    os_unfair_lock_lock(&lock_);
+    on_speech_activity_callback_ = cb;
+    os_unfair_lock_unlock(&lock_);
+  }
+
+  bool IsAnyCallbackAttached() {
+    os_unfair_lock_lock(&lock_);
+    bool result =
+        on_devices_did_update_callback_ != nullptr || on_speech_activity_callback_ != nullptr;
+    os_unfair_lock_unlock(&lock_);
+    return result;
   }
 
- // private:
-  RTCOnAudioDevicesDidUpdate callback_handler_;
+ private:
+  os_unfair_lock lock_;
+  RTCDevicesDidUpdateCallback on_devices_did_update_callback_;
+  RTCSpeechActivityCallback on_speech_activity_callback_;
+
+  RTCSpeechActivityEvent ConvertSpeechActivityEvent(
+      webrtc::AudioDeviceModule::SpeechActivityEvent event) {
+    switch (event) {
+      case webrtc::AudioDeviceModule::SpeechActivityEvent::kStarted:
+        return RTCSpeechActivityEvent::RTCSpeechActivityEventStarted;
+      case webrtc::AudioDeviceModule::SpeechActivityEvent::kEnded:
+        return RTCSpeechActivityEvent::RTCSpeechActivityEventEnded;
+      default:
+        return RTCSpeechActivityEvent::RTCSpeechActivityEventEnded;
+    }
+  }
 };
 
 @implementation RTC_OBJC_TYPE (RTCAudioDeviceModule) {
@@ -57,10 +98,6 @@ - (instancetype)initWithNativeModule:(rtc::scoped_refptr<webrtc::AudioDeviceModu
 
   _observer = new AudioDeviceObserver();
 
-  _workerThread->BlockingCall([self] {
-    _native->SetObserver(_observer);
-  });
-
   return self;
 }
 
@@ -240,9 +277,19 @@ - (BOOL)initRecording {
   });
 }
 
-- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler {
-  _sink->callback_handler_ = handler;
-  _observer->callback_handler_ = callback;
+- (BOOL)setDevicesDidUpdateCallback:(nullable RTCDevicesDidUpdateCallback)callback {
+  _observer->SetDevicesUpdatedCallBack(callback);
+  webrtc::AudioDeviceObserver *observer = _observer->IsAnyCallbackAttached() ? _observer : nullptr;
+  _workerThread->BlockingCall([self, observer] { _native->SetObserver(observer); });
+
+  return YES;
+}
+
+- (BOOL)setSpeechActivityCallback:(nullable RTCSpeechActivityCallback)callback {
+  _observer->SetOnSpeechActivityCallBack(callback);
+  webrtc::AudioDeviceObserver *observer = _observer->IsAnyCallbackAttached() ? _observer : nullptr;
+  _workerThread->BlockingCall([self, observer] { _native->SetObserver(observer); });
+
   return YES;
 }
 

sdk/objc/native/src/audio/audio_device_module_ios.mm

@@ -667,9 +667,10 @@
 #endif  // WEBRTC_IOS
 
   int32_t AudioDeviceModuleIOS::SetObserver(AudioDeviceObserver* observer) const {
-    // not implemented
-    RTC_LOG(LS_WARNING) << __FUNCTION__ << "(" << observer << ") Not implemented";
-    return -1;
+    RTC_DLOG(LS_INFO) << __FUNCTION__;
+    int r = audio_device_->SetObserver(observer);
+    RTC_DLOG(LS_INFO) << "output: " << r;
+    return r;
   }
 }
 }