[camera_avfoundation] enable more than 30 fps (#7394)

Camera plugin was crashing when I tried to set fps to 60 on most media presets (except maybe on 1280x720, although tested device supports 60 fps for up to 1920x1440 and can do 240 fps on 1280x720) because when is `activeVideoMinFrameDuration` and `activeVideoMaxFrameDuration` set to fps outside of what active format supports it throws exception. Now it tries to find a format which supports fps closest to wanted fps and clamps it if it cannot be set to exact value to prevent crashes. It searches for formats with the exact same resolution. For example in format list it can be like "1920x1080 { 3- 30 fps}", "1920x1080 { 3- 60 fps}" and "1920x1080 { 6-120 fps}, but when setting `sessionPreset` then "1920x1080 { 3- 30 fps}" is selected by default. On the tested device there are 2 "media subtypes" `420f` and `420v` for each format where the first is denoted as "supports wide color" in debug description and the system has tendency to choose this one. So it tries to preserve the media subtype to what is preferred by the system and this is also added to `highestResolutionFormatForCaptureDevice` (with lower priority than max resolution/fps). Also there was nested `lockForConfiguration` and `unlockForConfiguration` when using `ResolutionPreset.max` together with setting up fps.

Author: misos1

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.17+5
+
+* Adds ability to use any supported FPS and fixes crash when using unsupported FPS.
+
 ## 0.9.17+4
 
 * Updates Pigeon for non-nullable collection type support.
@@ -13,7 +17,7 @@
 
 ## 0.9.17+1
 
-* Fixes a crash due to appending sample buffers when readyForMoreMediaData is NO
+* Fixes a crash due to appending sample buffers when readyForMoreMediaData is NO.
 
 ## 0.9.17
 

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraSettingsTests.m

@@ -202,4 +202,20 @@ - (void)testSettings_ShouldBeSupportedByMethodCall {
   XCTAssertNotNil(resultValue);
 }
 
+- (void)testSettings_ShouldSelectFormatWhichSupports60FPS {
+  FCPPlatformMediaSettings *settings =
+      [FCPPlatformMediaSettings makeWithResolutionPreset:gTestResolutionPreset
+                                         framesPerSecond:@(60)
+                                            videoBitrate:@(gTestVideoBitrate)
+                                            audioBitrate:@(gTestAudioBitrate)
+                                             enableAudio:gTestEnableAudio];
+
+  FLTCam *camera = FLTCreateCamWithCaptureSessionQueueAndMediaSettings(
+      dispatch_queue_create("test", NULL), settings, nil, nil);
+
+  AVFrameRateRange *range = camera.captureDevice.activeFormat.videoSupportedFrameRateRanges[0];
+  XCTAssertLessThanOrEqual(range.minFrameRate, 60);
+  XCTAssertGreaterThanOrEqual(range.maxFrameRate, 60);
+}
+
 @end

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m

@@ -52,14 +52,44 @@
   OCMStub([audioSessionMock addInputWithNoConnections:[OCMArg any]]);
   OCMStub([audioSessionMock canSetSessionPreset:[OCMArg any]]).andReturn(YES);
 
+  id frameRateRangeMock1 = OCMClassMock([AVFrameRateRange class]);
+  OCMStub([frameRateRangeMock1 minFrameRate]).andReturn(3);
+  OCMStub([frameRateRangeMock1 maxFrameRate]).andReturn(30);
+  id captureDeviceFormatMock1 = OCMClassMock([AVCaptureDeviceFormat class]);
+  OCMStub([captureDeviceFormatMock1 videoSupportedFrameRateRanges]).andReturn(@[
+    frameRateRangeMock1
+  ]);
+
+  id frameRateRangeMock2 = OCMClassMock([AVFrameRateRange class]);
+  OCMStub([frameRateRangeMock2 minFrameRate]).andReturn(3);
+  OCMStub([frameRateRangeMock2 maxFrameRate]).andReturn(60);
+  id captureDeviceFormatMock2 = OCMClassMock([AVCaptureDeviceFormat class]);
+  OCMStub([captureDeviceFormatMock2 videoSupportedFrameRateRanges]).andReturn(@[
+    frameRateRangeMock2
+  ]);
+
+  id captureDeviceMock = OCMClassMock([AVCaptureDevice class]);
+  OCMStub([captureDeviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES);
+  OCMStub([captureDeviceMock formats]).andReturn((@[
+    captureDeviceFormatMock1, captureDeviceFormatMock2
+  ]));
+  __block AVCaptureDeviceFormat *format = captureDeviceFormatMock1;
+  OCMStub([captureDeviceMock setActiveFormat:[OCMArg any]]).andDo(^(NSInvocation *invocation) {
+    [invocation retainArguments];
+    [invocation getArgument:&format atIndex:2];
+  });
+  OCMStub([captureDeviceMock activeFormat]).andDo(^(NSInvocation *invocation) {
+    [invocation setReturnValue:&format];
+  });
+
   id fltCam = [[FLTCam alloc] initWithMediaSettings:mediaSettings
                              mediaSettingsAVWrapper:mediaSettingsAVWrapper
                              orientation:UIDeviceOrientationPortrait
                              videoCaptureSession:videoSessionMock
                              audioCaptureSession:audioSessionMock
                              captureSessionQueue:captureSessionQueue
                                captureDeviceFactory:captureDeviceFactory ?: ^AVCaptureDevice *(void) {
-                               return [AVCaptureDevice deviceWithUniqueID:@"camera"];
+                               return captureDeviceMock;
                              }
                              videoDimensionsForFormat:^CMVideoDimensions(AVCaptureDeviceFormat *format) {
                                return CMVideoFormatDescriptionGetDimensions(format.formatDescription);

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/FLTCam.m

@@ -153,6 +153,59 @@ - (instancetype)initWithCameraName:(NSString *)cameraName
       error:error];
 }
 
+// Returns frame rate supported by format closest to targetFrameRate.
+static double bestFrameRateForFormat(AVCaptureDeviceFormat *format, double targetFrameRate) {
+  double bestFrameRate = 0;
+  double minDistance = DBL_MAX;
+  for (AVFrameRateRange *range in format.videoSupportedFrameRateRanges) {
+    double frameRate = MIN(MAX(targetFrameRate, range.minFrameRate), range.maxFrameRate);
+    double distance = fabs(frameRate - targetFrameRate);
+    if (distance < minDistance) {
+      bestFrameRate = frameRate;
+      minDistance = distance;
+    }
+  }
+  return bestFrameRate;
+}
+
+// Finds format with same resolution as current activeFormat in captureDevice for which
+// bestFrameRateForFormat returned frame rate closest to mediaSettings.framesPerSecond.
+// Preferred are formats with the same subtype as current activeFormat. Sets this format
+// as activeFormat and also updates mediaSettings.framesPerSecond to value which
+// bestFrameRateForFormat returned for that format.
+static void selectBestFormatForRequestedFrameRate(
+    AVCaptureDevice *captureDevice, FCPPlatformMediaSettings *mediaSettings,
+    VideoDimensionsForFormat videoDimensionsForFormat) {
+  CMVideoDimensions targetResolution = videoDimensionsForFormat(captureDevice.activeFormat);
+  double targetFrameRate = mediaSettings.framesPerSecond.doubleValue;
+  FourCharCode preferredSubType =
+      CMFormatDescriptionGetMediaSubType(captureDevice.activeFormat.formatDescription);
+  AVCaptureDeviceFormat *bestFormat = captureDevice.activeFormat;
+  double bestFrameRate = bestFrameRateForFormat(bestFormat, targetFrameRate);
+  double minDistance = fabs(bestFrameRate - targetFrameRate);
+  BOOL isBestSubTypePreferred = YES;
+  for (AVCaptureDeviceFormat *format in captureDevice.formats) {
+    CMVideoDimensions resolution = videoDimensionsForFormat(format);
+    if (resolution.width != targetResolution.width ||
+        resolution.height != targetResolution.height) {
+      continue;
+    }
+    double frameRate = bestFrameRateForFormat(format, targetFrameRate);
+    double distance = fabs(frameRate - targetFrameRate);
+    FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+    BOOL isSubTypePreferred = subType == preferredSubType;
+    if (distance < minDistance ||
+        (distance == minDistance && isSubTypePreferred && !isBestSubTypePreferred)) {
+      bestFormat = format;
+      bestFrameRate = frameRate;
+      minDistance = distance;
+      isBestSubTypePreferred = isSubTypePreferred;
+    }
+  }
+  captureDevice.activeFormat = bestFormat;
+  mediaSettings.framesPerSecond = @(bestFrameRate);
+}
+
 - (instancetype)initWithMediaSettings:(FCPPlatformMediaSettings *)mediaSettings
                mediaSettingsAVWrapper:(FLTCamMediaSettingsAVWrapper *)mediaSettingsAVWrapper
                           orientation:(UIDeviceOrientation)orientation
@@ -226,6 +279,9 @@ - (instancetype)initWithMediaSettings:(FCPPlatformMediaSettings *)mediaSettings
         return nil;
       }
 
+      selectBestFormatForRequestedFrameRate(_captureDevice, _mediaSettings,
+                                            _videoDimensionsForFormat);
+
       // Set frame rate with 1/10 precision allowing not integral values.
       int fpsNominator = floor([_mediaSettings.framesPerSecond doubleValue] * 10.0);
       CMTime duration = CMTimeMake(10, fpsNominator);
@@ -474,56 +530,42 @@ - (BOOL)setCaptureSessionPreset:(FCPPlatformResolutionPreset)resolutionPreset
           // Set the best device format found and finish the device configuration.
           _captureDevice.activeFormat = bestFormat;
           [_captureDevice unlockForConfiguration];
-
-          // Set the preview size based on values from the current capture device.
-          _previewSize =
-              CGSizeMake(_captureDevice.activeFormat.highResolutionStillImageDimensions.width,
-                         _captureDevice.activeFormat.highResolutionStillImageDimensions.height);
           break;
         }
       }
     }
     case FCPPlatformResolutionPresetUltraHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset3840x2160]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset3840x2160;
-        _previewSize = CGSizeMake(3840, 2160);
         break;
       }
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPresetHigh]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPresetHigh;
-        _previewSize =
-            CGSizeMake(_captureDevice.activeFormat.highResolutionStillImageDimensions.width,
-                       _captureDevice.activeFormat.highResolutionStillImageDimensions.height);
         break;
       }
     case FCPPlatformResolutionPresetVeryHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset1920x1080]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset1920x1080;
-        _previewSize = CGSizeMake(1920, 1080);
         break;
       }
     case FCPPlatformResolutionPresetHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset1280x720;
-        _previewSize = CGSizeMake(1280, 720);
         break;
       }
     case FCPPlatformResolutionPresetMedium:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset640x480]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset640x480;
-        _previewSize = CGSizeMake(640, 480);
         break;
       }
     case FCPPlatformResolutionPresetLow:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset352x288]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset352x288;
-        _previewSize = CGSizeMake(352, 288);
         break;
       }
     default:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPresetLow]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPresetLow;
-        _previewSize = CGSizeMake(352, 288);
       } else {
         if (error != nil) {
           *error =
@@ -537,23 +579,33 @@ - (BOOL)setCaptureSessionPreset:(FCPPlatformResolutionPreset)resolutionPreset
         return NO;
       }
   }
+  CMVideoDimensions size = self.videoDimensionsForFormat(_captureDevice.activeFormat);
+  _previewSize = CGSizeMake(size.width, size.height);
   _audioCaptureSession.sessionPreset = _videoCaptureSession.sessionPreset;
   return YES;
 }
 
 /// Finds the highest available resolution in terms of pixel count for the given device.
+/// Preferred are formats with the same subtype as current activeFormat.
 - (AVCaptureDeviceFormat *)highestResolutionFormatForCaptureDevice:
     (AVCaptureDevice *)captureDevice {
+  FourCharCode preferredSubType =
+      CMFormatDescriptionGetMediaSubType(_captureDevice.activeFormat.formatDescription);
   AVCaptureDeviceFormat *bestFormat = nil;
   NSUInteger maxPixelCount = 0;
+  BOOL isBestSubTypePreferred = NO;
   for (AVCaptureDeviceFormat *format in _captureDevice.formats) {
     CMVideoDimensions res = self.videoDimensionsForFormat(format);
     NSUInteger height = res.height;
     NSUInteger width = res.width;
     NSUInteger pixelCount = height * width;
-    if (pixelCount > maxPixelCount) {
-      maxPixelCount = pixelCount;
+    FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+    BOOL isSubTypePreferred = subType == preferredSubType;
+    if (pixelCount > maxPixelCount ||
+        (pixelCount == maxPixelCount && isSubTypePreferred && !isBestSubTypePreferred)) {
       bestFormat = format;
+      maxPixelCount = pixelCount;
+      isBestSubTypePreferred = isSubTypePreferred;
     }
   }
   return bestFormat;

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.17+4
+version: 0.9.17+5
 
 environment:
   sdk: ^3.3.0