Skip to content

Commit d6e0d1f

Browse files
authored
[camera_android] Support concurrently image capture and image streaming (#4332)
Properly configures surface needed for image capture when image streaming/recording is started to support concurrently still capture and image streaming. Fixes flutter/flutter#125314. Apologies for the many commits :(
1 parent cbbc2fb commit d6e0d1f

File tree

4 files changed

+145
-8
lines changed

4 files changed

+145
-8
lines changed

packages/camera/camera_android/CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
## NEXT
1+
## 0.10.8+3
22

33
* Fixes unawaited_futures violations.
44
* Removes duplicate line in `MediaRecorderBuilder.java`.
5+
* Adds support for concurrently capturing images and image streaming/recording.
56

67
## 0.10.8+2
78

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import android.hardware.camera2.params.SessionConfiguration;
2222
import android.media.CamcorderProfile;
2323
import android.media.EncoderProfiles;
24+
import android.media.Image;
2425
import android.media.ImageReader;
2526
import android.media.MediaRecorder;
2627
import android.os.Build;
@@ -414,8 +415,14 @@ private void createCaptureSession(
414415

415416
List<Surface> remainingSurfaces = Arrays.asList(surfaces);
416417
if (templateType != CameraDevice.TEMPLATE_PREVIEW) {
417-
// If it is not preview mode, add all surfaces as targets.
418+
// If it is not preview mode, add all surfaces as targets
419+
// except the surface used for still capture as this should
420+
// not be part of a repeating request.
421+
Surface pictureImageReaderSurface = pictureImageReader.getSurface();
418422
for (Surface surface : remainingSurfaces) {
423+
if (surface == pictureImageReaderSurface) {
424+
continue;
425+
}
419426
previewRequestBuilder.addTarget(surface);
420427
}
421428
}
@@ -539,6 +546,10 @@ private void startCapture(boolean record, boolean stream) throws CameraAccessExc
539546
surfaces.add(imageStreamReader.getSurface());
540547
}
541548

549+
// Add pictureImageReader surface to allow for still capture
550+
// during recording/image streaming.
551+
surfaces.add(pictureImageReader.getSurface());
552+
542553
createCaptureSession(
543554
CameraDevice.TEMPLATE_RECORD, successCallback, surfaces.toArray(new Surface[0]));
544555
}
@@ -659,7 +670,6 @@ public void onCaptureCompleted(
659670
};
660671

661672
try {
662-
captureSession.stopRepeating();
663673
Log.i(TAG, "sending capture request");
664674
captureSession.capture(stillBuilder.build(), captureCallback, backgroundHandler);
665675
} catch (CameraAccessException e) {
@@ -1140,10 +1150,15 @@ public void startPreviewWithImageStream(EventChannel imageStreamChannel)
11401150
public void onImageAvailable(ImageReader reader) {
11411151
Log.i(TAG, "onImageAvailable");
11421152

1153+
// Use acquireNextImage since image reader is only for one image.
1154+
Image image = reader.acquireNextImage();
1155+
if (image == null) {
1156+
return;
1157+
}
1158+
11431159
backgroundHandler.post(
11441160
new ImageSaver(
1145-
// Use acquireNextImage since image reader is only for one image.
1146-
reader.acquireNextImage(),
1161+
image,
11471162
captureFile,
11481163
new ImageSaver.Callback() {
11491164
@Override
@@ -1159,7 +1174,8 @@ public void onError(String errorCode, String errorMessage) {
11591174
cameraCaptureCallback.setCameraState(CameraState.STATE_PREVIEW);
11601175
}
11611176

1162-
private void prepareRecording(@NonNull Result result) {
1177+
@VisibleForTesting
1178+
void prepareRecording(@NonNull Result result) {
11631179
final File outputDir = applicationContext.getCacheDir();
11641180
try {
11651181
captureFile = File.createTempFile("REC", ".mp4", outputDir);

packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/CameraTest.java

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,14 @@
88
import static org.junit.Assert.assertFalse;
99
import static org.junit.Assert.assertNotNull;
1010
import static org.mockito.ArgumentMatchers.any;
11+
import static org.mockito.ArgumentMatchers.anyInt;
1112
import static org.mockito.ArgumentMatchers.eq;
13+
import static org.mockito.Mockito.doNothing;
1214
import static org.mockito.Mockito.doThrow;
1315
import static org.mockito.Mockito.mock;
1416
import static org.mockito.Mockito.mockStatic;
1517
import static org.mockito.Mockito.never;
18+
import static org.mockito.Mockito.spy;
1619
import static org.mockito.Mockito.times;
1720
import static org.mockito.Mockito.verify;
1821
import static org.mockito.Mockito.when;
@@ -36,6 +39,7 @@
3639
import androidx.annotation.Nullable;
3740
import androidx.lifecycle.LifecycleObserver;
3841
import io.flutter.embedding.engine.systemchannels.PlatformChannel;
42+
import io.flutter.plugin.common.EventChannel;
3943
import io.flutter.plugin.common.MethodChannel;
4044
import io.flutter.plugins.camera.features.CameraFeatureFactory;
4145
import io.flutter.plugins.camera.features.CameraFeatures;
@@ -56,8 +60,10 @@
5660
import io.flutter.plugins.camera.features.sensororientation.DeviceOrientationManager;
5761
import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature;
5862
import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature;
63+
import io.flutter.plugins.camera.media.ImageStreamReader;
5964
import io.flutter.plugins.camera.utils.TestUtils;
6065
import io.flutter.view.TextureRegistry;
66+
import java.io.IOException;
6167
import java.util.ArrayList;
6268
import java.util.List;
6369
import org.junit.After;
@@ -638,6 +644,8 @@ public void startPreview_shouldPullStreamFromVideoRenderer()
638644
TestUtils.setPrivateField(camera, "videoRenderer", mockVideoRenderer);
639645
CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
640646
TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
647+
ImageReader mockPictureImageReader = mock(ImageReader.class);
648+
TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
641649

642650
TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
643651
(TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
@@ -674,9 +682,10 @@ public void startPreview_shouldPullStreamFromImageReader()
674682

675683
when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
676684
when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
685+
when(mockImageReader.getSurface()).thenReturn(mock(Surface.class));
677686

678687
camera.startPreview();
679-
verify(mockImageReader, times(1))
688+
verify(mockImageReader, times(2)) // we expect two calls to start regular preview.
680689
.getSurface(); // stream pulled from regular imageReader's surface.
681690
}
682691

@@ -692,6 +701,8 @@ public void startPreview_shouldFlipRotation() throws InterruptedException, Camer
692701
TestUtils.setPrivateField(camera, "initialCameraFacing", CameraMetadata.LENS_FACING_BACK);
693702
CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
694703
TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
704+
ImageReader mockPictureImageReader = mock(ImageReader.class);
705+
TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
695706

696707
TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
697708
(TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
@@ -707,6 +718,39 @@ public void startPreview_shouldFlipRotation() throws InterruptedException, Camer
707718
verify(mockVideoRenderer, times(1)).setRotation(180);
708719
}
709720

721+
@Test
722+
public void startPreviewWithImageStream_shouldPullStreamsFromImageReaders()
723+
throws InterruptedException, CameraAccessException {
724+
ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
725+
mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
726+
SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
727+
Size mockSize = mock(Size.class);
728+
ImageReader mockPictureImageReader = mock(ImageReader.class);
729+
ImageStreamReader mockImageStreamReader = mock(ImageStreamReader.class);
730+
TestUtils.setPrivateField(camera, "recordingVideo", false);
731+
TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
732+
CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
733+
TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
734+
camera.imageStreamReader = mockImageStreamReader;
735+
736+
TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
737+
(TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
738+
ResolutionFeature resolutionFeature =
739+
(ResolutionFeature)
740+
TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
741+
742+
when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
743+
when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
744+
745+
camera.startPreviewWithImageStream(mock(EventChannel.class));
746+
verify(mockImageStreamReader, times(1))
747+
.getSurface(); // stream pulled from image streaming imageReader's surface.
748+
verify(
749+
mockPictureImageReader,
750+
times(2)) // we expect one call to start the capture, one to create the capture session.
751+
.getSurface(); // stream pulled from regular imageReader's surface.
752+
}
753+
710754
@Test
711755
public void setDescriptionWhileRecording_shouldErrorWhenNotRecording() {
712756
MethodChannel.Result mockResult = mock(MethodChannel.Result.class);
@@ -806,6 +850,43 @@ public void setFocusMode_shouldSendErrorEventOnUnlockAutoFocusCameraAccessExcept
806850
verify(mockDartMessenger, times(1)).sendCameraErrorEvent(any());
807851
}
808852

853+
@Test
854+
public void startVideoRecording_shouldPullStreamsFromMediaRecorderAndImageReader()
855+
throws InterruptedException, IOException, CameraAccessException {
856+
Camera cameraSpy = spy(camera);
857+
ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
858+
mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
859+
SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
860+
Size mockSize = mock(Size.class);
861+
MediaRecorder mockMediaRecorder = mock(MediaRecorder.class);
862+
ImageReader mockPictureImageReader = mock(ImageReader.class);
863+
TestUtils.setPrivateField(cameraSpy, "mediaRecorder", mockMediaRecorder);
864+
TestUtils.setPrivateField(cameraSpy, "recordingVideo", false);
865+
TestUtils.setPrivateField(cameraSpy, "pictureImageReader", mockPictureImageReader);
866+
CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
867+
TestUtils.setPrivateField(cameraSpy, "cameraDevice", fakeCamera);
868+
MethodChannel.Result mockResult = mock(MethodChannel.Result.class);
869+
870+
TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
871+
(TextureRegistry.SurfaceTextureEntry)
872+
TestUtils.getPrivateField(cameraSpy, "flutterTexture");
873+
ResolutionFeature resolutionFeature =
874+
(ResolutionFeature)
875+
TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
876+
877+
when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
878+
when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
879+
doNothing().when(cameraSpy).prepareRecording(mockResult);
880+
881+
cameraSpy.startVideoRecording(mockResult, null);
882+
verify(mockMediaRecorder, times(1))
883+
.getSurface(); // stream pulled from media recorder's surface.
884+
verify(
885+
mockPictureImageReader,
886+
times(2)) // we expect one call to start the capture, one to create the capture session.
887+
.getSurface(); // stream pulled from image streaming imageReader's surface.
888+
}
889+
809890
@Test
810891
public void setFocusMode_shouldLockAutoFocusForLockedMode() throws CameraAccessException {
811892
camera.setFocusMode(mock(MethodChannel.Result.class), FocusMode.locked);
@@ -1013,6 +1094,45 @@ public void createCaptureSession_doesNotCloseCaptureSession() throws CameraAcces
10131094
verify(mockCaptureSession, never()).close();
10141095
}
10151096

1097+
@Test
1098+
public void createCaptureSession_shouldNotAddPictureImageSurfaceToPreviewRequest()
1099+
throws CameraAccessException {
1100+
Surface mockSurface = mock(Surface.class);
1101+
Surface mockSecondarySurface = mock(Surface.class);
1102+
SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
1103+
ResolutionFeature mockResolutionFeature = mock(ResolutionFeature.class);
1104+
Size mockSize = mock(Size.class);
1105+
ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
1106+
mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
1107+
CameraDeviceWrapper fakeCamera = spy(new FakeCameraDeviceWrapper(mockRequestBuilders));
1108+
ImageReader mockPictureImageReader = mock(ImageReader.class);
1109+
TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
1110+
TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
1111+
CaptureRequest.Builder mockPreviewRequestBuilder = mock(CaptureRequest.Builder.class);
1112+
1113+
TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
1114+
(TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
1115+
CameraFeatures cameraFeatures =
1116+
(CameraFeatures) TestUtils.getPrivateField(camera, "cameraFeatures");
1117+
ResolutionFeature resolutionFeature =
1118+
(ResolutionFeature)
1119+
TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
1120+
1121+
when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
1122+
when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
1123+
when(fakeCamera.createCaptureRequest(anyInt())).thenReturn(mockPreviewRequestBuilder);
1124+
when(mockPictureImageReader.getSurface()).thenReturn(mockSurface);
1125+
1126+
// Test with preview template.
1127+
camera.createCaptureSession(CameraDevice.TEMPLATE_PREVIEW, mockSurface, mockSecondarySurface);
1128+
verify(mockPreviewRequestBuilder, times(0)).addTarget(mockSurface);
1129+
1130+
// Test with non-preview template.
1131+
camera.createCaptureSession(CameraDevice.TEMPLATE_RECORD, mockSurface, mockSecondarySurface);
1132+
verify(mockPreviewRequestBuilder, times(0)).addTarget(mockSurface);
1133+
verify(mockPreviewRequestBuilder).addTarget(mockSecondarySurface);
1134+
}
1135+
10161136
@Test
10171137
public void close_doesCloseCaptureSessionWhenCameraDeviceNull() {
10181138
camera.close();

packages/camera/camera_android/pubspec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ description: Android implementation of the camera plugin.
33
repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_android
44
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
55

6-
version: 0.10.8+2
6+
version: 0.10.8+3
77

88
environment:
99
sdk: ">=2.18.0 <4.0.0"

0 commit comments

Comments
 (0)