diff --git a/packages/camera/camera_platform_interface/CHANGELOG.md b/packages/camera/camera_platform_interface/CHANGELOG.md index 3cad35d71ae5..5ecd8891fe20 100644 --- a/packages/camera/camera_platform_interface/CHANGELOG.md +++ b/packages/camera/camera_platform_interface/CHANGELOG.md @@ -1,5 +1,6 @@ -## NEXT +## 2.2.0 +* Adds image streaming to the platform interface. * Removes unnecessary imports. ## 2.1.6 diff --git a/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart b/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart index c856f3467821..babef144b086 100644 --- a/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart +++ b/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart @@ -12,6 +12,8 @@ import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; import 'package:stream_transform/stream_transform.dart'; +import 'type_conversion.dart'; + const MethodChannel _channel = MethodChannel('plugins.flutter.io/camera'); /// An implementation of [CameraPlatform] that uses method channels. @@ -48,6 +50,12 @@ class MethodChannelCamera extends CameraPlatform { final StreamController deviceEventStreamController = StreamController.broadcast(); + // The stream to receive frames from the native code. + StreamSubscription? _platformImageStreamSubscription; + + // The stream for vending frames to platform interface clients. + StreamController? _frameStreamController; + Stream _cameraEvents(int cameraId) => cameraEventStreamController.stream .where((CameraEvent event) => event.cameraId == cameraId); @@ -267,6 +275,52 @@ class MethodChannelCamera extends CameraPlatform { {'cameraId': cameraId}, ); + @override + Stream onStreamedFrameAvailable(int cameraId, + {CameraImageStreamOptions? options}) { + _frameStreamController = StreamController( + onListen: _onFrameStreamListen, + onPause: _onFrameStreamPauseResume, + onResume: _onFrameStreamPauseResume, + onCancel: _onFrameStreamCancel, + ); + return _frameStreamController!.stream; + } + + void _onFrameStreamListen() { + _startPlatformStream(); + } + + Future _startPlatformStream() async { + await _channel.invokeMethod('startImageStream'); + const EventChannel cameraEventChannel = + EventChannel('plugins.flutter.io/camera/imageStream'); + _platformImageStreamSubscription = + cameraEventChannel.receiveBroadcastStream().listen((dynamic imageData) { + if (defaultTargetPlatform == TargetPlatform.iOS) { + try { + _channel.invokeMethod('receivedImageStreamData'); + } on PlatformException catch (e) { + throw CameraException(e.code, e.message); + } + } + _frameStreamController! + .add(cameraImageFromPlatformData(imageData as Map)); + }); + } + + FutureOr _onFrameStreamCancel() async { + await _channel.invokeMethod('stopImageStream'); + await _platformImageStreamSubscription?.cancel(); + _platformImageStreamSubscription = null; + _frameStreamController = null; + } + + void _onFrameStreamPauseResume() { + throw CameraException('InvalidCall', + 'Pause and resume are not supported for onStreamedFrameAvailable'); + } + @override Future setFlashMode(int cameraId, FlashMode mode) => _channel.invokeMethod( diff --git a/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart b/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart new file mode 100644 index 000000000000..9dffbbf6ae3a --- /dev/null +++ b/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart @@ -0,0 +1,61 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; + +import '../types/types.dart'; + +/// Converts method channel call [data] for `receivedImageStreamData` to a +/// [CameraImageData]. +CameraImageData cameraImageFromPlatformData(Map data) { + return CameraImageData( + format: _cameraImageFormatFromPlatformData(data['format']), + height: data['height'] as int, + width: data['width'] as int, + lensAperture: data['lensAperture'] as double?, + sensorExposureTime: data['sensorExposureTime'] as int?, + sensorSensitivity: data['sensorSensitivity'] as double?, + planes: List.unmodifiable( + (data['planes'] as List).map( + (dynamic planeData) => _cameraImagePlaneFromPlatformData( + planeData as Map)))); +} + +CameraImageFormat _cameraImageFormatFromPlatformData(dynamic data) { + return CameraImageFormat(_imageFormatGroupFromPlatformData(data), raw: data); +} + +ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) { + if (defaultTargetPlatform == TargetPlatform.android) { + switch (data) { + case 35: // android.graphics.ImageFormat.YUV_420_888 + return ImageFormatGroup.yuv420; + case 256: // android.graphics.ImageFormat.JPEG + return ImageFormatGroup.jpeg; + } + } + + if (defaultTargetPlatform == TargetPlatform.iOS) { + switch (data) { + case 875704438: // kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + return ImageFormatGroup.yuv420; + + case 1111970369: // kCVPixelFormatType_32BGRA + return ImageFormatGroup.bgra8888; + } + } + + return ImageFormatGroup.unknown; +} + +CameraImagePlane _cameraImagePlaneFromPlatformData(Map data) { + return CameraImagePlane( + bytes: data['bytes'] as Uint8List, + bytesPerPixel: data['bytesPerPixel'] as int?, + bytesPerRow: data['bytesPerRow'] as int, + height: data['height'] as int?, + width: data['width'] as int?); +} diff --git a/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart b/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart index daa19b8b4011..eaa779a943db 100644 --- a/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart +++ b/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart @@ -149,6 +149,21 @@ abstract class CameraPlatform extends PlatformInterface { throw UnimplementedError('resumeVideoRecording() is not implemented.'); } + /// A new streamed frame is available. + /// + /// Listening to this stream will start streaming, and canceling will stop. + /// Pausing will throw a [CameraException], as pausing the stream would cause + /// very high memory usage; to temporarily stop receiving frames, cancel, then + /// listen again later. + /// + /// + // TODO(bmparr): Add options to control streaming settings (e.g., + // resolution and FPS). + Stream onStreamedFrameAvailable(int cameraId, + {CameraImageStreamOptions? options}) { + throw UnimplementedError('onStreamedFrameAvailable() is not implemented.'); + } + /// Sets the flash mode for the selected camera. /// On Web [FlashMode.auto] corresponds to [FlashMode.always]. Future setFlashMode(int cameraId, FlashMode mode) { diff --git a/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart b/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart new file mode 100644 index 000000000000..6971dbb39737 --- /dev/null +++ b/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart @@ -0,0 +1,126 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; + +import '../../camera_platform_interface.dart'; + +/// Options for configuring camera streaming. +/// +/// Currently unused; this exists for future-proofing of the platform interface +/// API. +@immutable +class CameraImageStreamOptions {} + +/// A single color plane of image data. +/// +/// The number and meaning of the planes in an image are determined by its +/// format. +@immutable +class CameraImagePlane { + /// Creates a new instance with the given bytes and optional metadata. + const CameraImagePlane({ + required this.bytes, + required this.bytesPerRow, + this.bytesPerPixel, + this.height, + this.width, + }); + + /// Bytes representing this plane. + final Uint8List bytes; + + /// The row stride for this color plane, in bytes. + final int bytesPerRow; + + /// The distance between adjacent pixel samples in bytes, when available. + final int? bytesPerPixel; + + /// Height of the pixel buffer, when available. + final int? height; + + /// Width of the pixel buffer, when available. + final int? width; +} + +/// Describes how pixels are represented in an image. +@immutable +class CameraImageFormat { + /// Create a new format with the given cross-platform group and raw underyling + /// platform identifier. + const CameraImageFormat(this.group, {required this.raw}); + + /// Describes the format group the raw image format falls into. + final ImageFormatGroup group; + + /// Raw version of the format from the underlying platform. + /// + /// On Android, this should be an `int` from class + /// `android.graphics.ImageFormat`. See + /// https://developer.android.com/reference/android/graphics/ImageFormat + /// + /// On iOS, this should be a `FourCharCode` constant from Pixel Format + /// Identifiers. See + /// https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers + final dynamic raw; +} + +/// A single complete image buffer from the platform camera. +/// +/// This class allows for direct application access to the pixel data of an +/// Image through one or more [Uint8List]. Each buffer is encapsulated in a +/// [CameraImagePlane] that describes the layout of the pixel data in that +/// plane. [CameraImageData] is not directly usable as a UI resource. +/// +/// Although not all image formats are planar on all platforms, this class +/// treats 1-dimensional images as single planar images. +@immutable +class CameraImageData { + /// Creates a new instance with the given format, planes, and metadata. + const CameraImageData({ + required this.format, + required this.planes, + required this.height, + required this.width, + this.lensAperture, + this.sensorExposureTime, + this.sensorSensitivity, + }); + + /// Format of the image provided. + /// + /// Determines the number of planes needed to represent the image, and + /// the general layout of the pixel data in each [Uint8List]. + final CameraImageFormat format; + + /// Height of the image in pixels. + /// + /// For formats where some color channels are subsampled, this is the height + /// of the largest-resolution plane. + final int height; + + /// Width of the image in pixels. + /// + /// For formats where some color channels are subsampled, this is the width + /// of the largest-resolution plane. + final int width; + + /// The pixels planes for this image. + /// + /// The number of planes is determined by the format of the image. + final List planes; + + /// The aperture settings for this image. + /// + /// Represented as an f-stop value. + final double? lensAperture; + + /// The sensor exposure time for this image in nanoseconds. + final int? sensorExposureTime; + + /// The sensor sensitivity in standard ISO arithmetic units. + final double? sensorSensitivity; +} diff --git a/packages/camera/camera_platform_interface/lib/src/types/types.dart b/packages/camera/camera_platform_interface/lib/src/types/types.dart index 0c24839d6445..3eb09fcb833c 100644 --- a/packages/camera/camera_platform_interface/lib/src/types/types.dart +++ b/packages/camera/camera_platform_interface/lib/src/types/types.dart @@ -4,6 +4,7 @@ export 'camera_description.dart'; export 'camera_exception.dart'; +export 'camera_image_data.dart'; export 'exposure_mode.dart'; export 'flash_mode.dart'; export 'focus_mode.dart'; diff --git a/packages/camera/camera_platform_interface/pubspec.yaml b/packages/camera/camera_platform_interface/pubspec.yaml index ab163b4e9f3f..473dcb552c82 100644 --- a/packages/camera/camera_platform_interface/pubspec.yaml +++ b/packages/camera/camera_platform_interface/pubspec.yaml @@ -4,7 +4,7 @@ repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera_ issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 # NOTE: We strongly prefer non-breaking changes, even at the expense of a # less-clean API. See https://flutter.dev/go/platform-interface-breaking-changes -version: 2.1.6 +version: 2.2.0 environment: sdk: '>=2.12.0 <3.0.0' diff --git a/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart b/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart index 7da4262cdf79..d096f0012c86 100644 --- a/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart +++ b/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart @@ -1038,6 +1038,52 @@ void main() { arguments: {'cameraId': cameraId}), ]); }); + + test('Should start streaming', () async { + // Arrange + final MethodChannelMock channel = MethodChannelMock( + channelName: 'plugins.flutter.io/camera', + methods: { + 'startImageStream': null, + 'stopImageStream': null, + }, + ); + + // Act + final StreamSubscription subscription = camera + .onStreamedFrameAvailable(cameraId) + .listen((CameraImageData imageData) {}); + + // Assert + expect(channel.log, [ + isMethodCall('startImageStream', arguments: null), + ]); + + subscription.cancel(); + }); + + test('Should stop streaming', () async { + // Arrange + final MethodChannelMock channel = MethodChannelMock( + channelName: 'plugins.flutter.io/camera', + methods: { + 'startImageStream': null, + 'stopImageStream': null, + }, + ); + + // Act + final StreamSubscription subscription = camera + .onStreamedFrameAvailable(cameraId) + .listen((CameraImageData imageData) {}); + subscription.cancel(); + + // Assert + expect(channel.log, [ + isMethodCall('startImageStream', arguments: null), + isMethodCall('stopImageStream', arguments: null), + ]); + }); }); }); } diff --git a/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart b/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart new file mode 100644 index 000000000000..a8ca45eca43b --- /dev/null +++ b/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart @@ -0,0 +1,85 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:camera_platform_interface/camera_platform_interface.dart'; +import 'package:camera_platform_interface/src/method_channel/type_conversion.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('CameraImageData can be created', () { + final CameraImageData cameraImage = + cameraImageFromPlatformData({ + 'format': 35, + 'height': 1, + 'width': 4, + 'lensAperture': 1.8, + 'sensorExposureTime': 9991324, + 'sensorSensitivity': 92.0, + 'planes': [ + { + 'bytes': Uint8List.fromList([1, 2, 3, 4]), + 'bytesPerPixel': 1, + 'bytesPerRow': 4, + 'height': 1, + 'width': 4 + } + ] + }); + expect(cameraImage.height, 1); + expect(cameraImage.width, 4); + expect(cameraImage.format.group, ImageFormatGroup.yuv420); + expect(cameraImage.planes.length, 1); + }); + + test('CameraImageData has ImageFormatGroup.yuv420 for iOS', () { + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + + final CameraImageData cameraImage = + cameraImageFromPlatformData({ + 'format': 875704438, + 'height': 1, + 'width': 4, + 'lensAperture': 1.8, + 'sensorExposureTime': 9991324, + 'sensorSensitivity': 92.0, + 'planes': [ + { + 'bytes': Uint8List.fromList([1, 2, 3, 4]), + 'bytesPerPixel': 1, + 'bytesPerRow': 4, + 'height': 1, + 'width': 4 + } + ] + }); + expect(cameraImage.format.group, ImageFormatGroup.yuv420); + }); + + test('CameraImageData has ImageFormatGroup.yuv420 for Android', () { + debugDefaultTargetPlatformOverride = TargetPlatform.android; + + final CameraImageData cameraImage = + cameraImageFromPlatformData({ + 'format': 35, + 'height': 1, + 'width': 4, + 'lensAperture': 1.8, + 'sensorExposureTime': 9991324, + 'sensorSensitivity': 92.0, + 'planes': [ + { + 'bytes': Uint8List.fromList([1, 2, 3, 4]), + 'bytesPerPixel': 1, + 'bytesPerRow': 4, + 'height': 1, + 'width': 4 + } + ] + }); + expect(cameraImage.format.group, ImageFormatGroup.yuv420); + }); +} diff --git a/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart b/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart new file mode 100644 index 000000000000..f06213e2b0e4 --- /dev/null +++ b/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart @@ -0,0 +1,38 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:camera_platform_interface/camera_platform_interface.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('CameraImageData can be created', () { + debugDefaultTargetPlatformOverride = TargetPlatform.android; + final CameraImageData cameraImage = CameraImageData( + format: const CameraImageFormat(ImageFormatGroup.jpeg, raw: 42), + height: 100, + width: 200, + lensAperture: 1.8, + sensorExposureTime: 11, + sensorSensitivity: 92.0, + planes: [ + CameraImagePlane( + bytes: Uint8List.fromList([1, 2, 3, 4]), + bytesPerRow: 4, + bytesPerPixel: 2, + height: 100, + width: 200) + ], + ); + expect(cameraImage.format.group, ImageFormatGroup.jpeg); + expect(cameraImage.lensAperture, 1.8); + expect(cameraImage.sensorExposureTime, 11); + expect(cameraImage.sensorSensitivity, 92.0); + expect(cameraImage.height, 100); + expect(cameraImage.width, 200); + expect(cameraImage.planes.length, 1); + }); +}