[camera] Move camera streaming to platform interface (#5783)
diff --git a/packages/camera/camera_platform_interface/CHANGELOG.md b/packages/camera/camera_platform_interface/CHANGELOG.md
index 3cad35d..5ecd889 100644
--- a/packages/camera/camera_platform_interface/CHANGELOG.md
+++ b/packages/camera/camera_platform_interface/CHANGELOG.md
@@ -1,5 +1,6 @@
-## NEXT
+## 2.2.0
+* Adds image streaming to the platform interface.
* Removes unnecessary imports.
## 2.1.6
diff --git a/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart b/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart
index c856f34..babef14 100644
--- a/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart
+++ b/packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart
@@ -12,6 +12,8 @@
import 'package:flutter/widgets.dart';
import 'package:stream_transform/stream_transform.dart';
+import 'type_conversion.dart';
+
const MethodChannel _channel = MethodChannel('plugins.flutter.io/camera');
/// An implementation of [CameraPlatform] that uses method channels.
@@ -48,6 +50,12 @@
final StreamController<DeviceEvent> deviceEventStreamController =
StreamController<DeviceEvent>.broadcast();
+ // The stream to receive frames from the native code.
+ StreamSubscription<dynamic>? _platformImageStreamSubscription;
+
+ // The stream for vending frames to platform interface clients.
+ StreamController<CameraImageData>? _frameStreamController;
+
Stream<CameraEvent> _cameraEvents(int cameraId) =>
cameraEventStreamController.stream
.where((CameraEvent event) => event.cameraId == cameraId);
@@ -268,6 +276,52 @@
);
@override
+ Stream<CameraImageData> onStreamedFrameAvailable(int cameraId,
+ {CameraImageStreamOptions? options}) {
+ _frameStreamController = StreamController<CameraImageData>(
+ onListen: _onFrameStreamListen,
+ onPause: _onFrameStreamPauseResume,
+ onResume: _onFrameStreamPauseResume,
+ onCancel: _onFrameStreamCancel,
+ );
+ return _frameStreamController!.stream;
+ }
+
+ void _onFrameStreamListen() {
+ _startPlatformStream();
+ }
+
+ Future<void> _startPlatformStream() async {
+ await _channel.invokeMethod<void>('startImageStream');
+ const EventChannel cameraEventChannel =
+ EventChannel('plugins.flutter.io/camera/imageStream');
+ _platformImageStreamSubscription =
+ cameraEventChannel.receiveBroadcastStream().listen((dynamic imageData) {
+ if (defaultTargetPlatform == TargetPlatform.iOS) {
+ try {
+ _channel.invokeMethod<void>('receivedImageStreamData');
+ } on PlatformException catch (e) {
+ throw CameraException(e.code, e.message);
+ }
+ }
+ _frameStreamController!
+ .add(cameraImageFromPlatformData(imageData as Map<dynamic, dynamic>));
+ });
+ }
+
+ FutureOr<void> _onFrameStreamCancel() async {
+ await _channel.invokeMethod<void>('stopImageStream');
+ await _platformImageStreamSubscription?.cancel();
+ _platformImageStreamSubscription = null;
+ _frameStreamController = null;
+ }
+
+ void _onFrameStreamPauseResume() {
+ throw CameraException('InvalidCall',
+ 'Pause and resume are not supported for onStreamedFrameAvailable');
+ }
+
+ @override
Future<void> setFlashMode(int cameraId, FlashMode mode) =>
_channel.invokeMethod<void>(
'setFlashMode',
diff --git a/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart b/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart
new file mode 100644
index 0000000..9dffbbf
--- /dev/null
+++ b/packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart
@@ -0,0 +1,61 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:flutter/foundation.dart';
+
+import '../types/types.dart';
+
+/// Converts method channel call [data] for `receivedImageStreamData` to a
+/// [CameraImageData].
+CameraImageData cameraImageFromPlatformData(Map<dynamic, dynamic> data) {
+ return CameraImageData(
+ format: _cameraImageFormatFromPlatformData(data['format']),
+ height: data['height'] as int,
+ width: data['width'] as int,
+ lensAperture: data['lensAperture'] as double?,
+ sensorExposureTime: data['sensorExposureTime'] as int?,
+ sensorSensitivity: data['sensorSensitivity'] as double?,
+ planes: List<CameraImagePlane>.unmodifiable(
+ (data['planes'] as List<dynamic>).map<CameraImagePlane>(
+ (dynamic planeData) => _cameraImagePlaneFromPlatformData(
+ planeData as Map<dynamic, dynamic>))));
+}
+
+CameraImageFormat _cameraImageFormatFromPlatformData(dynamic data) {
+ return CameraImageFormat(_imageFormatGroupFromPlatformData(data), raw: data);
+}
+
+ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) {
+ if (defaultTargetPlatform == TargetPlatform.android) {
+ switch (data) {
+ case 35: // android.graphics.ImageFormat.YUV_420_888
+ return ImageFormatGroup.yuv420;
+ case 256: // android.graphics.ImageFormat.JPEG
+ return ImageFormatGroup.jpeg;
+ }
+ }
+
+ if (defaultTargetPlatform == TargetPlatform.iOS) {
+ switch (data) {
+ case 875704438: // kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+ return ImageFormatGroup.yuv420;
+
+ case 1111970369: // kCVPixelFormatType_32BGRA
+ return ImageFormatGroup.bgra8888;
+ }
+ }
+
+ return ImageFormatGroup.unknown;
+}
+
+CameraImagePlane _cameraImagePlaneFromPlatformData(Map<dynamic, dynamic> data) {
+ return CameraImagePlane(
+ bytes: data['bytes'] as Uint8List,
+ bytesPerPixel: data['bytesPerPixel'] as int?,
+ bytesPerRow: data['bytesPerRow'] as int,
+ height: data['height'] as int?,
+ width: data['width'] as int?);
+}
diff --git a/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart b/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart
index daa19b8..eaa779a 100644
--- a/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart
+++ b/packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart
@@ -149,6 +149,21 @@
throw UnimplementedError('resumeVideoRecording() is not implemented.');
}
+ /// A new streamed frame is available.
+ ///
+ /// Listening to this stream will start streaming, and canceling will stop.
+ /// Pausing will throw a [CameraException], as pausing the stream would cause
+ /// very high memory usage; to temporarily stop receiving frames, cancel, then
+ /// listen again later.
+ ///
+ ///
+ // TODO(bmparr): Add options to control streaming settings (e.g.,
+ // resolution and FPS).
+ Stream<CameraImageData> onStreamedFrameAvailable(int cameraId,
+ {CameraImageStreamOptions? options}) {
+ throw UnimplementedError('onStreamedFrameAvailable() is not implemented.');
+ }
+
/// Sets the flash mode for the selected camera.
/// On Web [FlashMode.auto] corresponds to [FlashMode.always].
Future<void> setFlashMode(int cameraId, FlashMode mode) {
diff --git a/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart b/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart
new file mode 100644
index 0000000..6971dbb
--- /dev/null
+++ b/packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart
@@ -0,0 +1,126 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:flutter/foundation.dart';
+
+import '../../camera_platform_interface.dart';
+
+/// Options for configuring camera streaming.
+///
+/// Currently unused; this exists for future-proofing of the platform interface
+/// API.
+@immutable
+class CameraImageStreamOptions {}
+
+/// A single color plane of image data.
+///
+/// The number and meaning of the planes in an image are determined by its
+/// format.
+@immutable
+class CameraImagePlane {
+ /// Creates a new instance with the given bytes and optional metadata.
+ const CameraImagePlane({
+ required this.bytes,
+ required this.bytesPerRow,
+ this.bytesPerPixel,
+ this.height,
+ this.width,
+ });
+
+ /// Bytes representing this plane.
+ final Uint8List bytes;
+
+ /// The row stride for this color plane, in bytes.
+ final int bytesPerRow;
+
+ /// The distance between adjacent pixel samples in bytes, when available.
+ final int? bytesPerPixel;
+
+ /// Height of the pixel buffer, when available.
+ final int? height;
+
+ /// Width of the pixel buffer, when available.
+ final int? width;
+}
+
+/// Describes how pixels are represented in an image.
+@immutable
+class CameraImageFormat {
+ /// Create a new format with the given cross-platform group and raw underyling
+ /// platform identifier.
+ const CameraImageFormat(this.group, {required this.raw});
+
+ /// Describes the format group the raw image format falls into.
+ final ImageFormatGroup group;
+
+ /// Raw version of the format from the underlying platform.
+ ///
+ /// On Android, this should be an `int` from class
+ /// `android.graphics.ImageFormat`. See
+ /// https://developer.android.com/reference/android/graphics/ImageFormat
+ ///
+ /// On iOS, this should be a `FourCharCode` constant from Pixel Format
+ /// Identifiers. See
+ /// https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers
+ final dynamic raw;
+}
+
+/// A single complete image buffer from the platform camera.
+///
+/// This class allows for direct application access to the pixel data of an
+/// Image through one or more [Uint8List]. Each buffer is encapsulated in a
+/// [CameraImagePlane] that describes the layout of the pixel data in that
+/// plane. [CameraImageData] is not directly usable as a UI resource.
+///
+/// Although not all image formats are planar on all platforms, this class
+/// treats 1-dimensional images as single planar images.
+@immutable
+class CameraImageData {
+ /// Creates a new instance with the given format, planes, and metadata.
+ const CameraImageData({
+ required this.format,
+ required this.planes,
+ required this.height,
+ required this.width,
+ this.lensAperture,
+ this.sensorExposureTime,
+ this.sensorSensitivity,
+ });
+
+ /// Format of the image provided.
+ ///
+ /// Determines the number of planes needed to represent the image, and
+ /// the general layout of the pixel data in each [Uint8List].
+ final CameraImageFormat format;
+
+ /// Height of the image in pixels.
+ ///
+ /// For formats where some color channels are subsampled, this is the height
+ /// of the largest-resolution plane.
+ final int height;
+
+ /// Width of the image in pixels.
+ ///
+ /// For formats where some color channels are subsampled, this is the width
+ /// of the largest-resolution plane.
+ final int width;
+
+ /// The pixels planes for this image.
+ ///
+ /// The number of planes is determined by the format of the image.
+ final List<CameraImagePlane> planes;
+
+ /// The aperture settings for this image.
+ ///
+ /// Represented as an f-stop value.
+ final double? lensAperture;
+
+ /// The sensor exposure time for this image in nanoseconds.
+ final int? sensorExposureTime;
+
+ /// The sensor sensitivity in standard ISO arithmetic units.
+ final double? sensorSensitivity;
+}
diff --git a/packages/camera/camera_platform_interface/lib/src/types/types.dart b/packages/camera/camera_platform_interface/lib/src/types/types.dart
index 0c24839..3eb09fc 100644
--- a/packages/camera/camera_platform_interface/lib/src/types/types.dart
+++ b/packages/camera/camera_platform_interface/lib/src/types/types.dart
@@ -4,6 +4,7 @@
export 'camera_description.dart';
export 'camera_exception.dart';
+export 'camera_image_data.dart';
export 'exposure_mode.dart';
export 'flash_mode.dart';
export 'focus_mode.dart';
diff --git a/packages/camera/camera_platform_interface/pubspec.yaml b/packages/camera/camera_platform_interface/pubspec.yaml
index ab163b4..473dcb5 100644
--- a/packages/camera/camera_platform_interface/pubspec.yaml
+++ b/packages/camera/camera_platform_interface/pubspec.yaml
@@ -4,7 +4,7 @@
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
# NOTE: We strongly prefer non-breaking changes, even at the expense of a
# less-clean API. See https://flutter.dev/go/platform-interface-breaking-changes
-version: 2.1.6
+version: 2.2.0
environment:
sdk: '>=2.12.0 <3.0.0'
diff --git a/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart b/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart
index 7da4262..d096f00 100644
--- a/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart
+++ b/packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart
@@ -1038,6 +1038,52 @@
arguments: <String, Object?>{'cameraId': cameraId}),
]);
});
+
+ test('Should start streaming', () async {
+ // Arrange
+ final MethodChannelMock channel = MethodChannelMock(
+ channelName: 'plugins.flutter.io/camera',
+ methods: <String, dynamic>{
+ 'startImageStream': null,
+ 'stopImageStream': null,
+ },
+ );
+
+ // Act
+ final StreamSubscription<CameraImageData> subscription = camera
+ .onStreamedFrameAvailable(cameraId)
+ .listen((CameraImageData imageData) {});
+
+ // Assert
+ expect(channel.log, <Matcher>[
+ isMethodCall('startImageStream', arguments: null),
+ ]);
+
+ subscription.cancel();
+ });
+
+ test('Should stop streaming', () async {
+ // Arrange
+ final MethodChannelMock channel = MethodChannelMock(
+ channelName: 'plugins.flutter.io/camera',
+ methods: <String, dynamic>{
+ 'startImageStream': null,
+ 'stopImageStream': null,
+ },
+ );
+
+ // Act
+ final StreamSubscription<CameraImageData> subscription = camera
+ .onStreamedFrameAvailable(cameraId)
+ .listen((CameraImageData imageData) {});
+ subscription.cancel();
+
+ // Assert
+ expect(channel.log, <Matcher>[
+ isMethodCall('startImageStream', arguments: null),
+ isMethodCall('stopImageStream', arguments: null),
+ ]);
+ });
});
});
}
diff --git a/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart b/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart
new file mode 100644
index 0000000..a8ca45e
--- /dev/null
+++ b/packages/camera/camera_platform_interface/test/method_channel/type_conversion_test.dart
@@ -0,0 +1,85 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:camera_platform_interface/camera_platform_interface.dart';
+import 'package:camera_platform_interface/src/method_channel/type_conversion.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+ test('CameraImageData can be created', () {
+ final CameraImageData cameraImage =
+ cameraImageFromPlatformData(<dynamic, dynamic>{
+ 'format': 35,
+ 'height': 1,
+ 'width': 4,
+ 'lensAperture': 1.8,
+ 'sensorExposureTime': 9991324,
+ 'sensorSensitivity': 92.0,
+ 'planes': <dynamic>[
+ <dynamic, dynamic>{
+ 'bytes': Uint8List.fromList(<int>[1, 2, 3, 4]),
+ 'bytesPerPixel': 1,
+ 'bytesPerRow': 4,
+ 'height': 1,
+ 'width': 4
+ }
+ ]
+ });
+ expect(cameraImage.height, 1);
+ expect(cameraImage.width, 4);
+ expect(cameraImage.format.group, ImageFormatGroup.yuv420);
+ expect(cameraImage.planes.length, 1);
+ });
+
+ test('CameraImageData has ImageFormatGroup.yuv420 for iOS', () {
+ debugDefaultTargetPlatformOverride = TargetPlatform.iOS;
+
+ final CameraImageData cameraImage =
+ cameraImageFromPlatformData(<dynamic, dynamic>{
+ 'format': 875704438,
+ 'height': 1,
+ 'width': 4,
+ 'lensAperture': 1.8,
+ 'sensorExposureTime': 9991324,
+ 'sensorSensitivity': 92.0,
+ 'planes': <dynamic>[
+ <dynamic, dynamic>{
+ 'bytes': Uint8List.fromList(<int>[1, 2, 3, 4]),
+ 'bytesPerPixel': 1,
+ 'bytesPerRow': 4,
+ 'height': 1,
+ 'width': 4
+ }
+ ]
+ });
+ expect(cameraImage.format.group, ImageFormatGroup.yuv420);
+ });
+
+ test('CameraImageData has ImageFormatGroup.yuv420 for Android', () {
+ debugDefaultTargetPlatformOverride = TargetPlatform.android;
+
+ final CameraImageData cameraImage =
+ cameraImageFromPlatformData(<dynamic, dynamic>{
+ 'format': 35,
+ 'height': 1,
+ 'width': 4,
+ 'lensAperture': 1.8,
+ 'sensorExposureTime': 9991324,
+ 'sensorSensitivity': 92.0,
+ 'planes': <dynamic>[
+ <dynamic, dynamic>{
+ 'bytes': Uint8List.fromList(<int>[1, 2, 3, 4]),
+ 'bytesPerPixel': 1,
+ 'bytesPerRow': 4,
+ 'height': 1,
+ 'width': 4
+ }
+ ]
+ });
+ expect(cameraImage.format.group, ImageFormatGroup.yuv420);
+ });
+}
diff --git a/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart b/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart
new file mode 100644
index 0000000..f06213e
--- /dev/null
+++ b/packages/camera/camera_platform_interface/test/types/camera_image_data_test.dart
@@ -0,0 +1,38 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+import 'package:camera_platform_interface/camera_platform_interface.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+ test('CameraImageData can be created', () {
+ debugDefaultTargetPlatformOverride = TargetPlatform.android;
+ final CameraImageData cameraImage = CameraImageData(
+ format: const CameraImageFormat(ImageFormatGroup.jpeg, raw: 42),
+ height: 100,
+ width: 200,
+ lensAperture: 1.8,
+ sensorExposureTime: 11,
+ sensorSensitivity: 92.0,
+ planes: <CameraImagePlane>[
+ CameraImagePlane(
+ bytes: Uint8List.fromList(<int>[1, 2, 3, 4]),
+ bytesPerRow: 4,
+ bytesPerPixel: 2,
+ height: 100,
+ width: 200)
+ ],
+ );
+ expect(cameraImage.format.group, ImageFormatGroup.jpeg);
+ expect(cameraImage.lensAperture, 1.8);
+ expect(cameraImage.sensorExposureTime, 11);
+ expect(cameraImage.sensorSensitivity, 92.0);
+ expect(cameraImage.height, 100);
+ expect(cameraImage.width, 200);
+ expect(cameraImage.planes.length, 1);
+ });
+}
