[firebase_ml_vision] Add support for cloud text recognizer (#1847)
diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index 058b0d0..f25d393 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md
@@ -1,3 +1,7 @@ +## 0.9.1 + +* Add support for cloud text recognizer. + ## 0.9.0+3 * Automatically use version from pubspec.yaml when reporting usage to Firebase.
diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java index 55aa3c8..9a0f823 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java
@@ -21,7 +21,15 @@ private final FirebaseVisionTextRecognizer recognizer; TextRecognizer(FirebaseVision vision, Map<String, Object> options) { - recognizer = vision.getOnDeviceTextRecognizer(); + final String modelType = (String) options.get("modelType"); + if (modelType.equals("onDevice")) { + recognizer = vision.getOnDeviceTextRecognizer(); + } else if (modelType.equals("cloud")) { + recognizer = vision.getCloudTextRecognizer(); + } else { + final String message = String.format("No model for type: %s", modelType); + throw new IllegalArgumentException(message); + } } @Override
diff --git a/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart b/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart index c8c5eed..f40e20d 100644 --- a/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart +++ b/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart
@@ -29,6 +29,8 @@ final ImageLabeler _cloudImageLabeler = FirebaseVision.instance.cloudImageLabeler(); final TextRecognizer _recognizer = FirebaseVision.instance.textRecognizer(); + final TextRecognizer _cloudRecognizer = + FirebaseVision.instance.cloudTextRecognizer(); @override void initState() { @@ -72,6 +74,8 @@ switch (_currentDetector) { case Detector.text: return _recognizer.processImage; + case Detector.cloudText: + return _cloudRecognizer.processImage; case Detector.barcode: return _barcodeDetector.detectInImage; case Detector.label: @@ -119,7 +123,8 @@ painter = LabelDetectorPainter(imageSize, _scanResults); break; default: - assert(_currentDetector == Detector.text); + assert(_currentDetector == Detector.text || + _currentDetector == Detector.cloudText); if (_scanResults is! VisionText) return noResultsText; painter = TextDetectorPainter(imageSize, _scanResults); } @@ -200,6 +205,10 @@ child: Text('Detect Text'), value: Detector.text, ), + const PopupMenuItem<Detector>( + child: Text('Detect Cloud Text'), + value: Detector.cloudText, + ), ], ), ], @@ -222,6 +231,7 @@ _imageLabeler.close(); _cloudImageLabeler.close(); _recognizer.close(); + _cloudRecognizer.close(); }); _currentDetector = null;
diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index 684503b..d8d7316 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart
@@ -7,7 +7,7 @@ import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:flutter/material.dart'; -enum Detector { barcode, face, label, cloudLabel, text } +enum Detector { barcode, face, label, cloudLabel, text, cloudText } class BarcodeDetectorPainter extends CustomPainter { BarcodeDetectorPainter(this.absoluteImageSize, this.barcodeLocations);
diff --git a/packages/firebase_ml_vision/example/lib/picture_scanner.dart b/packages/firebase_ml_vision/example/lib/picture_scanner.dart index 88b1c5a..182a55a 100644 --- a/packages/firebase_ml_vision/example/lib/picture_scanner.dart +++ b/packages/firebase_ml_vision/example/lib/picture_scanner.dart
@@ -29,6 +29,8 @@ final ImageLabeler _cloudImageLabeler = FirebaseVision.instance.cloudImageLabeler(); final TextRecognizer _recognizer = FirebaseVision.instance.textRecognizer(); + final TextRecognizer _cloudRecognizer = + FirebaseVision.instance.cloudTextRecognizer(); Future<void> _getAndScanImage() async { setState(() { @@ -93,6 +95,9 @@ case Detector.text: results = await _recognizer.processImage(visionImage); break; + case Detector.cloudText: + results = await _cloudRecognizer.processImage(visionImage); + break; default: return; } @@ -121,6 +126,9 @@ case Detector.text: painter = TextDetectorPainter(_imageSize, results); break; + case Detector.cloudText: + painter = TextDetectorPainter(_imageSize, results); + break; default: break; } @@ -185,6 +193,10 @@ child: Text('Detect Text'), value: Detector.text, ), + const PopupMenuItem<Detector>( + child: Text('Detect Cloud Text'), + value: Detector.cloudText, + ), ], ), ], @@ -207,6 +219,7 @@ _imageLabeler.close(); _cloudImageLabeler.close(); _recognizer.close(); + _cloudRecognizer.close(); super.dispose(); } }
diff --git a/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m b/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m index fa9bdd3..cc8cc93 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m +++ b/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m
@@ -8,7 +8,17 @@ - (instancetype)initWithVision:(FIRVision *)vision options:(NSDictionary *)options { self = [super init]; if (self) { - _recognizer = [vision onDeviceTextRecognizer]; + if ([@"onDevice" isEqualToString:options[@"modelType"]]) { + _recognizer = [vision onDeviceTextRecognizer]; + } else if ([@"cloud" isEqualToString:options[@"modelType"]]) { + _recognizer = [vision cloudTextRecognizer]; + } else { + NSString *reason = + [NSString stringWithFormat:@"Invalid model type: %@", options[@"modelType"]]; + @throw [[NSException alloc] initWithName:NSInvalidArgumentException + reason:reason + userInfo:nil]; + } } return self; }
diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 6f6b044..d0f593c 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart
@@ -67,7 +67,12 @@ } /// Creates an instance of [TextRecognizer]. - TextRecognizer textRecognizer() => TextRecognizer._(nextHandle++); + TextRecognizer textRecognizer() { + return TextRecognizer._( + modelType: ModelType.onDevice, + handle: nextHandle++, + ); + } /// Creates a cloud instance of [ImageLabeler]. ImageLabeler cloudImageLabeler([CloudImageLabelerOptions options]) { @@ -77,6 +82,14 @@ handle: nextHandle++, ); } + + /// Creates a cloud instance of [TextRecognizer]. + TextRecognizer cloudTextRecognizer() { + return TextRecognizer._( + modelType: ModelType.cloud, + handle: nextHandle++, + ); + } } /// Represents an image object used for both on-device and cloud API detectors.
diff --git a/packages/firebase_ml_vision/lib/src/text_recognizer.dart b/packages/firebase_ml_vision/lib/src/text_recognizer.dart index 888f5fd..98936f6 100644 --- a/packages/firebase_ml_vision/lib/src/text_recognizer.dart +++ b/packages/firebase_ml_vision/lib/src/text_recognizer.dart
@@ -19,7 +19,13 @@ /// await textRecognizer.processImage(image); /// ``` class TextRecognizer { - TextRecognizer._(this._handle); + TextRecognizer._({ + @required this.modelType, + @required int handle, + }) : _handle = handle, + assert(modelType != null); + + final ModelType modelType; final int _handle; bool _hasBeenOpened = false; @@ -35,7 +41,9 @@ 'TextRecognizer#processImage', <String, dynamic>{ 'handle': _handle, - 'options': <String, dynamic>{}, + 'options': <String, dynamic>{ + 'modelType': _enumToString(modelType), + }, }..addAll(visionImage._serialize()), );
diff --git a/packages/firebase_ml_vision/pubspec.yaml b/packages/firebase_ml_vision/pubspec.yaml index 765b72b..c2138b1 100644 --- a/packages/firebase_ml_vision/pubspec.yaml +++ b/packages/firebase_ml_vision/pubspec.yaml
@@ -2,7 +2,7 @@ description: Flutter plugin for Firebase machine learning vision services. author: Flutter Team <flutter-dev@googlegroups.com> homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision -version: 0.9.0+3 +version: 0.9.1 dependencies: flutter:
diff --git a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart index 9420dda..85e0697 100644 --- a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart +++ b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart
@@ -83,7 +83,9 @@ }, ], }, - 'options': <String, dynamic>{}, + 'options': <String, dynamic>{ + 'modelType': 'onDevice', + }, }, ), ]); @@ -778,7 +780,6 @@ expect(block.confidence, 0.5); block = text.blocks[1]; - // ignore: prefer_const_constructors // TODO(jackson): Use const Rect when available in minimum Flutter SDK // ignore: prefer_const_constructors expect(block.boundingBox, Rect.fromLTWH(14.0, 13.0, 16.0, 15.0)); @@ -865,7 +866,272 @@ 'path': 'empty', 'bytes': null, 'metadata': null, - 'options': <String, dynamic>{}, + 'options': <String, dynamic>{ + 'modelType': 'onDevice', + }, + }, + ), + ]); + }); + + test('processImage no bounding box', () async { + returnValue = <dynamic, dynamic>{ + 'blocks': <dynamic>[ + <dynamic, dynamic>{ + 'text': '', + 'points': <dynamic>[], + 'recognizedLanguages': <dynamic>[], + 'lines': <dynamic>[], + }, + ], + }; + + final VisionText text = await recognizer.processImage(image); + + final TextBlock block = text.blocks[0]; + expect(block.boundingBox, null); + }); + }); + + group('Cloud $TextRecognizer', () { + TextRecognizer recognizer; + final FirebaseVisionImage image = FirebaseVisionImage.fromFilePath( + 'empty', + ); + + setUp(() { + recognizer = FirebaseVision.instance.cloudTextRecognizer(); + final List<dynamic> elements = <dynamic>[ + <dynamic, dynamic>{ + 'text': 'hello', + 'left': 1.0, + 'top': 2.0, + 'width': 3.0, + 'height': 4.0, + 'points': <dynamic>[ + <dynamic>[5.0, 6.0], + <dynamic>[7.0, 8.0], + ], + 'recognizedLanguages': <dynamic>[ + <dynamic, dynamic>{ + 'languageCode': 'ab', + }, + <dynamic, dynamic>{ + 'languageCode': 'cd', + } + ], + 'confidence': 0.1, + }, + <dynamic, dynamic>{ + 'text': 'my', + 'left': 4.0, + 'top': 3.0, + 'width': 2.0, + 'height': 1.0, + 'points': <dynamic>[ + <dynamic>[6.0, 5.0], + <dynamic>[8.0, 7.0], + ], + 'recognizedLanguages': <dynamic>[], + 'confidence': 0.2, + }, + ]; + + final List<dynamic> lines = <dynamic>[ + <dynamic, dynamic>{ + 'text': 'friend', + 'left': 5.0, + 'top': 6.0, + 'width': 7.0, + 'height': 8.0, + 'points': <dynamic>[ + <dynamic>[9.0, 10.0], + <dynamic>[11.0, 12.0], + ], + 'recognizedLanguages': <dynamic>[ + <dynamic, dynamic>{ + 'languageCode': 'ef', + }, + <dynamic, dynamic>{ + 'languageCode': 'gh', + } + ], + 'elements': elements, + 'confidence': 0.3, + }, + <dynamic, dynamic>{ + 'text': 'how', + 'left': 8.0, + 'top': 7.0, + 'width': 4.0, + 'height': 5.0, + 'points': <dynamic>[ + <dynamic>[10.0, 9.0], + <dynamic>[12.0, 11.0], + ], + 'recognizedLanguages': <dynamic>[], + 'elements': <dynamic>[], + 'confidence': 0.4, + }, + ]; + + final List<dynamic> blocks = <dynamic>[ + <dynamic, dynamic>{ + 'text': 'friend', + 'left': 13.0, + 'top': 14.0, + 'width': 15.0, + 'height': 16.0, + 'points': <dynamic>[ + <dynamic>[17.0, 18.0], + <dynamic>[19.0, 20.0], + ], + 'recognizedLanguages': <dynamic>[ + <dynamic, dynamic>{ + 'languageCode': 'ij', + }, + <dynamic, dynamic>{ + 'languageCode': 'kl', + } + ], + 'lines': lines, + 'confidence': 0.5, + }, + <dynamic, dynamic>{ + 'text': 'hello', + 'left': 14.0, + 'top': 13.0, + 'width': 16.0, + 'height': 15.0, + 'points': <dynamic>[ + <dynamic>[18.0, 17.0], + <dynamic>[20.0, 19.0], + ], + 'recognizedLanguages': <dynamic>[], + 'lines': <dynamic>[], + 'confidence': 0.6, + }, + ]; + + final dynamic visionText = <dynamic, dynamic>{ + 'text': 'testext', + 'blocks': blocks, + }; + + returnValue = visionText; + }); + + group('$TextBlock', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + expect(text.blocks, hasLength(2)); + + TextBlock block = text.blocks[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(block.boundingBox, Rect.fromLTWH(13.0, 14.0, 15.0, 16.0)); + expect(block.text, 'friend'); + expect(block.cornerPoints, const <Offset>[ + Offset(17.0, 18.0), + Offset(19.0, 20.0), + ]); + expect(block.recognizedLanguages, hasLength(2)); + expect(block.recognizedLanguages[0].languageCode, 'ij'); + expect(block.recognizedLanguages[1].languageCode, 'kl'); + expect(block.confidence, 0.5); + + block = text.blocks[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(block.boundingBox, Rect.fromLTWH(14.0, 13.0, 16.0, 15.0)); + expect(block.text, 'hello'); + expect(block.cornerPoints, const <Offset>[ + Offset(18.0, 17.0), + Offset(20.0, 19.0), + ]); + expect(block.confidence, 0.6); + }); + }); + + group('$TextLine', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + TextLine line = text.blocks[0].lines[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(line.boundingBox, Rect.fromLTWH(5, 6, 7, 8)); + expect(line.text, 'friend'); + expect(line.cornerPoints, const <Offset>[ + Offset(9.0, 10.0), + Offset(11.0, 12.0), + ]); + expect(line.recognizedLanguages, hasLength(2)); + expect(line.recognizedLanguages[0].languageCode, 'ef'); + expect(line.recognizedLanguages[1].languageCode, 'gh'); + expect(line.confidence, 0.3); + + line = text.blocks[0].lines[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(line.boundingBox, Rect.fromLTWH(8.0, 7.0, 4.0, 5.0)); + expect(line.text, 'how'); + expect(line.cornerPoints, const <Offset>[ + Offset(10.0, 9.0), + Offset(12.0, 11.0), + ]); + expect(line.confidence, 0.4); + }); + }); + + group('$TextElement', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + TextElement element = text.blocks[0].lines[0].elements[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(element.boundingBox, Rect.fromLTWH(1.0, 2.0, 3.0, 4.0)); + expect(element.text, 'hello'); + expect(element.cornerPoints, const <Offset>[ + Offset(5.0, 6.0), + Offset(7.0, 8.0), + ]); + expect(element.recognizedLanguages, hasLength(2)); + expect(element.recognizedLanguages[0].languageCode, 'ab'); + expect(element.recognizedLanguages[1].languageCode, 'cd'); + expect(element.confidence, 0.1); + + element = text.blocks[0].lines[0].elements[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(element.boundingBox, Rect.fromLTWH(4.0, 3.0, 2.0, 1.0)); + expect(element.text, 'my'); + expect(element.cornerPoints, const <Offset>[ + Offset(6.0, 5.0), + Offset(8.0, 7.0), + ]); + expect(element.confidence, 0.2); + }); + }); + + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + expect(text.text, 'testext'); + expect(log, <Matcher>[ + isMethodCall( + 'TextRecognizer#processImage', + arguments: <String, dynamic>{ + 'handle': 0, + 'type': 'file', + 'path': 'empty', + 'bytes': null, + 'metadata': null, + 'options': <String, dynamic>{ + 'modelType': 'cloud', + }, }, ), ]);