Skip to content

fix: Add support for YUV_420_888 Image format. #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

fix: Add support for YUV_420_888 Image format. #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 35 additions & 12 deletions packages/example/lib/vision_detector_views/camera_view.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,29 +358,52 @@ class _CameraViewState extends State<CameraView> {

// get image format
final format = InputImageFormatValue.fromRawValue(image.format.raw);
// validate format depending on platform
// only supported formats:
// * nv21 for Android
// * bgra8888 for iOS
if (format == null ||
(Platform.isAndroid && format != InputImageFormat.nv21) ||
if (format == null) {
print('could not find format from raw value: $image.format.raw');
return null;
}
// Validate format depending on platform
final androidSupportedFormats = [
InputImageFormat.nv21,
InputImageFormat.yv12,
InputImageFormat.yuv_420_888
];
if ((Platform.isAndroid && !androidSupportedFormats.contains(format)) ||
(Platform.isIOS && format != InputImageFormat.bgra8888)) {
print('image format is not supported: $format');
return null;
}

// since format is constraint to nv21 or bgra8888, both only have one plane
if (image.planes.length != 1) return null;
final plane = image.planes.first;
// Compile a flat list of all image data. For image formats with multiple planes,
// takes some copying.
final Uint8List bytes = image.planes.length == 1
? image.planes.first.bytes
: _concatenatePlanes(image);

// compose InputImage using bytes
return InputImage.fromBytes(
bytes: plane.bytes,
bytes: bytes,
metadata: InputImageMetadata(
size: Size(image.width.toDouble(), image.height.toDouble()),
rotation: rotation, // used only in Android
format: format, // used only in iOS
bytesPerRow: plane.bytesPerRow, // used only in iOS
format: format,
bytesPerRow: image.planes.first.bytesPerRow, // used only in iOS
),
);
}

Uint8List _concatenatePlanes(CameraImage image) {
int length = 0;
for (final Plane p in image.planes) {
length += p.bytes.length;
}

final Uint8List bytes = Uint8List(length);
int offset = 0;
for (final Plane p in image.planes) {
bytes.setRange(offset, offset + p.bytes.length, p.bytes);
offset += p.bytes.length;
}
return bytes;
}
}
20 changes: 15 additions & 5 deletions ...code_scanning/android/src/main/java/com/google_mlkit_barcode_scanning/BarcodeScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.flutter.plugin.common.MethodChannel;

public class BarcodeScanner implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startBarcodeScanner";
private static final String CLOSE = "vision#closeBarcodeScanner";

Expand Down Expand Up @@ -67,8 +68,11 @@ private com.google.mlkit.vision.barcode.BarcodeScanner initialize(MethodCall cal

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
Expand Down Expand Up @@ -191,7 +195,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
}
result.success(barcodeList);
})
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), null));
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), e))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());
}

private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points) {
Expand All @@ -205,7 +211,9 @@ private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points)

private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
Map<String, Integer> frame = new HashMap<>();
if (rect == null) return frame;
if (rect == null) {
return frame;
}
frame.put("left", rect.left);
frame.put("right", rect.right);
frame.put("top", rect.top);
Expand All @@ -216,7 +224,9 @@ private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
if (barcodeScanner == null) return;
if (barcodeScanner == null) {
return;
}
barcodeScanner.close();
instances.remove(id);
}
Expand Down
50 changes: 46 additions & 4 deletions ...gle_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,30 @@

import android.content.Context;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.Image;
import android.media.ImageWriter;
import android.net.Uri;
import android.util.Log;
import android.view.Surface;

import com.google.mlkit.vision.common.InputImage;

import java.io.File;
import java.io.IOException;
import java.lang.AutoCloseable;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Objects;

import io.flutter.plugin.common.MethodChannel;

public class InputImageConverter {
public class InputImageConverter implements AutoCloseable {

ImageWriter writer;

//Returns an [InputImage] from the image data received
public static InputImage getInputImageFromData(Map<String, Object> imageData,
public InputImage getInputImageFromData(Map<String, Object> imageData,
Context context,
MethodChannel.Result result) {
//Differentiates whether the image data is a path for a image file, contains image data in form of bytes, or a bitmap
Expand Down Expand Up @@ -116,9 +124,36 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
rotationDegrees,
imageFormat);
}
if (imageFormat == ImageFormat.YUV_420_888) {
// This image format is only supported in InputImage.fromMediaImage, which requires to transform the data to the right java type.
// TODO: Consider reusing the same Surface across multiple calls to save on allocations.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reusing this ImageWriter across multiple calls, thus avoiding the allocation, would make this likely more performant. Could any of the reviewers take a look?

Is it something that could be left as a TODO and fixed later?

writer = new ImageWriter.Builder(new Surface(new SurfaceTexture(true)))
.setWidthAndHeight(width, height)
.setImageFormat(imageFormat)
.build();
Image image = writer.dequeueInputImage();
if (image == null) {
result.error("InputImageConverterError", "failed to allocate space for input image", null);
return null;
}
// Deconstruct individual planes again from flattened array.
Image.Plane[] planes = image.getPlanes();
// Y plane
ByteBuffer yBuffer = planes[0].getBuffer();
yBuffer.put(data, 0, width * height);

// U plane
ByteBuffer uBuffer = planes[1].getBuffer();
int uOffset = width * height;
uBuffer.put(data, uOffset, (width * height) / 4);

// V plane
ByteBuffer vBuffer = planes[2].getBuffer();
int vOffset = uOffset + (width * height) / 4;
vBuffer.put(data, vOffset, (width * height) / 4);
return InputImage.fromMediaImage(image, rotationDegrees);
}
result.error("InputImageConverterError", "ImageFormat is not supported.", null);
// TODO: Use InputImage.fromMediaImage, which supports more types, e.g. IMAGE_FORMAT_YUV_420_888.
// See https://developers.google.com/android/reference/com/google/mlkit/vision/common/InputImage#fromMediaImage(android.media.Image,%20int)
return null;
} catch (Exception e) {
Log.e("ImageError", "Getting Image failed");
Expand All @@ -133,4 +168,11 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
}
}

@Override
public void close() {
if (writer != null) {
writer.close();
}
}

}
19 changes: 13 additions & 6 deletions ...it_face_detection/android/src/main/java/com/google_mlkit_face_detection/FaceDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.flutter.plugin.common.MethodChannel;

class FaceDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startFaceDetector";
private static final String CLOSE = "vision#closeFaceDetector";

Expand Down Expand Up @@ -52,9 +53,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null)
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
Expand Down Expand Up @@ -115,7 +118,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
result.success(faces);
})
.addOnFailureListener(
e -> result.error("FaceDetectorError", e.toString(), null));
e -> result.error("FaceDetectorError", e.toString(), null))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());

}

private FaceDetectorOptions parseOptions(Map<String, Object> options) {
Expand Down Expand Up @@ -206,7 +212,7 @@ private Map<String, List<double[]>> getContourData(Face face) {
private double[] landmarkPosition(Face face, int landmarkInt) {
FaceLandmark landmark = face.getLandmark(landmarkInt);
if (landmark != null) {
return new double[] { landmark.getPosition().x, landmark.getPosition().y };
return new double[]{landmark.getPosition().x, landmark.getPosition().y};
}
return null;
}
Expand All @@ -217,7 +223,7 @@ private List<double[]> contourPosition(Face face, int contourInt) {
List<PointF> contourPoints = contour.getPoints();
List<double[]> result = new ArrayList<>();
for (int i = 0; i < contourPoints.size(); i++) {
result.add(new double[] { contourPoints.get(i).x, contourPoints.get(i).y });
result.add(new double[]{contourPoints.get(i).x, contourPoints.get(i).y});
}
return result;
}
Expand All @@ -227,8 +233,9 @@ private List<double[]> contourPosition(Face face, int contourInt) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
if (detector == null)
if (detector == null) {
return;
}
detector.close();
instances.remove(id);
}
Expand Down
41 changes: 25 additions & 16 deletions ...etection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.flutter.plugin.common.MethodChannel;

class FaceMeshDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startFaceMeshDetector";
private static final String CLOSE = "vision#closeFaceMeshDetector";

Expand Down Expand Up @@ -51,8 +52,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
Expand Down Expand Up @@ -104,18 +108,18 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
meshData.put("triangles", triangles);

int[] types = {
FaceMesh.FACE_OVAL,
FaceMesh.LEFT_EYEBROW_TOP,
FaceMesh.LEFT_EYEBROW_BOTTOM,
FaceMesh.RIGHT_EYEBROW_TOP,
FaceMesh.RIGHT_EYEBROW_BOTTOM,
FaceMesh.LEFT_EYE,
FaceMesh.RIGHT_EYE,
FaceMesh.UPPER_LIP_TOP,
FaceMesh.UPPER_LIP_BOTTOM,
FaceMesh.LOWER_LIP_TOP,
FaceMesh.LOWER_LIP_BOTTOM,
FaceMesh.NOSE_BRIDGE
FaceMesh.FACE_OVAL,
FaceMesh.LEFT_EYEBROW_TOP,
FaceMesh.LEFT_EYEBROW_BOTTOM,
FaceMesh.RIGHT_EYEBROW_TOP,
FaceMesh.RIGHT_EYEBROW_BOTTOM,
FaceMesh.LEFT_EYE,
FaceMesh.RIGHT_EYE,
FaceMesh.UPPER_LIP_TOP,
FaceMesh.UPPER_LIP_BOTTOM,
FaceMesh.LOWER_LIP_TOP,
FaceMesh.LOWER_LIP_BOTTOM,
FaceMesh.NOSE_BRIDGE
};
Map<Integer, List<Map<String, Object>>> contours = new HashMap<>();
for (int type : types) {
Expand All @@ -129,7 +133,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
result.success(faceMeshes);
})
.addOnFailureListener(
e -> result.error("FaceMeshDetectorError", e.toString(), null));
e -> result.error("FaceMeshDetectorError", e.toString(), null))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());

}

private List<Map<String, Object>> pointsToList(List<FaceMeshPoint> points) {
Expand All @@ -152,7 +159,9 @@ private Map<String, Object> pointToMap(FaceMeshPoint point) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
if (detector == null) return;
if (detector == null) {
return;
}
detector.close();
instances.remove(id);
}
Expand Down
16 changes: 12 additions & 4 deletions ...ge_labeling/android/src/main/java/com/google_mlkit_image_labeling/ImageLabelDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import io.flutter.plugin.common.MethodChannel;

public class ImageLabelDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startImageLabelDetector";
private static final String CLOSE = "vision#closeImageLabelDetector";
private static final String MANAGE = "vision#manageFirebaseModels";
Expand Down Expand Up @@ -59,8 +60,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
ImageLabeler imageLabeler = instances.get(id);
Expand Down Expand Up @@ -106,7 +110,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)

result.success(labels);
})
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), null));
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), e))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());
}

//Labeler options that are provided to default image labeler(uses inbuilt model).
Expand Down Expand Up @@ -152,7 +158,9 @@ private CustomImageLabelerOptions getRemoteOptions(Map<String, Object> labelerOp
private void closeDetector(MethodCall call) {
String id = call.argument("id");
ImageLabeler imageLabeler = instances.get(id);
if (imageLabeler == null) return;
if (imageLabeler == null) {
return;
}
imageLabeler.close();
instances.remove(id);
}
Expand Down
Loading