Skip to content

Commit

Permalink
List of included changes:
Browse files Browse the repository at this point in the history 
  - Update segmentation mask demo to overlay a transparent colored mask.
  - Fix some compilation errors.
  - Made some minor code cleanup changes in the Vision Quickstart app.

PiperOrigin-RevId: 359199258
Change-Id: I1f77ac957e13b4ef7f4cef0cf4744fddfda26f12
  • Loading branch information
Google ML Kit authored and Dong Chen committed Feb 24, 2021
1 parent a69bf24 commit 5a5e92e
Show file tree
Hide file tree
Showing 16 changed files with 139 additions and 103 deletions.
2 changes: 1 addition & 1 deletion ios/quickstarts/vision/Podfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ pod 'GoogleMLKit/TextRecognition'
target 'VisionExample' do
end

target 'VisionExampleObjc' do
target 'VisionExampleObjC' do
end
24 changes: 12 additions & 12 deletions ios/quickstarts/vision/VisionExample.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
1036DA2B2087CC16003253C9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
1036DA392087CC26003253C9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
1077670D210FBBB6007A19A7 /* liberty.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = liberty.jpg; sourceTree = "<group>"; };
10E114E020E5CBAA0013E4A4 /* VisionExampleObjc.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VisionExampleObjc.app; sourceTree = BUILT_PRODUCTS_DIR; };
10E114E020E5CBAA0013E4A4 /* VisionExampleObjC.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VisionExampleObjC.app; sourceTree = BUILT_PRODUCTS_DIR; };
10E114E220E5CBAA0013E4A4 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = "<group>"; };
10E114E320E5CBAA0013E4A4 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = "<group>"; };
10E114E520E5CBAA0013E4A4 /* ViewController.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ViewController.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -138,7 +138,7 @@
name = Resources;
sourceTree = "<group>";
};
10E114E120E5CBAA0013E4A4 /* VisionExampleObjc */ = {
10E114E120E5CBAA0013E4A4 /* VisionExampleObjC */ = {
isa = PBXGroup;
children = (
10E114E220E5CBAA0013E4A4 /* AppDelegate.h */,
Expand All @@ -153,7 +153,7 @@
10E1151820EBF7E10013E4A4 /* UIImage+VisionDetection.h */,
10E1151920EBF7E10013E4A4 /* UIImage+VisionDetection.m */,
);
path = VisionExampleObjc;
path = VisionExampleObjC;
sourceTree = "<group>";
};
BB287B2C20729CE90069707A = {
Expand All @@ -162,7 +162,7 @@
1001E6FC20CF0C8F00445CB3 /* Assets.xcassets */,
10224B642087E52600A77316 /* Resources */,
BB287B3720729CE90069707A /* VisionExample */,
10E114E120E5CBAA0013E4A4 /* VisionExampleObjc */,
10E114E120E5CBAA0013E4A4 /* VisionExampleObjC */,
BB287B3620729CE90069707A /* Products */,
);
sourceTree = "<group>";
Expand All @@ -171,7 +171,7 @@
isa = PBXGroup;
children = (
BB287B3520729CE90069707A /* VisionExample.app */,
10E114E020E5CBAA0013E4A4 /* VisionExampleObjc.app */,
10E114E020E5CBAA0013E4A4 /* VisionExampleObjC.app */,
);
name = Products;
sourceTree = "<group>";
Expand Down Expand Up @@ -201,9 +201,9 @@
/* End PBXGroup section */

/* Begin PBXNativeTarget section */
10E114DF20E5CBAA0013E4A4 /* VisionExampleObjc */ = {
10E114DF20E5CBAA0013E4A4 /* VisionExampleObjC */ = {
isa = PBXNativeTarget;
buildConfigurationList = 10E1150F20E5CBAE0013E4A4 /* Build configuration list for PBXNativeTarget "VisionExampleObjc" */;
buildConfigurationList = 10E1150F20E5CBAE0013E4A4 /* Build configuration list for PBXNativeTarget "VisionExampleObjC" */;
buildPhases = (
10E114DC20E5CBAA0013E4A4 /* Sources */,
10E114DD20E5CBAA0013E4A4 /* Frameworks */,
Expand All @@ -213,9 +213,9 @@
);
dependencies = (
);
name = VisionExampleObjc;
productName = VisionExampleObjc;
productReference = 10E114E020E5CBAA0013E4A4 /* VisionExampleObjc.app */;
name = VisionExampleObjC;
productName = VisionExampleObjC;
productReference = 10E114E020E5CBAA0013E4A4 /* VisionExampleObjC.app */;
productType = "com.apple.product-type.application";
};
BB287B3420729CE90069707A /* VisionExample */ = {
Expand Down Expand Up @@ -275,7 +275,7 @@
projectRoot = "";
targets = (
BB287B3420729CE90069707A /* VisionExample */,
10E114DF20E5CBAA0013E4A4 /* VisionExampleObjc */,
10E114DF20E5CBAA0013E4A4 /* VisionExampleObjC */,
);
};
/* End PBXProject section */
Expand Down Expand Up @@ -568,7 +568,7 @@
/* End XCBuildConfiguration section */

/* Begin XCConfigurationList section */
10E1150F20E5CBAE0013E4A4 /* Build configuration list for PBXNativeTarget "VisionExampleObjc" */ = {
10E1150F20E5CBAE0013E4A4 /* Build configuration list for PBXNativeTarget "VisionExampleObjC" */ = {
isa = XCConfigurationList;
buildConfigurations = (
10E1150920E5CBAE0013E4A4 /* Debug */,
Expand Down
4 changes: 3 additions & 1 deletion ios/quickstarts/vision/VisionExample/CameraViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,8 @@ class CameraViewController: UIViewController {
}

UIUtilities.applySegmentationMask(
mask: mask, to: imageBuffer, backgroundColor: UIColor.blue,
mask: mask, to: imageBuffer,
backgroundColor: UIColor.purple.withAlphaComponent(Constant.segmentationMaskAlpha),
foregroundColor: nil)
strongSelf.updatePreviewOverlayViewWithImageBuffer(imageBuffer)
}
Expand Down Expand Up @@ -1052,4 +1053,5 @@ private enum Constant {
static let imageLabelResultFrameY = 0.1
static let imageLabelResultFrameWidth = 0.5
static let imageLabelResultFrameHeight = 0.8
static let segmentationMaskAlpha: CGFloat = 0.5
}
85 changes: 46 additions & 39 deletions ios/quickstarts/vision/VisionExample/UIUtilities.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,59 +160,66 @@ public class UIUtilities {
var redFG: CGFloat = 0.0
var greenFG: CGFloat = 0.0
var blueFG: CGFloat = 0.0
var alphaFG: CGFloat = 0.0
var redBG: CGFloat = 0.0
var greenBG: CGFloat = 0.0
var blueBG: CGFloat = 0.0
var alphaBG: CGFloat = 0.0

if let backgroundColor = backgroundColor {
backgroundColor.getRed(&redBG, green: &greenBG, blue: &blueBG, alpha: nil)
}
if let foregroundColor = foregroundColor {
foregroundColor.getRed(&redFG, green: &greenFG, blue: &blueFG, alpha: nil)
}

redFG *= Constants.maxColorComponentValue
greenFG *= Constants.maxColorComponentValue
blueFG *= Constants.maxColorComponentValue
redBG *= Constants.maxColorComponentValue
greenBG *= Constants.maxColorComponentValue
blueBG *= Constants.maxColorComponentValue
let backgroundColor = backgroundColor != nil ? backgroundColor : .clear
let foregroundColor = foregroundColor != nil ? foregroundColor : .clear
backgroundColor!.getRed(&redBG, green: &greenBG, blue: &blueBG, alpha: &alphaBG)
foregroundColor!.getRed(&redFG, green: &greenFG, blue: &blueFG, alpha: &alphaFG)

for _ in 0...(height - 1) {
for col in 0...(width - 1) {
let pixelOffset = col * Constants.bgraBytesPerPixel
let blueOffset = pixelOffset
let greenOffset = pixelOffset + 1
let redOffset = pixelOffset + 2
let alphaOffset = pixelOffset + 3

let maskValue: Float32 = maskAddress[col]
let backgroundRegionRatio: Float32 = 1.0 - maskValue
let maskValue: CGFloat = CGFloat(maskAddress[col])
let backgroundRegionRatio: CGFloat = 1.0 - maskValue
let foregroundRegionRatio = maskValue

let originalPixelRed: UInt8 = imageAddress[redOffset]
let originalPixelGreen: UInt8 = imageAddress[greenOffset]
let originalPixelBlue: UInt8 = imageAddress[blueOffset]

let redBGComponent: Float32 =
backgroundColor != nil ? Float32(redBG) : Float32(originalPixelRed)
let greenBGComponent: Float32 =
backgroundColor != nil ? Float32(greenBG) : Float32(originalPixelGreen)
let blueBGComponent: Float32 =
backgroundColor != nil ? Float32(blueBG) : Float32(originalPixelBlue)

let redFGComponent: Float32 =
foregroundColor != nil ? Float32(redFG) : Float32(originalPixelRed)
let greenFGComponent: Float32 =
foregroundColor != nil ? Float32(greenFG) : Float32(originalPixelGreen)
let blueFGComponent: Float32 =
foregroundColor != nil ? Float32(blueFG) : Float32(originalPixelBlue)

imageAddress[redOffset] = UInt8(
redBGComponent * backgroundRegionRatio + redFGComponent * foregroundRegionRatio)
imageAddress[greenOffset] = UInt8(
greenBGComponent * backgroundRegionRatio + greenFGComponent * foregroundRegionRatio)
imageAddress[blueOffset] = UInt8(
blueBGComponent * backgroundRegionRatio + blueFGComponent * foregroundRegionRatio)
let originalPixelRed: CGFloat =
CGFloat(imageAddress[redOffset]) / Constants.maxColorComponentValue
let originalPixelGreen: CGFloat =
CGFloat(imageAddress[greenOffset]) / Constants.maxColorComponentValue
let originalPixelBlue: CGFloat =
CGFloat(imageAddress[blueOffset]) / Constants.maxColorComponentValue
let originalPixelAlpha: CGFloat =
CGFloat(imageAddress[alphaOffset]) / Constants.maxColorComponentValue

let redOverlay = redBG * backgroundRegionRatio + redFG * foregroundRegionRatio
let greenOverlay = greenBG * backgroundRegionRatio + greenFG * foregroundRegionRatio
let blueOverlay = blueBG * backgroundRegionRatio + blueFG * foregroundRegionRatio
let alphaOverlay = alphaBG * backgroundRegionRatio + alphaFG * foregroundRegionRatio

// Calculate composite color component values.
// Derived from https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending
let compositeAlpha: CGFloat = ((1.0 - alphaOverlay) * originalPixelAlpha) + alphaOverlay
var compositeRed: CGFloat = 0.0
var compositeGreen: CGFloat = 0.0
var compositeBlue: CGFloat = 0.0
// Only perform rgb blending calculations if the output alpha is > 0. A zero-value alpha
// means none of the color channels actually matter, and would introduce division by 0.
if abs(compositeAlpha) > CGFloat(Float.ulpOfOne) {
compositeRed =
(((1.0 - alphaOverlay) * originalPixelAlpha * originalPixelRed)
+ (alphaOverlay * redOverlay)) / compositeAlpha
compositeGreen =
(((1.0 - alphaOverlay) * originalPixelAlpha * originalPixelGreen)
+ (alphaOverlay * greenOverlay)) / compositeAlpha
compositeBlue =
(((1.0 - alphaOverlay) * originalPixelAlpha * originalPixelBlue)
+ (alphaOverlay * blueOverlay)) / compositeAlpha
}

imageAddress[redOffset] = UInt8(compositeRed * Constants.maxColorComponentValue)
imageAddress[greenOffset] = UInt8(compositeGreen * Constants.maxColorComponentValue)
imageAddress[blueOffset] = UInt8(compositeBlue * Constants.maxColorComponentValue)
}

imageAddress += imageBytesPerRow / MemoryLayout<UInt8>.size
Expand Down
5 changes: 4 additions & 1 deletion ios/quickstarts/vision/VisionExample/ViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,9 @@ extension ViewController {
}

UIUtilities.applySegmentationMask(
mask: mask, to: imageBuffer, backgroundColor: UIColor.blue, foregroundColor: nil)
mask: mask, to: imageBuffer,
backgroundColor: UIColor.purple.withAlphaComponent(Constants.segmentationMaskAlpha),
foregroundColor: nil)
let maskedImage = UIUtilities.createUIImage(from: imageBuffer, orientation: .up)

let imageView = UIImageView()
Expand Down Expand Up @@ -1159,6 +1161,7 @@ private enum Constants {
static let lineColor = UIColor.yellow.cgColor
static let lineWidth: CGFloat = 3.0
static let fillColor = UIColor.clear.cgColor
static let segmentationMaskAlpha: CGFloat = 0.5
}

// Helper function inserted by Swift 4.2 migrator.
Expand Down
2 changes: 1 addition & 1 deletion ...ts/vision/VisionExampleObjc/AppDelegate.h → ...ts/vision/VisionExampleObjC/AppDelegate.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN

@interface AppDelegate : UIResponder <UIApplicationDelegate>

@property(nonatomic, strong) UIWindow *window;
@property(nonatomic) UIWindow *window;

@end

Expand Down
6 changes: 5 additions & 1 deletion ...ts/vision/VisionExampleObjc/AppDelegate.m → ...ts/vision/VisionExampleObjC/AppDelegate.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@

#import "AppDelegate.h"

NS_ASSUME_NONNULL_BEGIN

@interface AppDelegate ()
@end

@implementation AppDelegate

- (BOOL)application:(UIApplication *)application
didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
didFinishLaunchingWithOptions:(nullable NSDictionary *)launchOptions {
return YES;
}

@end

NS_ASSUME_NONNULL_END
0 .../VisionExampleObjc/CameraViewController.h → .../VisionExampleObjC/CameraViewController.h
View file
File renamed without changes.
5 changes: 4 additions & 1 deletion .../VisionExampleObjc/CameraViewController.m → .../VisionExampleObjC/CameraViewController.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
static const CGFloat MLKImageLabelResultFrameY = 0.1;
static const CGFloat MLKImageLabelResultFrameWidth = 0.5;
static const CGFloat MLKImageLabelResultFrameHeight = 0.8;
static const CGFloat MLKSegmentationMaskAlpha = 0.5;

@interface CameraViewController () <AVCaptureVideoDataOutputSampleBufferDelegate>

Expand Down Expand Up @@ -373,9 +374,11 @@ - (void)detectSegmentationMaskInImage:(MLKVisionImage *)image
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);

if (mask != nil) {
UIColor *backgroundColor =
[UIColor.purpleColor colorWithAlphaComponent:MLKSegmentationMaskAlpha];
[UIUtilities applySegmentationMask:mask
toImageBuffer:imageBuffer
withBackgroundColor:UIColor.blueColor
withBackgroundColor:backgroundColor
foregroundColor:nil];
} else {
NSLog(@"Failed to segment image with error: %@", error.localizedDescription);
Expand Down
2 changes: 1 addition & 1 deletion ...sionExampleObjc/UIImage+VisionDetection.h → ...sionExampleObjC/UIImage+VisionDetection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

NS_ASSUME_NONNULL_BEGIN

/// A `UIImage` category used for vision detection.
/** A `UIImage` category used for vision detection. */
@interface UIImage (VisionDetection)

- (UIImage *)scaledImageWithSize:(CGSize)size;
Expand Down
10 changes: 6 additions & 4 deletions ...sionExampleObjc/UIImage+VisionDetection.m → ...sionExampleObjC/UIImage+VisionDetection.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@

@implementation UIImage (VisionDetection)

/// Returns a scaled image to the given size.
///
/// - Parameter size: Maximum size of the returned image.
/// - Return: Image scaled according to the give size or `nil` if image resize fails.
/**
* Returns a scaled image to the given size.
*
* @param size Maximum size of the returned image.
* @return Image scaled according to the give size or `nil` if image resize fails.
*/
- (UIImage *)scaledImageWithSize:(CGSize)size {
UIGraphicsBeginImageContextWithOptions(size, NO, self.scale);
[self drawInRect:CGRectMake(0, 0, size.width, size.height)];
Expand Down
0 ...ts/vision/VisionExampleObjc/UIUtilities.h → ...ts/vision/VisionExampleObjC/UIUtilities.h
View file
File renamed without changes.
Loading

0 comments on commit 5a5e92e

Please sign in to comment.