Skip to content

Commit

Permalink
Push latest Vision Quickstart app changes to internal devrel.
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 333823900
Change-Id: If2b1e270ce5517c5a146f1ada9c3e57080acf2f1
  • Loading branch information
Google ML Kit authored and Daniel Furlong committed Sep 28, 2020
1 parent 7a1f27c commit 08c3e1f
Show file tree
Hide file tree
Showing 4 changed files with 190 additions and 113 deletions.
84 changes: 45 additions & 39 deletions ios/quickstarts/vision/VisionExample/CameraViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,37 +59,13 @@ class CameraViewController: UIViewController {
return annotationOverlayView
}()

/// Serial queue used for synchronizing access to `_poseDetector`. This is needed because Swift
/// lacks ObjC-style synchronization and the detector is accessed on different threads across
/// initialization, usage, and deallocation. Note that just using the main queue for
/// synchronization from the getter/setter overrides is unsafe because it could allow a deadlock
/// if the `poseDetector` property were accessed on the main thread.
private let poseDetectorQueue = DispatchQueue(label: "com.google.mlkit.pose")

/// The detector used for detecting poses. The pose detector's lifecycle is managed manually, so
/// it is initialized on-demand via the getter override and set to `nil` when a new detector is
/// chosen.
private var _poseDetector: PoseDetector? = nil
private var poseDetector: PoseDetector? {
get {
var detector: PoseDetector? = nil
poseDetectorQueue.sync {
if _poseDetector == nil {
let options = currentDetector == .pose ? PoseDetectorOptions()
: AccuratePoseDetectorOptions()
options.detectorMode = .stream
_poseDetector = PoseDetector.poseDetector(options: options)
}
detector = _poseDetector
}
return detector
}
set(newDetector) {
poseDetectorQueue.sync {
_poseDetector = newDetector
}
}
}
/// Initialized when one of the pose detector rows are chosen. Reset to `nil` when neither are.
private var poseDetector: PoseDetector? = nil

/// The detector mode with which detection was most recently run. Only used on the video output
/// queue. Useful for inferring when to reset detector instances which use a conventional
/// lifecyle paradigm.
private var lastDetector: Detector?

// MARK: - IBOutlets

Expand Down Expand Up @@ -504,12 +480,8 @@ class CameraViewController: UIViewController {
guard let detector = Detector(rawValue: value) else { return }
self.currentDetector = detector
self.removeDetectionAnnotations()

// Reset the pose detector to `nil` when a new detector row is chosen. The detector will be
// re-initialized via its getter when it is needed for detection again.
self.poseDetector = nil
}
if detectorType.rawValue == currentDetector.rawValue { action.isEnabled = false }
if detectorType.rawValue == self.currentDetector.rawValue { action.isEnabled = false }
alertController.addAction(action)
}
alertController.addAction(UIAlertAction(title: Constant.cancelActionTitleText, style: .cancel))
Expand Down Expand Up @@ -725,6 +697,35 @@ class CameraViewController: UIViewController {
}
}
}

/// Resets any detector instances which use a conventional lifecycle paradigm. This method is
/// expected to be invoked on the AVCaptureOutput queue - the same queue on which detection is
/// run.
private func resetManagedLifecycleDetectors(activeDetector: Detector) {
if activeDetector == self.lastDetector {
// Same row as before, no need to reset any detectors.
return;
}
// Clear the old detector, if applicable.
switch (self.lastDetector) {
case .pose, .poseAccurate:
self.poseDetector = nil
break
default:
break
}
// Initialize the new detector, if applicable.
switch (activeDetector) {
case .pose, .poseAccurate:
let options = activeDetector == .pose ? PoseDetectorOptions() : AccuratePoseDetectorOptions()
options.detectorMode = .stream
self.poseDetector = PoseDetector.poseDetector(options: options)
break
default:
break
}
self.lastDetector = activeDetector
}
}

// MARK: AVCaptureVideoDataOutputSampleBufferDelegate
Expand All @@ -740,6 +741,11 @@ extension CameraViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
print("Failed to get image buffer from sample buffer.")
return
}
// Evaluate `self.currentDetector` once to ensure consistency throughout this method since it
// can be concurrently modified from the main thread.
let activeDetector = self.currentDetector;
resetManagedLifecycleDetectors(activeDetector: activeDetector)

lastFrame = sampleBuffer
let visionImage = VisionImage(buffer: sampleBuffer)
let orientation = UIUtilities.imageOrientation(
Expand All @@ -751,22 +757,22 @@ extension CameraViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
let imageHeight = CGFloat(CVPixelBufferGetHeight(imageBuffer))
var shouldEnableClassification = false
var shouldEnableMultipleObjects = false
switch currentDetector {
switch activeDetector {
case .onDeviceObjectProminentWithClassifier, .onDeviceObjectMultipleWithClassifier,
.onDeviceObjectCustomProminentWithClassifier, .onDeviceObjectCustomMultipleWithClassifier:
shouldEnableClassification = true
default:
break
}
switch currentDetector {
switch activeDetector {
case .onDeviceObjectMultipleNoClassifier, .onDeviceObjectMultipleWithClassifier,
.onDeviceObjectCustomMultipleNoClassifier, .onDeviceObjectCustomMultipleWithClassifier:
shouldEnableMultipleObjects = true
default:
break
}

switch currentDetector {
switch activeDetector {
case .onDeviceBarcode:
scanBarcodesOnDevice(in: visionImage, width: imageWidth, height: imageHeight)
case .onDeviceFace:
Expand Down
72 changes: 45 additions & 27 deletions ios/quickstarts/vision/VisionExample/ViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,12 @@ class ViewController: UIViewController, UINavigationControllerDelegate {
// Image counter.
var currentImage = 0

/// The detector used for detecting poses. The pose detector's lifecycle is managed manually, so
/// it is initialized on-demand via the getter override and set to nil when a new detector is
/// chosen.
private var _poseDetector: PoseDetector? = nil
private var poseDetector: PoseDetector? {
get {
if _poseDetector == nil {
let options = AccuratePoseDetectorOptions()
options.detectorMode = .singleImage
_poseDetector = PoseDetector.poseDetector(options: options)
}
return _poseDetector
}
set(newDetector) {
_poseDetector = newDetector
}
}
/// Initialized when one of the pose detector rows are chosen. Reset to `nil` when neither are.
private var poseDetector: PoseDetector? = nil

/// The detector row with which detection was most recently run. Useful for inferring when to
/// reset detector instances which use a conventional lifecyle paradigm.
private var lastDetectorRow: DetectorPickerRow?

// MARK: - IBOutlets

Expand Down Expand Up @@ -121,6 +110,8 @@ class ViewController: UIViewController, UINavigationControllerDelegate {
clearResults()
let row = detectorPicker.selectedRow(inComponent: 0)
if let rowIndex = DetectorPickerRow(rawValue: row) {
resetManagedLifecycleDetectors(activeDetectorRow: rowIndex)

let shouldEnableClassification =
(rowIndex == .detectObjectsProminentWithClassifier)
|| (rowIndex == .detectObjectsMultipleWithClassifier)
Expand Down Expand Up @@ -166,7 +157,7 @@ class ViewController: UIViewController, UINavigationControllerDelegate {
options.shouldEnableMultipleObjects = shouldEnableMultipleObjects
options.detectorMode = .singleImage
detectObjectsOnDevice(in: imageView.image, options: options)
case .detectPoseAccurate:
case .detectPose, .detectPoseAccurate:
detectPose(image: imageView.image)
}
} else {
Expand Down Expand Up @@ -628,14 +619,6 @@ extension ViewController: UIPickerViewDataSource, UIPickerViewDelegate {

func pickerView(_ pickerView: UIPickerView, didSelectRow row: Int, inComponent component: Int) {
clearResults()

if let rowIndex = DetectorPickerRow(rawValue: row) {
if rowIndex != .detectPoseAccurate {
// Reset the pose detector to `nil` when a new detector row is chosen. The detector will be
// re-initialized via its getter when it is needed for detection again.
poseDetector = nil
}
}
}
}

Expand Down Expand Up @@ -992,6 +975,38 @@ extension ViewController {
}
// [END detect_object]
}

/// Resets any detector instances which use a conventional lifecycle paradigm. This method should
/// be invoked immediately prior to performing detection. This approach is advantageous to tearing
/// down old detectors in the `UIPickerViewDelegate` method because that method isn't actually
/// invoked in-sync with when the selected row changes and can result in tearing down the wrong
/// detector in the event of a race condition.
private func resetManagedLifecycleDetectors(activeDetectorRow: DetectorPickerRow) {
if activeDetectorRow == self.lastDetectorRow {
// Same row as before, no need to reset any detectors.
return;
}
// Clear the old detector, if applicable.
switch (self.lastDetectorRow) {
case .detectPose, .detectPoseAccurate:
self.poseDetector = nil
break
default:
break
}
// Initialize the new detector, if applicable.
switch (activeDetectorRow) {
case .detectPose, .detectPoseAccurate:
let options = activeDetectorRow == .detectPose ? PoseDetectorOptions()
: AccuratePoseDetectorOptions()
options.detectorMode = .singleImage
self.poseDetector = PoseDetector.poseDetector(options: options)
break
default:
break
}
self.lastDetectorRow = activeDetectorRow
}
}

// MARK: - Enums
Expand All @@ -1012,9 +1027,10 @@ private enum DetectorPickerRow: Int {
detectObjectsCustomProminentWithClassifier,
detectObjectsCustomMultipleNoClassifier,
detectObjectsCustomMultipleWithClassifier,
detectPose,
detectPoseAccurate

static let rowsCount = 14
static let rowsCount = 15
static let componentsCount = 1

public var description: String {
Expand Down Expand Up @@ -1045,6 +1061,8 @@ private enum DetectorPickerRow: Int {
return "ODT, custom, multiple, no labeling"
case .detectObjectsCustomMultipleWithClassifier:
return "ODT, custom, multiple, labeling"
case .detectPose:
return "Pose Detection"
case .detectPoseAccurate:
return "Pose Detection, accurate"
}
Expand Down
Loading

0 comments on commit 08c3e1f

Please sign in to comment.