Polish Pose demo and update SDK versions.

PiperOrigin-RevId: 353298692
Change-Id: I592e465d43ba1a9608e8f3f1b0590fa0550e280c

android/vision-quickstart/app/build.gradle

@@ -57,39 +57,39 @@ dependencies {
     implementation 'androidx.multidex:multidex:2.0.1'
 
     // Barcode model
-    implementation 'com.google.mlkit:barcode-scanning:16.1.0'
+    implementation 'com.google.mlkit:barcode-scanning:16.1.1'
     // Or comment the dependency above and uncomment the dependency below to
     // use unbundled model that depends on Google Play Services
-    // implementation 'com.google.android.gms:play-services-mlkit-barcode-scanning:16.1.3'
+    // implementation 'com.google.android.gms:play-services-mlkit-barcode-scanning:16.1.4'
 
     // Object detection feature with bundled default classifier
-    implementation 'com.google.mlkit:object-detection:16.2.2'
+    implementation 'com.google.mlkit:object-detection:16.2.3'
 
     // Object detection feature with custom classifier support
-    implementation 'com.google.mlkit:object-detection-custom:16.3.0'
+    implementation 'com.google.mlkit:object-detection-custom:16.3.1'
 
     // Face features
-    implementation 'com.google.mlkit:face-detection:16.0.3'
+    implementation 'com.google.mlkit:face-detection:16.0.4'
     // Or comment the dependency above and uncomment the dependency below to
     // use unbundled model that depends on Google Play Services
-    // implementation 'com.google.android.gms:play-services-mlkit-face-detection:16.1.2'
+    // implementation 'com.google.android.gms:play-services-mlkit-face-detection:16.1.3'
 
     // Text features
-    implementation 'com.google.android.gms:play-services-mlkit-text-recognition:16.1.2'
+    implementation 'com.google.android.gms:play-services-mlkit-text-recognition:16.1.3'
 
     // Image labeling
-    implementation 'com.google.mlkit:image-labeling:17.0.1'
+    implementation 'com.google.mlkit:image-labeling:17.0.2'
     // Or comment the dependency above and uncomment the dependency below to
     // use unbundled model that depends on Google Play Services
-    // implementation 'com.google.android.gms:play-services-mlkit-image-labeling:16.0.1'
+    // implementation 'com.google.android.gms:play-services-mlkit-image-labeling:16.0.2'
 
     // Image labeling custom
-    implementation 'com.google.mlkit:image-labeling-custom:16.3.0'
+    implementation 'com.google.mlkit:image-labeling-custom:16.3.1'
 
     // Pose detection with default models
-    implementation 'com.google.mlkit:pose-detection:17.0.1-beta1'
+    implementation 'com.google.mlkit:pose-detection:17.0.1-beta2'
     // Pose detection with accurate models
-    implementation 'com.google.mlkit:pose-detection-accurate:17.0.1-beta1'
+    implementation 'com.google.mlkit:pose-detection-accurate:17.0.1-beta2'
 
     // -------------------------------------------------------
 

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/InferenceInfoGraphic.java

@@ -57,7 +57,7 @@ public synchronized void draw(Canvas canvas) {
     float y = TEXT_SIZE * 1.5f;
 
     canvas.drawText(
-        "InputImage size: " + overlay.getImageWidth() + "x" + overlay.getImageHeight(),
+        "InputImage size: " + overlay.getImageHeight() + "x" + overlay.getImageWidth(),
         x,
         y,
         textPaint);

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/java/CameraXLivePreviewActivity.java

@@ -377,9 +377,11 @@ private void bindAnalysisUseCase() {
               PreferenceUtils.getPoseDetectorOptionsForLivePreview(this);
           boolean shouldShowInFrameLikelihood =
               PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodLivePreview(this);
+          boolean visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this);
+          boolean rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this);
           imageProcessor =
               new PoseDetectorProcessor(
-                  this, poseDetectorOptions, shouldShowInFrameLikelihood);
+                  this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ);
           break;
         default:
           throw new IllegalStateException("Invalid model name");

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/java/LivePreviewActivity.java

@@ -259,9 +259,10 @@ private void createCameraSource(String model) {
           Log.i(TAG, "Using Pose Detector with options " + poseDetectorOptions);
           boolean shouldShowInFrameLikelihood =
               PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodLivePreview(this);
-          cameraSource.setMachineLearningFrameProcessor(
-              new PoseDetectorProcessor(
-                  this, poseDetectorOptions, shouldShowInFrameLikelihood));
+          boolean visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this);
+          boolean rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this);
+          cameraSource.setMachineLearningFrameProcessor(new PoseDetectorProcessor(
+              this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ));
           break;
         default:
           Log.e(TAG, "Unknown model: " + model);

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/java/StillImageActivity.java

@@ -423,9 +423,11 @@ private void createImageProcessor() {
           Log.i(TAG, "Using Pose Detector with options " + poseDetectorOptions);
           boolean shouldShowInFrameLikelihood =
               PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodStillImage(this);
+          boolean visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this);
+          boolean rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this);
           imageProcessor =
-             new PoseDetectorProcessor(
-                this, poseDetectorOptions, shouldShowInFrameLikelihood);
+              new PoseDetectorProcessor(
+                  this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ);
           break;
         default:
           Log.e(TAG, "Unknown selectedMode: " + selectedMode);

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/java/posedetector/PoseDetectorProcessor.java

@@ -36,13 +36,19 @@ public class PoseDetectorProcessor extends VisionProcessorBase<Pose> {
   private final PoseDetector detector;
 
   private final boolean showInFrameLikelihood;
+  private final boolean visualizeZ;
+  private final boolean rescaleZForVisualization;
 
   public PoseDetectorProcessor(
       Context context,
       PoseDetectorOptionsBase options,
-      boolean showInFrameLikelihood) {
+      boolean showInFrameLikelihood,
+      boolean visualizeZ,
+      boolean rescaleZForVisualization) {
     super(context);
     this.showInFrameLikelihood = showInFrameLikelihood;
+    this.visualizeZ = visualizeZ;
+    this.rescaleZForVisualization = rescaleZForVisualization;
     detector = PoseDetection.getClient(options);
   }
 
@@ -60,7 +66,8 @@ protected Task<Pose> detectInImage(InputImage image) {
   @Override
   protected void onSuccess(@NonNull Pose pose, @NonNull GraphicOverlay graphicOverlay) {
     graphicOverlay.add(
-        new PoseGraphic(graphicOverlay, pose, showInFrameLikelihood));
+        new PoseGraphic(
+            graphicOverlay, pose, showInFrameLikelihood, visualizeZ, rescaleZForVisualization));
   }
 
   @Override

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/java/posedetector/PoseGraphic.java

@@ -16,10 +16,15 @@
 
 package com.google.mlkit.vision.demo.java.posedetector;
 
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
 import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.Paint;
 import android.graphics.PointF;
+import com.google.common.primitives.Ints;
+import com.google.mlkit.vision.common.PointF3D;
 import com.google.mlkit.vision.demo.GraphicOverlay;
 import com.google.mlkit.vision.demo.GraphicOverlay.Graphic;
 import com.google.mlkit.vision.pose.Pose;
@@ -32,9 +37,14 @@ public class PoseGraphic extends Graphic {
 
   private static final float DOT_RADIUS = 8.0f;
   private static final float IN_FRAME_LIKELIHOOD_TEXT_SIZE = 30.0f;
+  private static final float STROKE_WIDTH = 10.0f;
 
   private final Pose pose;
   private final boolean showInFrameLikelihood;
+  private final boolean visualizeZ;
+  private final boolean rescaleZForVisualization;
+  private float zMin = Float.MAX_VALUE;
+  private float zMax = Float.MIN_VALUE;
 
   private final Paint leftPaint;
   private final Paint rightPaint;
@@ -43,18 +53,24 @@ public class PoseGraphic extends Graphic {
   PoseGraphic(
       GraphicOverlay overlay,
       Pose pose,
-      boolean showInFrameLikelihood) {
+      boolean showInFrameLikelihood,
+      boolean visualizeZ,
+      boolean rescaleZForVisualization) {
     super(overlay);
-
     this.pose = pose;
     this.showInFrameLikelihood = showInFrameLikelihood;
+    this.visualizeZ = visualizeZ;
+    this.rescaleZForVisualization = rescaleZForVisualization;
 
     whitePaint = new Paint();
+    whitePaint.setStrokeWidth(STROKE_WIDTH);
     whitePaint.setColor(Color.WHITE);
     whitePaint.setTextSize(IN_FRAME_LIKELIHOOD_TEXT_SIZE);
     leftPaint = new Paint();
+    leftPaint.setStrokeWidth(STROKE_WIDTH);
     leftPaint.setColor(Color.GREEN);
     rightPaint = new Paint();
+    rightPaint.setStrokeWidth(STROKE_WIDTH);
     rightPaint.setColor(Color.YELLOW);
   }
 
@@ -64,17 +80,16 @@ public void draw(Canvas canvas) {
     if (landmarks.isEmpty()) {
       return;
     }
+
     // Draw all the points
     for (PoseLandmark landmark : landmarks) {
       drawPoint(canvas, landmark, whitePaint);
-      if (showInFrameLikelihood) {
-        canvas.drawText(
-            String.format(Locale.US, "%.2f", landmark.getInFrameLikelihood()),
-            translateX(landmark.getPosition().x),
-            translateY(landmark.getPosition().y),
-            whitePaint);
+      if (visualizeZ && rescaleZForVisualization) {
+        zMin = min(zMin, landmark.getPosition3D().getZ());
+        zMax = max(zMax, landmark.getPosition3D().getZ());
       }
     }
+
     PoseLandmark leftShoulder = pose.getPoseLandmark(PoseLandmark.LEFT_SHOULDER);
     PoseLandmark rightShoulder = pose.getPoseLandmark(PoseLandmark.RIGHT_SHOULDER);
     PoseLandmark leftElbow = pose.getPoseLandmark(PoseLandmark.LEFT_ELBOW);
@@ -111,6 +126,7 @@ public void draw(Canvas canvas) {
     drawLine(canvas, leftWrist, leftThumb, leftPaint);
     drawLine(canvas, leftWrist, leftPinky, leftPaint);
     drawLine(canvas, leftWrist, leftIndex, leftPaint);
+    drawLine(canvas, leftIndex, leftPinky, leftPaint);
     drawLine(canvas, leftAnkle, leftHeel, leftPaint);
     drawLine(canvas, leftHeel, leftFootIndex, leftPaint);
 
@@ -123,8 +139,20 @@ public void draw(Canvas canvas) {
     drawLine(canvas, rightWrist, rightThumb, rightPaint);
     drawLine(canvas, rightWrist, rightPinky, rightPaint);
     drawLine(canvas, rightWrist, rightIndex, rightPaint);
+    drawLine(canvas, rightIndex, rightPinky, rightPaint);
     drawLine(canvas, rightAnkle, rightHeel, rightPaint);
     drawLine(canvas, rightHeel, rightFootIndex, rightPaint);
+
+    // Draw inFrameLikelihood for all points
+    if (showInFrameLikelihood) {
+      for (PoseLandmark landmark : landmarks) {
+        canvas.drawText(
+            String.format(Locale.US, "%.2f", landmark.getInFrameLikelihood()),
+            translateX(landmark.getPosition().x),
+            translateY(landmark.getPosition().y),
+            whitePaint);
+      }
+    }
   }
 
   void drawPoint(Canvas canvas, PoseLandmark landmark, Paint paint) {
@@ -133,9 +161,58 @@ void drawPoint(Canvas canvas, PoseLandmark landmark, Paint paint) {
   }
 
   void drawLine(Canvas canvas, PoseLandmark startLandmark, PoseLandmark endLandmark, Paint paint) {
-    PointF start = startLandmark.getPosition();
-    PointF end = endLandmark.getPosition();
-    canvas.drawLine(
-        translateX(start.x), translateY(start.y), translateX(end.x), translateY(end.y), paint);
+    // When visualizeZ is true, sets up the paint to draw body line in different colors based on
+    // their z values.
+    if (visualizeZ) {
+      PointF3D start = startLandmark.getPosition3D();
+      PointF3D end = endLandmark.getPosition3D();
+
+      // Gets the range of z value.
+      float zLowerBoundInScreenPixel;
+      float zUpperBoundInScreenPixel;
+
+      if (rescaleZForVisualization) {
+        zLowerBoundInScreenPixel = min(-0.001f, scale(zMin));
+        zUpperBoundInScreenPixel = max(0.001f, scale(zMax));
+      } else {
+        // By default, assume the range of z value in screen pixel is [-canvasWidth, canvasWidth].
+        float defaultRangeFactor = 1f;
+        zLowerBoundInScreenPixel = -defaultRangeFactor * canvas.getWidth();
+        zUpperBoundInScreenPixel = defaultRangeFactor * canvas.getWidth();
+      }
+
+      // Gets average z for the current body line
+      float avgZInImagePixel = (start.getZ() + end.getZ()) / 2;
+      float zInScreenPixel = scale(avgZInImagePixel);
+
+      if (zInScreenPixel < 0) {
+        // Sets up the paint to draw the body line in red if it is in front of the z origin.
+        // Maps values within [zLowerBoundInScreenPixel, 0) to [255, 0) and use it to control the
+        // color. The larger the value is, the more red it will be.
+        int v = (int) (zInScreenPixel / zLowerBoundInScreenPixel * 255);
+        v = Ints.constrainToRange(v, 0, 255);
+        paint.setARGB(255, 255, 255 - v, 255 - v);
+      } else {
+        // Sets up the paint to draw the body line in blue if it is behind the z origin.
+        // Maps values within [0, zUpperBoundInScreenPixel] to [0, 255] and use it to control the
+        // color. The larger the value is, the more blue it will be.
+        int v = (int) (zInScreenPixel / zUpperBoundInScreenPixel * 255);
+        v = Ints.constrainToRange(v, 0, 255);
+        paint.setARGB(255, 255 - v, 255 - v, 255);
+      }
+
+      canvas.drawLine(
+          translateX(start.getX()),
+          translateY(start.getY()),
+          translateX(end.getX()),
+          translateY(end.getY()),
+          paint);
+
+    } else {
+      PointF start = startLandmark.getPosition();
+      PointF end = endLandmark.getPosition();
+      canvas.drawLine(
+          translateX(start.x), translateY(start.y), translateX(end.x), translateY(end.y), paint);
+    }
   }
 }

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/kotlin/CameraXLivePreviewActivity.kt

@@ -392,7 +392,11 @@ class CameraXLivePreviewActivity :
             PreferenceUtils.getPoseDetectorOptionsForLivePreview(this)
           val shouldShowInFrameLikelihood =
             PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodLivePreview(this)
-          PoseDetectorProcessor(this, poseDetectorOptions, shouldShowInFrameLikelihood)
+          val visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this)
+          val rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this)
+          PoseDetectorProcessor(
+            this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ
+          )
         }
         else -> throw IllegalStateException("Invalid model name")
       }

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/kotlin/LivePreviewActivity.kt

@@ -267,9 +267,11 @@ class LivePreviewActivity :
           Log.i(TAG, "Using Pose Detector with options $poseDetectorOptions")
           val shouldShowInFrameLikelihood =
             PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodLivePreview(this)
+          val visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this)
+          val rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this)
           cameraSource!!.setMachineLearningFrameProcessor(
             PoseDetectorProcessor(
-              this, poseDetectorOptions, shouldShowInFrameLikelihood)
+              this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ)
           )
         }
         else -> Log.e(TAG, "Unknown model: $model")

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/kotlin/StillImageActivity.kt

@@ -463,9 +463,11 @@ class StillImageActivity : AppCompatActivity() {
           Log.i(TAG, "Using Pose Detector with options $poseDetectorOptions")
           val shouldShowInFrameLikelihood =
             PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodStillImage(this)
+          val visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this)
+          val rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this)
           imageProcessor =
             PoseDetectorProcessor(
-              this, poseDetectorOptions, shouldShowInFrameLikelihood)
+              this, poseDetectorOptions, shouldShowInFrameLikelihood, visualizeZ, rescaleZ)
         }
         else -> Log.e(
           TAG,

android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/kotlin/posedetector/PoseDetectorProcessor.kt

@@ -31,7 +31,9 @@ import com.google.mlkit.vision.pose.PoseDetectorOptionsBase
 class PoseDetectorProcessor(
   context: Context,
   options: PoseDetectorOptionsBase,
-  private val showInFrameLikelihood: Boolean
+  private val showInFrameLikelihood: Boolean,
+  private val visualizeZ: Boolean,
+  private val rescaleZForVisualization: Boolean
 ) :
   VisionProcessorBase<Pose>(context) {
   private val detector: PoseDetector
@@ -49,7 +51,8 @@ class PoseDetectorProcessor(
     graphicOverlay: GraphicOverlay
   ) {
     graphicOverlay.add(
-      PoseGraphic(graphicOverlay, pose, showInFrameLikelihood))
+      PoseGraphic(
+        graphicOverlay, pose, showInFrameLikelihood, visualizeZ, rescaleZForVisualization))
   }
 
   override fun onFailure(e: Exception) {