Android Translate Showcase: Update to use CameraX Beta and PreviewView

android/translate-showcase/app/build.gradle

@@ -61,9 +61,11 @@ dependencies {
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
 
     // Add CameraX dependencies
-    def camerax_version = "1.0.0-alpha05"
+    def camerax_version = "1.0.0-beta05"
     implementation "androidx.camera:camera-core:${camerax_version}"
     implementation "androidx.camera:camera-camera2:${camerax_version}"
+    implementation "androidx.camera:camera-lifecycle:$camerax_version"
+    implementation "androidx.camera:camera-view:1.0.0-alpha12"
 
     // Add ML Kit dependencies
     implementation 'com.google.android.gms:play-services-mlkit-text-recognition:16.0.0'

android/translate-showcase/app/src/main/java/com/google/mlkit/showcase/translate/analyzer/TextAnalyzer.kt

@@ -49,17 +49,20 @@ class TextAnalyzer(
     // Flag to skip analyzing new available frames until previous analysis has finished.
     private var isBusy = false
 
-    override fun analyze(imageProxy: ImageProxy, imageRotationDegrees: Int) {
+    @androidx.camera.core.ExperimentalGetImage
+    override fun analyze(imageProxy: ImageProxy) {
         val mediaImage = imageProxy.image ?: return
         if (isBusy) return
 
         isBusy = true
         val convertImageToBitmap = ImageUtils.convertYuv420888ImageToBitmap(mediaImage)
         val cropRect = Rect(0, 0, mediaImage.width, mediaImage.height)
 
+        val rotationDegrees = imageProxy.imageInfo.rotationDegrees
+
         // If the image is rotated by 90 (or 270) degrees, swap height and width when calculating
         // the crop.
-        val (widthCrop, heightCrop) = when(imageRotationDegrees) {
+        val (widthCrop, heightCrop) = when(rotationDegrees) {
             90, 270 -> Pair(heightCropPercent / 100f, widthCropPercent / 100f)
             else -> Pair(widthCropPercent / 100f, heightCropPercent / 100f)
         }
@@ -68,9 +71,10 @@ class TextAnalyzer(
             (mediaImage.height * heightCrop / 2).toInt()
         )
         val croppedBitmap =
-            ImageUtils.rotateAndCrop(convertImageToBitmap, imageRotationDegrees, cropRect);
+            ImageUtils.rotateAndCrop(convertImageToBitmap, rotationDegrees, cropRect);
         recognizeTextOnDevice(InputImage.fromBitmap(croppedBitmap, 0)).addOnCompleteListener {
             isBusy = false
+            imageProxy.close()
         }
     }
 

android/translate-showcase/app/src/main/java/com/google/mlkit/showcase/translate/main/MainFragment.kt

@@ -18,43 +18,72 @@
 package com.google.mlkit.showcase.translate.main
 
 import android.Manifest
+import android.content.Context
 import android.content.pm.PackageManager
 import android.graphics.*
+import android.hardware.display.DisplayManager
 import android.os.Bundle
-import android.os.Handler
-import android.os.HandlerThread
-import android.util.Rational
-import android.view.*
+import android.util.DisplayMetrics
+import android.util.Log
+import android.view.LayoutInflater
+import android.view.SurfaceHolder
+import android.view.View
+import android.view.ViewGroup
 import android.widget.AdapterView
 import android.widget.ArrayAdapter
 import android.widget.Toast
 import androidx.camera.core.*
+import androidx.camera.core.Camera
+import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.camera.view.PreviewView
+import androidx.constraintlayout.widget.ConstraintLayout
 import androidx.core.content.ContextCompat
 import androidx.fragment.app.Fragment
+import androidx.fragment.app.viewModels
 import androidx.lifecycle.Observer
-import androidx.lifecycle.ViewModelProviders
 import com.google.mlkit.showcase.translate.R
 import com.google.mlkit.showcase.translate.analyzer.TextAnalyzer
 import com.google.mlkit.showcase.translate.util.Language
 import kotlinx.android.synthetic.main.main_fragment.*
-import androidx.fragment.app.viewModels
-
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import kotlin.math.abs
+import kotlin.math.max
+import kotlin.math.min
 
 class MainFragment : Fragment() {
 
     companion object {
         fun newInstance() = MainFragment()
+
         // This is an arbitrary number we are using to keep tab of the permission
         // request. Where an app has multiple context for requesting permission,
         // this can help differentiate the different contexts
         private const val REQUEST_CODE_PERMISSIONS = 10
+
         // This is an array of all the permission specified in the manifest
         private val REQUIRED_PERMISSIONS = arrayOf(Manifest.permission.CAMERA)
         private const val WIDTH_CROP_PERCENT = 8
         private const val HEIGHT_CROP_PERCENT = 74
+        private const val RATIO_4_3_VALUE = 4.0 / 3.0
+        private const val RATIO_16_9_VALUE = 16.0 / 9.0
+        private const val TAG = "MainFragment"
     }
 
+    private var displayId: Int = -1
     private val viewModel: MainViewModel by viewModels()
+    private var cameraProvider: ProcessCameraProvider? = null
+    private var camera: Camera? = null
+    private var imageAnalyzer: ImageAnalysis? = null
+    private lateinit var container: ConstraintLayout
+    private lateinit var viewFinder: PreviewView
+
+    /** Blocking camera operations are performed using this executor */
+    private lateinit var cameraExecutor: ExecutorService
+
+    private val displayManager by lazy {
+        requireContext().getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
+    }
 
     override fun onCreateView(
         inflater: LayoutInflater, container: ViewGroup?,
@@ -63,26 +92,49 @@ class MainFragment : Fragment() {
         return inflater.inflate(R.layout.main_fragment, container, false)
     }
 
+    override fun onDestroyView() {
+        super.onDestroyView()
+
+        // Shut down our background executor
+        cameraExecutor.shutdown()
+
+        // Unregister listeners
+        displayManager.unregisterDisplayListener(displayListener)
+    }
+
     override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
         super.onViewCreated(view, savedInstanceState)
 
+        container = view as ConstraintLayout
+        viewFinder = container.findViewById(R.id.viewfinder)
+
+        // Initialize our background executor
+        cameraExecutor = Executors.newSingleThreadExecutor()
+
         // Request camera permissions
         if (allPermissionsGranted()) {
-            viewfinder.post { startCamera() }
+            // Wait for the views to be properly laid out
+            viewFinder.post {
+                // Keep track of the display in which this view is attached
+                displayId = viewFinder.display.displayId
+
+                // Set up the camera and its use cases
+                setUpCamera()
+            }
         } else {
             requestPermissions(REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
         }
 
+        // Every time the orientation of device changes, update rotation for use cases
+        displayManager.registerDisplayListener(displayListener, null)
+
         // Get available language list and set up the target language spinner
         // with default selections.
         val adapter = ArrayAdapter(
             requireContext(),
             android.R.layout.simple_spinner_dropdown_item, viewModel.availableLanguages
         )
-        // Every time the provided texture view changes, recompute layout
-        viewfinder.addOnLayoutChangeListener { _, _, _, _, _, _, _, _, _ ->
-            updateTransform()
-        }
+
         targetLangSelector.adapter = adapter
         targetLangSelector.setSelection(adapter.getPosition(Language("en")))
         targetLangSelector.onItemSelectedListener = object : AdapterView.OnItemSelectedListener {
@@ -140,77 +192,73 @@ class MainFragment : Fragment() {
         }
     }
 
-    private fun startCamera() {
-        // Create configuration object for the viewfinder use case
-        val previewConfig = PreviewConfig.Builder().apply {
-            setTargetAspectRatio(Rational(1, 1))
 
-        }.build()
+    /** Initialize CameraX, and prepare to bind the camera use cases  */
+    private fun setUpCamera() {
+        val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext())
+        cameraProviderFuture.addListener(Runnable {
 
-        // Build the viewfinder use case
-        val preview = Preview(previewConfig)
+            // CameraProvider
+            cameraProvider = cameraProviderFuture.get()
 
-        // Every time the viewfinder is updated, recompute layout
-        preview.setOnPreviewOutputUpdateListener {
+            // Build and bind the camera use cases
+            bindCameraUseCases()
+        }, ContextCompat.getMainExecutor(requireContext()))
+    }
 
-            // To update the SurfaceTexture, we have to remove it and re-add it
-            val parent = viewfinder.parent as ViewGroup
-            parent.removeView(viewfinder)
-            parent.addView(viewfinder, 0)
+    private fun bindCameraUseCases() {
+        val cameraProvider = cameraProvider
+            ?: throw IllegalStateException("Camera initialization failed.")
 
-            viewfinder.surfaceTexture = it.surfaceTexture
-            updateTransform()
-        }
+        // Get screen metrics used to setup camera for full screen resolution
+        val metrics = DisplayMetrics().also { viewFinder.display.getRealMetrics(it) }
+        Log.d(TAG, "Screen metrics: ${metrics.widthPixels} x ${metrics.heightPixels}")
 
-        // Setup image analysis pipeline that computes average pixel luminance
-        val analyzerConfig = ImageAnalysisConfig.Builder().apply {
-            // Use a worker thread for image analysis to prevent glitches
-            val analyzerThread = HandlerThread(
-                "TextAnalysis"
-            ).apply { start() }
-            setCallbackHandler(Handler(analyzerThread.looper))
-            // In our analysis, we care more about the latest image than
-            // analyzing *every* image
-            setImageReaderMode(
-                ImageAnalysis.ImageReaderMode.ACQUIRE_LATEST_IMAGE
-            )
-        }.build()
+        val screenAspectRatio = aspectRatio(metrics.widthPixels, metrics.heightPixels)
+        Log.d(TAG, "Preview aspect ratio: $screenAspectRatio")
+
+        val rotation = viewFinder.display.rotation
+
+        val preview = Preview.Builder()
+            .setTargetAspectRatio(screenAspectRatio)
+            .setTargetRotation(rotation)
+            .build()
 
         // Build the image analysis use case and instantiate our analyzer
-        viewModel.sourceText.observe(viewLifecycleOwner, Observer { srcText.text = it })
-        val analyzerUseCase = ImageAnalysis(analyzerConfig).apply {
-            analyzer =
-                TextAnalyzer(
-                    requireContext(),
-                    viewModel.sourceText,
-                    widthCropPercent = WIDTH_CROP_PERCENT,
-                    heightCropPercent = HEIGHT_CROP_PERCENT
+        imageAnalyzer = ImageAnalysis.Builder()
+            // We request aspect ratio but no resolution
+            .setTargetAspectRatio(screenAspectRatio)
+            .setTargetRotation(rotation)
+            .build()
+            .also {
+                it.setAnalyzer(
+                    cameraExecutor
+                    , TextAnalyzer(
+                        requireContext(),
+                        viewModel.sourceText,
+                        widthCropPercent = WIDTH_CROP_PERCENT,
+                        heightCropPercent = HEIGHT_CROP_PERCENT
+                    )
                 )
-        }
-
-        // Bind use cases to lifecycle
-        CameraX.bindToLifecycle(this, preview, analyzerUseCase)
-    }
+            }
+        viewModel.sourceText.observe(viewLifecycleOwner, Observer { srcText.text = it })
 
-    private fun updateTransform() {
-        val matrix = Matrix()
+        // Select back camera since text detection does not work with front camera
+        val cameraSelector =
+            CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build()
 
-        // Compute the center of the view finder
-        val centerX = viewfinder.width / 2f
-        val centerY = viewfinder.height / 2f
+        try {
+            // Unbind use cases before rebinding
+            cameraProvider.unbindAll()
 
-        // Correct preview output to account for display rotation
-        val rotationDegrees = when (viewfinder.display.rotation) {
-            Surface.ROTATION_0 -> 0
-            Surface.ROTATION_90 -> 90
-            Surface.ROTATION_180 -> 180
-            Surface.ROTATION_270 -> 270
-            else -> return
+            // Bind use cases to camera
+            camera = cameraProvider.bindToLifecycle(
+                this, cameraSelector, preview, imageAnalyzer
+            )
+            preview.setSurfaceProvider(viewFinder.createSurfaceProvider())
+        } catch (exc: Exception) {
+            Log.e(TAG, "Use case binding failed", exc)
         }
-        matrix.postRotate(-rotationDegrees.toFloat(), centerX, centerY)
-
-        // Finally, apply transformations to our TextureView
-        viewfinder.setTransform(matrix)
     }
 
     private fun drawOverlay(holder: SurfaceHolder) {
@@ -256,6 +304,41 @@ class MainFragment : Fragment() {
         holder.unlockCanvasAndPost(canvas)
     }
 
+    /**
+     * We need a display listener for orientation changes that do not trigger a configuration
+     * change, for example if we choose to override config change in manifest or for 180-degree
+     * orientation changes.
+     */
+    private val displayListener = object : DisplayManager.DisplayListener {
+        override fun onDisplayAdded(displayId: Int) = Unit
+        override fun onDisplayRemoved(displayId: Int) = Unit
+        override fun onDisplayChanged(displayId: Int) = view?.let { view ->
+            if (displayId == this@MainFragment.displayId) {
+                Log.d(TAG, "Rotation changed: ${view.display.rotation}")
+                imageAnalyzer?.targetRotation = view.display.rotation
+            }
+        } ?: Unit
+    }
+
+    /**
+     *  [androidx.camera.core.ImageAnalysisConfig] requires enum value of
+     *  [androidx.camera.core.AspectRatio]. Currently it has values of 4:3 & 16:9.
+     *
+     *  Detecting the most suitable ratio for dimensions provided in @params by counting absolute
+     *  of preview ratio to one of the provided values.
+     *
+     *  @param width - preview width
+     *  @param height - preview height
+     *  @return suitable aspect ratio
+     */
+    private fun aspectRatio(width: Int, height: Int): Int {
+        val previewRatio = max(width, height).toDouble() / min(width, height)
+        if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE)) {
+            return AspectRatio.RATIO_4_3
+        }
+        return AspectRatio.RATIO_16_9
+    }
+
     /**
      * Process result from permission request dialog box, has the request
      * been granted? If yes, start Camera. Otherwise display a toast
@@ -265,7 +348,7 @@ class MainFragment : Fragment() {
     ) {
         if (requestCode == REQUEST_CODE_PERMISSIONS) {
             if (allPermissionsGranted()) {
-                viewfinder.post { startCamera() }
+                viewFinder.post { bindCameraUseCases() }
             } else {
                 Toast.makeText(
                     context,

android/translate-showcase/app/src/main/res/layout/main_fragment.xml

@@ -24,7 +24,7 @@
     android:layout_height="match_parent"
     tools:context="com.google.mlkit.showcase.translate.main.MainFragment">
 
-    <TextureView
+    <androidx.camera.view.PreviewView
         android:id="@+id/viewfinder"
         android:layout_width="match_parent"
         android:layout_height="0dp"