diff --git a/lite/examples/posenet/android/README.md b/lite/examples/posenet/android/README.md new file mode 100644 index 00000000000..3905561f37e --- /dev/null +++ b/lite/examples/posenet/android/README.md @@ -0,0 +1,54 @@ +# TensorFlow Lite PoseNet Android Demo +### Overview +This is an app that continuously detects the body parts in the frames seen by + your device's camera. These instructions walk you through building and running + the demo on an Android device. + +![Demo Image](posenetimage.png) + +## Build the demo using Android Studio + +### Prerequisites + +* If you don't have it already, install **[Android Studio]( + https://developer.android.com/studio/index.html)** 3.2 or + later, following the instructions on the website. + +* Android device and Android development environment with minimum API 21. + +### Building +* Open Android Studio, and from the `Welcome` screen, select +`Open an existing Android Studio project`. + +* From the `Open File or Project` window that appears, navigate to and select + the `tensorflow-lite/examples/posenet/android` directory from wherever you + cloned the TensorFlow Lite sample GitHub repo. Click `OK`. + +* If it asks you to do a `Gradle Sync`, click `OK`. + +* You may also need to install various platforms and tools, if you get errors + like `Failed to find target with hash string 'android-21'` and similar. Click + the `Run` button (the green arrow) or select `Run` > `Run 'android'` from the + top menu. You may need to rebuild the project using `Build` > `Rebuild Project`. + +* If it asks you to use `Instant Run`, click `Proceed Without Instant Run`. + +* Also, you need to have an Android device plugged in with developer options + enabled at this point. See **[here]( + https://developer.android.com/studio/run/device)** for more details + on setting up developer devices. + + +### Model used +Downloading, extraction and placement in assets folder has been managed + automatically by `download.gradle`. + +If you explicitly want to download the model, you can download it from + **[here]( + https://storage.googleapis.com/download.tensorflow.org/models/tflite/posenet_mobilenet_v1_100_513x513_multi_kpt_stripped.tflite)**. + +### Additional Note +_Please do not delete the assets folder content_. If you explicitly deleted the + files, then please choose `Build` > `Rebuild` from menu to re-download the + deleted model files into assets folder. + diff --git a/lite/examples/posenet/android/app/build.gradle b/lite/examples/posenet/android/app/build.gradle new file mode 100644 index 00000000000..c18f0242e50 --- /dev/null +++ b/lite/examples/posenet/android/app/build.gradle @@ -0,0 +1,46 @@ +apply plugin: 'com.android.application' + +apply plugin: 'kotlin-android' + +apply plugin: 'kotlin-android-extensions' + +android { + compileSdkVersion 29 + buildToolsVersion "29.0.0" + defaultConfig { + applicationId "org.tensorflow.lite.examples.posenet" + minSdkVersion 21 + targetSdkVersion 23 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + aaptOptions { + noCompress "tflite" + } + lintOptions { + checkReleaseBuilds false + // Or, if you prefer, you can continue to check for errors in release builds, + // but continue the build even when errors are found: + abortOnError false + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation project(":posenet") + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + implementation 'com.android.support:appcompat-v7:28.0.0' + implementation 'com.android.support:design:28.0.0' + implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + implementation 'org.tensorflow:tensorflow-lite:1.14.0' + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test:runner:1.2.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' +} diff --git a/lite/examples/posenet/android/app/proguard-rules.pro b/lite/examples/posenet/android/app/proguard-rules.pro new file mode 100644 index 00000000000..f1b424510da --- /dev/null +++ b/lite/examples/posenet/android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/lite/examples/posenet/android/app/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.kt b/lite/examples/posenet/android/app/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.kt new file mode 100644 index 00000000000..ea975741c43 --- /dev/null +++ b/lite/examples/posenet/android/app/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet + +import androidx.test.InstrumentationRegistry +import androidx.test.runner.AndroidJUnit4 +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getTargetContext() + assertEquals("org.tensorflow.lite.examples.posenet", appContext.packageName) + } +} diff --git a/lite/examples/posenet/android/app/src/main/AndroidManifest.xml b/lite/examples/posenet/android/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..e0e79f5a1fc --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/CameraActivity.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/CameraActivity.kt new file mode 100644 index 00000000000..7e0028e7077 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/CameraActivity.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet + +import android.os.Bundle +import android.support.v7.app.AppCompatActivity + +class CameraActivity : AppCompatActivity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_camera) + savedInstanceState ?: supportFragmentManager.beginTransaction() + .replace(R.id.container, PosenetActivity()) + .commit() + } +} diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ConfirmationDialog.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ConfirmationDialog.kt new file mode 100644 index 00000000000..4fd0b333f1f --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ConfirmationDialog.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet + +import android.Manifest +import android.app.AlertDialog +import android.app.Dialog +import android.os.Bundle +import android.support.v4.app.DialogFragment + +/** + * Shows OK/Cancel confirmation dialog about camera permission. + */ +class ConfirmationDialog : DialogFragment() { + + override fun onCreateDialog(savedInstanceState: Bundle?): Dialog = + AlertDialog.Builder(activity) + .setMessage(R.string.request_permission) + .setPositiveButton(android.R.string.ok) { _, _ -> + parentFragment!!.requestPermissions( + arrayOf(Manifest.permission.CAMERA), + REQUEST_CAMERA_PERMISSION + ) + } + .setNegativeButton(android.R.string.cancel) { _, _ -> + parentFragment!!.activity?.finish() + } + .create() +} diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/Constants.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/Constants.kt new file mode 100644 index 00000000000..15c4f4995c3 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/Constants.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("Constants") + +package org.tensorflow.lite.examples.posenet + +/** Request camera and external storage permission. */ +const val REQUEST_CAMERA_PERMISSION = 1 +const val REQUEST_CODE_WRITE_EXTERNAL_STORAGE_PERMISSION = 1 + +/** Model input shape for images. */ +const val MODEL_WIDTH = 513 +const val MODEL_HEIGHT = 513 diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ImageUtils.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ImageUtils.kt new file mode 100644 index 00000000000..808d8e7972f --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/ImageUtils.kt @@ -0,0 +1,122 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.posenet + +import android.graphics.Bitmap +import android.os.Environment +import android.util.Log +import java.io.File +import java.io.FileOutputStream + +/** Utility class for manipulating images. */ +object ImageUtils { + // This value is 2 ^ 18 - 1, and is used to hold the RGB values together before their ranges + // are normalized to eight bits. + private const val MAX_CHANNEL_VALUE = 262143 + + /** Directory where the bitmaps are saved for analysis. */ + private fun rootDirectory(): String { + return Environment.getExternalStorageDirectory().absolutePath + File.separator + + "tensorflow" + } + + /** + * Saves a Bitmap object to disk for analysis. + * + * @param bitmap The bitmap to save. + * @param filename The location to save the bitmap to. + */ + @JvmOverloads + fun saveBitmap(bitmap: Bitmap, filename: String = "preview.png") { + val root = rootDirectory() + val myDir = File(root) + if (!myDir.exists() or !myDir.isDirectory) { + if (!myDir.mkdirs()) { + Log.e("Local storage", "Failed to create directory for the app in root") + } + } + + val file = File(myDir, filename) + if (file.exists()) { + file.delete() + } + val out = FileOutputStream(file) + try { + bitmap.compress(Bitmap.CompressFormat.PNG, 99, out) + out.flush() + } catch (e: Exception) { + Log.e("Compressing output", e.toString()) + } finally { + out.close() + } + } + + /** Helper function to convert y,u,v integer values to RGB format */ + private fun convertYUVToRGB(y: Int, u: Int, v: Int): Int { + // Adjust and check YUV values + val yNew = if (y - 16 < 0) 0 else y - 16 + val uNew = u - 128 + val vNew = v - 128 + val expandY = 1192 * yNew + var r = expandY + 1634 * vNew + var g = expandY - 833 * vNew - 400 * uNew + var b = expandY + 2066 * uNew + + // Clipping RGB values to be inside boundaries [ 0 , MAX_CHANNEL_VALUE ] + val checkBoundaries = { x: Int -> + when { + x > MAX_CHANNEL_VALUE -> MAX_CHANNEL_VALUE + x < 0 -> 0 + else -> x + } + } + r = checkBoundaries(r) + g = checkBoundaries(g) + b = checkBoundaries(b) + return -0x1000000 or (r shl 6 and 0xff0000) or (g shr 2 and 0xff00) or (b shr 10 and 0xff) + } + + /** Converts YUV420 format image data (ByteArray) into ARGB8888 format with IntArray as output. */ + fun convertYUV420ToARGB8888( + yData: ByteArray, + uData: ByteArray, + vData: ByteArray, + width: Int, + height: Int, + yRowStride: Int, + uvRowStride: Int, + uvPixelStride: Int, + out: IntArray + ) { + var outputIndex = 0 + for (j in 0 until height) { + val positionY = yRowStride * j + val positionUV = uvRowStride * (j shr 1) + + for (i in 0 until width) { + val uvOffset = positionUV + (i shr 1) * uvPixelStride + + // "0xff and" is used to cut off bits from following value that are higher than + // the low 8 bits + out[outputIndex] = convertYUVToRGB( + 0xff and yData[positionY + i].toInt(), 0xff and uData[uvOffset].toInt(), + 0xff and vData[uvOffset].toInt() + ) + outputIndex += 1 + } + } + } +} diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/PosenetActivity.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/PosenetActivity.kt new file mode 100644 index 00000000000..22fb277028d --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/PosenetActivity.kt @@ -0,0 +1,689 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet + +import android.Manifest +import android.app.AlertDialog +import android.app.Dialog +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.ImageFormat +import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.Rect +import android.hardware.camera2.CameraAccessException +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult +import android.media.Image +import android.media.ImageReader +import android.media.ImageReader.OnImageAvailableListener +import android.os.Bundle +import android.os.Handler +import android.os.HandlerThread +import android.support.v4.app.ActivityCompat +import android.support.v4.app.DialogFragment +import android.support.v4.app.Fragment +import android.support.v4.content.ContextCompat +import android.util.Log +import android.util.Size +import android.util.SparseIntArray +import android.view.LayoutInflater +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import android.view.View +import android.view.ViewGroup +import android.widget.Toast +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit +import kotlin.math.abs +import kotlin.math.pow +import org.tensorflow.lite.examples.posenet.lib.BodyPart +import org.tensorflow.lite.examples.posenet.lib.Person +import org.tensorflow.lite.examples.posenet.lib.Posenet + +class PosenetActivity : + Fragment(), + ActivityCompat.OnRequestPermissionsResultCallback { + + /** List of body joints that should be connected. */ + private val bodyJoints = listOf( + Pair(BodyPart.LEFT_WRIST, BodyPart.LEFT_ELBOW), + Pair(BodyPart.LEFT_ELBOW, BodyPart.LEFT_SHOULDER), + Pair(BodyPart.LEFT_SHOULDER, BodyPart.RIGHT_SHOULDER), + Pair(BodyPart.RIGHT_SHOULDER, BodyPart.RIGHT_ELBOW), + Pair(BodyPart.RIGHT_ELBOW, BodyPart.RIGHT_WRIST), + Pair(BodyPart.LEFT_SHOULDER, BodyPart.LEFT_HIP), + Pair(BodyPart.LEFT_HIP, BodyPart.RIGHT_HIP), + Pair(BodyPart.RIGHT_HIP, BodyPart.RIGHT_SHOULDER), + Pair(BodyPart.LEFT_HIP, BodyPart.LEFT_KNEE), + Pair(BodyPart.LEFT_KNEE, BodyPart.LEFT_ANKLE), + Pair(BodyPart.RIGHT_HIP, BodyPart.RIGHT_KNEE), + Pair(BodyPart.RIGHT_KNEE, BodyPart.RIGHT_ANKLE) + ) + + /** Threshold for confidence score. */ + private val minConfidence = 0.2 + + /** Radius of circle used to draw keypoints. */ + private val circleRadius = 8.0f + + /** Paint class holds the style and color information to draw geometries,text and bitmaps. */ + private var paint = Paint() + + /** A shape for extracting frame data. */ + private val PREVIEW_WIDTH = 640 + private val PREVIEW_HEIGHT = 480 + + /** An object for the Posenet library. */ + private lateinit var posenet: Posenet + + /** ID of the current [CameraDevice]. */ + private var cameraId: String? = null + + /** A [SurfaceView] for camera preview. */ + private var surfaceView: SurfaceView? = null + + /** A [CameraCaptureSession] for camera preview. */ + private var captureSession: CameraCaptureSession? = null + + /** A reference to the opened [CameraDevice]. */ + private var cameraDevice: CameraDevice? = null + + /** The [android.util.Size] of camera preview. */ + private var previewSize: Size? = null + + /** The [android.util.Size.getWidth] of camera preview. */ + private var previewWidth = 0 + + /** The [android.util.Size.getHeight] of camera preview. */ + private var previewHeight = 0 + + /** A counter to keep count of total frames. */ + private var frameCounter = 0 + + /** An IntArray to save image data in ARGB8888 format */ + private lateinit var rgbBytes: IntArray + + /** A ByteArray to save image data in YUV format */ + private var yuvBytes = arrayOfNulls(3) + + /** An additional thread for running tasks that shouldn't block the UI. */ + private var backgroundThread: HandlerThread? = null + + /** A [Handler] for running tasks in the background. */ + private var backgroundHandler: Handler? = null + + /** An [ImageReader] that handles preview frame capture. */ + private var imageReader: ImageReader? = null + + /** [CaptureRequest.Builder] for the camera preview */ + private var previewRequestBuilder: CaptureRequest.Builder? = null + + /** [CaptureRequest] generated by [.previewRequestBuilder */ + private var previewRequest: CaptureRequest? = null + + /** A [Semaphore] to prevent the app from exiting before closing the camera. */ + private val cameraOpenCloseLock = Semaphore(1) + + /** Whether the current camera device supports Flash or not. */ + private var flashSupported = false + + /** Orientation of the camera sensor. */ + private var sensorOrientation: Int? = null + + /** Abstract interface to someone holding a display surface. */ + private var surfaceHolder: SurfaceHolder? = null + + /** [CameraDevice.StateCallback] is called when [CameraDevice] changes its state. */ + private val stateCallback = object : CameraDevice.StateCallback() { + + override fun onOpened(cameraDevice: CameraDevice) { + cameraOpenCloseLock.release() + this@PosenetActivity.cameraDevice = cameraDevice + createCameraPreviewSession() + } + + override fun onDisconnected(cameraDevice: CameraDevice) { + cameraOpenCloseLock.release() + cameraDevice.close() + this@PosenetActivity.cameraDevice = null + } + + override fun onError(cameraDevice: CameraDevice, error: Int) { + onDisconnected(cameraDevice) + this@PosenetActivity.activity?.finish() + } + } + + /** + * A [CameraCaptureSession.CaptureCallback] that handles events related to JPEG capture. + */ + private val captureCallback = object : CameraCaptureSession.CaptureCallback() { + override fun onCaptureProgressed( + session: CameraCaptureSession, + request: CaptureRequest, + partialResult: CaptureResult + ) { + } + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + } + } + + /** + * Shows a [Toast] on the UI thread. + * + * @param text The message to show + */ + private fun showToast(text: String) { + val activity = activity + activity?.runOnUiThread { Toast.makeText(activity, text, Toast.LENGTH_SHORT).show() } + } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? + ): View? = inflater.inflate(R.layout.activity_posenet, container, false) + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + surfaceView = view.findViewById(R.id.surfaceView) + surfaceHolder = surfaceView!!.holder + } + + override fun onResume() { + super.onResume() + startBackgroundThread() + } + + override fun onStart() { + super.onStart() + openCamera() + posenet = Posenet(this.context!!) + } + + override fun onPause() { + closeCamera() + stopBackgroundThread() + super.onPause() + } + + private fun requestCameraPermission() { + if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) { + ConfirmationDialog().show(childFragmentManager, FRAGMENT_DIALOG) + } else { + requestPermissions(arrayOf(Manifest.permission.CAMERA), REQUEST_CAMERA_PERMISSION) + } + } + + private fun requestStoragePermission() { + if (shouldShowRequestPermissionRationale(Manifest.permission.WRITE_EXTERNAL_STORAGE)) { + ConfirmationDialog().show(childFragmentManager, FRAGMENT_DIALOG) + } else { + requestPermissions( + arrayOf(Manifest.permission.WRITE_EXTERNAL_STORAGE), + REQUEST_CODE_WRITE_EXTERNAL_STORAGE_PERMISSION + ) + } + } + + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray + ) { + if (requestCode == REQUEST_CAMERA_PERMISSION) { + if (grantResults.size != 1 || grantResults[0] != PackageManager.PERMISSION_GRANTED) { + ErrorDialog.newInstance(getString(R.string.request_permission)) + .show(childFragmentManager, FRAGMENT_DIALOG) + } + } else { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + } + } + + /** + * Sets up member variables related to camera. + */ + private fun setUpCameraOutputs() { + + val activity = activity + val manager = activity!!.getSystemService(Context.CAMERA_SERVICE) as CameraManager + try { + for (cameraId in manager.cameraIdList) { + val characteristics = manager.getCameraCharacteristics(cameraId) + + // We don't use a front facing camera in this sample. + val cameraDirection = characteristics.get(CameraCharacteristics.LENS_FACING) + if (cameraDirection != null && + cameraDirection == CameraCharacteristics.LENS_FACING_FRONT + ) { + continue + } + + previewSize = Size(PREVIEW_WIDTH, PREVIEW_HEIGHT) + + imageReader = ImageReader.newInstance( + PREVIEW_WIDTH, PREVIEW_HEIGHT, + ImageFormat.YUV_420_888, /*maxImages*/ 2 + ) + + sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION)!! + + previewHeight = previewSize!!.height + previewWidth = previewSize!!.width + + // Initialize the storage bitmaps once when the resolution is known. + rgbBytes = IntArray(previewWidth * previewHeight) + + // Check if the flash is supported. + flashSupported = + characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) == true + + this.cameraId = cameraId + + // We've found a viable camera and finished setting up member variables, + // so we don't need to iterate through other available cameras. + return + } + } catch (e: CameraAccessException) { + Log.e(TAG, e.toString()) + } catch (e: NullPointerException) { + // Currently an NPE is thrown when the Camera2API is used but not supported on the + // device this code runs. + ErrorDialog.newInstance(getString(R.string.camera_error)) + .show(childFragmentManager, FRAGMENT_DIALOG) + } + } + + /** + * Opens the camera specified by [PosenetActivity.cameraId]. + */ + private fun openCamera() { + val permissionCamera = ContextCompat.checkSelfPermission(activity!!, Manifest.permission.CAMERA) + if (permissionCamera != PackageManager.PERMISSION_GRANTED) { + requestCameraPermission() + } else { + setUpCameraOutputs() + val manager = activity!!.getSystemService(Context.CAMERA_SERVICE) as CameraManager + try { + // Wait for camera to open - 2.5 seconds is sufficient + if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) { + throw RuntimeException("Time out waiting to lock camera opening.") + } + manager.openCamera(cameraId!!, stateCallback, backgroundHandler) + } catch (e: CameraAccessException) { + Log.e(TAG, e.toString()) + } catch (e: InterruptedException) { + throw RuntimeException("Interrupted while trying to lock camera opening.", e) + } + } + val permissionStorage = ContextCompat.checkSelfPermission( + activity!!, + Manifest.permission.WRITE_EXTERNAL_STORAGE + ) + if (permissionStorage != PackageManager.PERMISSION_GRANTED) { + requestStoragePermission() + } + } + + /** + * Closes the current [CameraDevice]. + */ + private fun closeCamera() { + if (captureSession == null) { + return + } + + try { + cameraOpenCloseLock.acquire() + captureSession!!.close() + captureSession = null + cameraDevice!!.close() + cameraDevice = null + imageReader!!.close() + imageReader = null + } catch (e: InterruptedException) { + throw RuntimeException("Interrupted while trying to lock camera closing.", e) + } finally { + cameraOpenCloseLock.release() + } + } + + /** + * Starts a background thread and its [Handler]. + */ + private fun startBackgroundThread() { + backgroundThread = HandlerThread("imageAvailableListener").also { it.start() } + backgroundHandler = Handler(backgroundThread!!.looper) + } + + /** + * Stops the background thread and its [Handler]. + */ + private fun stopBackgroundThread() { + backgroundThread?.quitSafely() + try { + backgroundThread?.join() + backgroundThread = null + backgroundHandler = null + } catch (e: InterruptedException) { + Log.e(TAG, e.toString()) + } + } + + /** Fill the yuvBytes with data from image planes. */ + private fun fillBytes(planes: Array, yuvBytes: Array) { + // Row stride is the total number of bytes occupied in memory by a row of an image. + // Because of the variable row stride it's not possible to know in + // advance the actual necessary dimensions of the yuv planes. + for (i in planes.indices) { + val buffer = planes[i].buffer + if (yuvBytes[i] == null) { + yuvBytes[i] = ByteArray(buffer.capacity()) + } + buffer.get(yuvBytes[i]!!) + } + } + + /** A [OnImageAvailableListener] to receive frames as they are available. */ + private var imageAvailableListener = object : OnImageAvailableListener { + override fun onImageAvailable(imageReader: ImageReader) { + // We need wait until we have some size from onPreviewSizeChosen + if (previewWidth == 0 || previewHeight == 0) { + return + } + + val image = imageReader.acquireLatestImage() ?: return + fillBytes(image.planes, yuvBytes) + + ImageUtils.convertYUV420ToARGB8888( + yuvBytes[0]!!, + yuvBytes[1]!!, + yuvBytes[2]!!, + previewWidth, + previewHeight, + /*yRowStride=*/ image.planes[0].rowStride, + /*uvRowStride=*/ image.planes[1].rowStride, + /*uvPixelStride=*/ image.planes[1].pixelStride, + rgbBytes + ) + + // Create bitmap from int array + val imageBitmap = Bitmap.createBitmap( + rgbBytes, previewWidth, previewHeight, + Bitmap.Config.ARGB_8888 + ) + + // Create rotated version for portrait display + val rotateMatrix = Matrix() + rotateMatrix.postRotate(90.0f) + + val rotatedBitmap = Bitmap.createBitmap( + imageBitmap, 0, 0, previewWidth, previewHeight, + rotateMatrix, true + ) + + // Save an image for analysis in every 30 frames. + frameCounter += 1 + if (frameCounter % 30 == 0) { + ImageUtils.saveBitmap(imageBitmap) + } + image.close() + processImage(rotatedBitmap) + } + } + + /** Crop Bitmap to maintain aspect ratio of model input. */ + private fun cropBitmap(bitmap: Bitmap): Bitmap { + // Rotated bitmap has previewWidth as its height and previewHeight as width. + val previewRatio = previewWidth.toFloat() / previewHeight + val modelInputRatio = MODEL_HEIGHT.toFloat() / MODEL_WIDTH + var croppedBitmap = bitmap + + // Acceptable difference between the modelInputRatio and previewRatio to skip cropping. + val maxDifference = 1.0f.pow(-5) + + // Checks if the previewing bitmap has similar aspect ratio as the required model input. + when { + abs(modelInputRatio - previewRatio) < maxDifference -> return croppedBitmap + modelInputRatio > previewRatio -> { + // New image is taller so we are height constrained. + val cropHeight = previewHeight - (previewWidth.toFloat() / modelInputRatio) + croppedBitmap = Bitmap.createBitmap( + bitmap, + 0, + (cropHeight / 2).toInt(), + previewHeight, + (previewWidth - (cropHeight / 2)).toInt() + ) + } + else -> { + val cropWidth = previewWidth - (previewHeight.toFloat() * modelInputRatio) + croppedBitmap = Bitmap.createBitmap( + bitmap, + (cropWidth / 2).toInt(), + 0, + (previewHeight - (cropWidth / 2)).toInt(), + previewWidth + ) + } + } + return croppedBitmap + } + + /** Set the paint color and size. */ + private fun setPaint() { + paint.color = Color.RED + paint.textSize = 80.0f + paint.strokeWidth = 8.0f + } + + /** Draw bitmap on Canvas. */ + private fun draw(canvas: Canvas, person: Person, bitmap: Bitmap) { + val screenWidth: Int = canvas.width + val screenHeight: Int = canvas.height + setPaint() + canvas.drawBitmap( + bitmap, + Rect(0, 0, previewHeight, previewWidth), + Rect(0, 0, screenWidth, screenHeight), + paint + ) + + val widthRatio = screenWidth.toFloat() / MODEL_WIDTH + val heightRatio = screenHeight.toFloat() / MODEL_HEIGHT + + // Draw key points over the image. + for (keyPoint in person.keyPoints) { + if (keyPoint.score > minConfidence) { + val position = keyPoint.position + val adjustedX: Float = position.x.toFloat() * widthRatio + val adjustedY: Float = position.y.toFloat() * heightRatio + canvas.drawCircle(adjustedX, adjustedY, circleRadius, paint) + } + } + + for (line in bodyJoints) { + if ( + (person.keyPoints[line.first.ordinal].score > minConfidence) and + (person.keyPoints[line.second.ordinal].score > minConfidence) + ) { + canvas.drawLine( + person.keyPoints[line.first.ordinal].position.x.toFloat() * widthRatio, + person.keyPoints[line.first.ordinal].position.y.toFloat() * heightRatio, + person.keyPoints[line.second.ordinal].position.x.toFloat() * widthRatio, + person.keyPoints[line.second.ordinal].position.y.toFloat() * heightRatio, + paint + ) + } + } + + // Draw confidence score of a person. + val scoreMessage = "SCORE: " + "%.2f".format(person.score) + canvas.drawText( + scoreMessage, + (15.0f * widthRatio), + (500.0f * heightRatio), + paint + ) + + // Draw! + surfaceHolder!!.unlockCanvasAndPost(canvas) + } + + /** Process image using Posenet library. */ + private fun processImage(bitmap: Bitmap) { + // Crop bitmap. + val croppedBitmap = cropBitmap(bitmap) + + // Created scaled version of bitmap for model input. + val scaledBitmap = Bitmap.createScaledBitmap(croppedBitmap, MODEL_WIDTH, MODEL_HEIGHT, true) + + // Perform inference. + val person = posenet.estimateSinglePose(scaledBitmap) + val canvas: Canvas = surfaceHolder!!.lockCanvas() + draw(canvas, person, bitmap) + } + + /** + * Creates a new [CameraCaptureSession] for camera preview. + */ + private fun createCameraPreviewSession() { + try { + + // We capture images from preview in YUV format. + imageReader = ImageReader.newInstance( + previewSize!!.width, previewSize!!.height, ImageFormat.YUV_420_888, 2 + ) + imageReader!!.setOnImageAvailableListener(imageAvailableListener, backgroundHandler) + + // This is the surface we need to record images for processing. + val recordingSurface = imageReader!!.surface + + // We set up a CaptureRequest.Builder with the output Surface. + previewRequestBuilder = cameraDevice!!.createCaptureRequest( + CameraDevice.TEMPLATE_PREVIEW + ) + previewRequestBuilder!!.addTarget(recordingSurface) + + // Here, we create a CameraCaptureSession for camera preview. + cameraDevice!!.createCaptureSession( + listOf(recordingSurface), + object : CameraCaptureSession.StateCallback() { + override fun onConfigured(cameraCaptureSession: CameraCaptureSession) { + // The camera is already closed + if (cameraDevice == null) return + + // When the session is ready, we start displaying the preview. + captureSession = cameraCaptureSession + try { + // Auto focus should be continuous for camera preview. + previewRequestBuilder!!.set( + CaptureRequest.CONTROL_AF_MODE, + CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE + ) + // Flash is automatically enabled when necessary. + setAutoFlash(previewRequestBuilder!!) + + // Finally, we start displaying the camera preview. + previewRequest = previewRequestBuilder!!.build() + captureSession!!.setRepeatingRequest( + previewRequest!!, + captureCallback, backgroundHandler + ) + } catch (e: CameraAccessException) { + Log.e(TAG, e.toString()) + } + } + + override fun onConfigureFailed(cameraCaptureSession: CameraCaptureSession) { + showToast("Failed") + } + }, + null + ) + } catch (e: CameraAccessException) { + Log.e(TAG, e.toString()) + } + } + + private fun setAutoFlash(requestBuilder: CaptureRequest.Builder) { + if (flashSupported) { + requestBuilder.set( + CaptureRequest.CONTROL_AE_MODE, + CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH + ) + } + } + + /** + * Shows an error message dialog. + */ + class ErrorDialog : DialogFragment() { + + override fun onCreateDialog(savedInstanceState: Bundle?): Dialog = + AlertDialog.Builder(activity) + .setMessage(arguments!!.getString(ARG_MESSAGE)) + .setPositiveButton(android.R.string.ok) { _, _ -> activity!!.finish() } + .create() + + companion object { + + @JvmStatic + private val ARG_MESSAGE = "message" + + @JvmStatic + fun newInstance(message: String): ErrorDialog = ErrorDialog().apply { + arguments = Bundle().apply { putString(ARG_MESSAGE, message) } + } + } + } + + companion object { + /** + * Conversion from screen rotation to JPEG orientation. + */ + private val ORIENTATIONS = SparseIntArray() + private val FRAGMENT_DIALOG = "dialog" + + init { + ORIENTATIONS.append(Surface.ROTATION_0, 90) + ORIENTATIONS.append(Surface.ROTATION_90, 0) + ORIENTATIONS.append(Surface.ROTATION_180, 270) + ORIENTATIONS.append(Surface.ROTATION_270, 180) + } + + /** + * Tag for the [Log]. + */ + private const val TAG = "PosenetActivity" + } +} diff --git a/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/TestActivity.kt b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/TestActivity.kt new file mode 100644 index 00000000000..92ec82a8cec --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/java/org/tensorflow/lite/examples/posenet/TestActivity.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.drawable.Drawable +import android.os.Bundle +import android.support.v4.content.res.ResourcesCompat +import android.support.v7.app.AppCompatActivity +import android.widget.ImageView +import org.tensorflow.lite.examples.posenet.lib.Posenet as Posenet + +class TestActivity : AppCompatActivity() { + /** Returns a resized bitmap of the drawable image. */ + private fun drawableToBitmap(drawable: Drawable): Bitmap { + val bitmap = Bitmap.createBitmap(257, 353, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + + drawable.setBounds(0, 0, canvas.width, canvas.height) + + drawable.draw(canvas) + return bitmap + } + + /** Calls the Posenet library functions. */ + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_test) + + val sampleImageView = findViewById(R.id.image) + val drawedImage = ResourcesCompat.getDrawable(resources, R.drawable.image, null) + val imageBitmap = drawableToBitmap(drawedImage!!) + sampleImageView.setImageBitmap(imageBitmap) + val posenet = Posenet(this.applicationContext) + val person = posenet.estimateSinglePose(imageBitmap) + + // Draw the keypoints over the image. + val paint = Paint() + paint.color = Color.RED + val size = 2.0f + + val mutableBitmap = imageBitmap.copy(Bitmap.Config.ARGB_8888, true) + val canvas = Canvas(mutableBitmap) + for (keypoint in person.keyPoints) { + canvas.drawCircle( + keypoint.position.x.toFloat(), + keypoint.position.y.toFloat(), size, paint + ) + } + sampleImageView.adjustViewBounds = true + sampleImageView.setImageBitmap(mutableBitmap) + } +} diff --git a/lite/examples/posenet/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/lite/examples/posenet/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 00000000000..1f1132fc18c --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/lite/examples/posenet/android/app/src/main/res/drawable/action.png b/lite/examples/posenet/android/app/src/main/res/drawable/action.png new file mode 100644 index 00000000000..dead90c10ad Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/drawable/action.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/drawable/ic_launcher_background.xml b/lite/examples/posenet/android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 00000000000..a7bb6206ebc --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lite/examples/posenet/android/app/src/main/res/drawable/image.jpg b/lite/examples/posenet/android/app/src/main/res/drawable/image.jpg new file mode 100644 index 00000000000..9615b7cc73c Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/drawable/image.jpg differ diff --git a/lite/examples/posenet/android/app/src/main/res/drawable/tf_ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/drawable/tf_ic_launcher.png new file mode 100644 index 00000000000..52cf2ab9529 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/drawable/tf_ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/drawable/tfl_logo.png b/lite/examples/posenet/android/app/src/main/res/drawable/tfl_logo.png new file mode 100644 index 00000000000..44396c642f7 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/drawable/tfl_logo.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/layout/activity_camera.xml b/lite/examples/posenet/android/app/src/main/res/layout/activity_camera.xml new file mode 100644 index 00000000000..381388aadcb --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/layout/activity_camera.xml @@ -0,0 +1,22 @@ + + \ No newline at end of file diff --git a/lite/examples/posenet/android/app/src/main/res/layout/activity_posenet.xml b/lite/examples/posenet/android/app/src/main/res/layout/activity_posenet.xml new file mode 100644 index 00000000000..5a3cbdfed27 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/layout/activity_posenet.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/lite/examples/posenet/android/app/src/main/res/layout/activity_test.xml b/lite/examples/posenet/android/app/src/main/res/layout/activity_test.xml new file mode 100644 index 00000000000..afdf177b696 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/layout/activity_test.xml @@ -0,0 +1,27 @@ + + + + + + diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 00000000000..6d5e5d094cb --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 00000000000..6d5e5d094cb --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_action_info.png b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_action_info.png new file mode 100644 index 00000000000..32bd1aabcab Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_action_info.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 00000000000..898f3ed59ac Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 00000000000..dffca3601eb Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/tf_ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/tf_ic_launcher.png new file mode 100644 index 00000000000..52cf2ab9529 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-hdpi/tf_ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_action_info.png b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_action_info.png new file mode 100644 index 00000000000..8efbbf8b3c4 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_action_info.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 00000000000..64ba76f75e9 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 00000000000..dae5e082342 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/tf_ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/tf_ic_launcher.png new file mode 100644 index 00000000000..b75f892c462 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-mdpi/tf_ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_action_info.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_action_info.png new file mode 100644 index 00000000000..ba143ea7a80 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_action_info.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 00000000000..e5ed46597ea Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 00000000000..14ed0af3502 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/tf_ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/tf_ic_launcher.png new file mode 100644 index 00000000000..36e14c48d14 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xhdpi/tf_ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_action_info.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_action_info.png new file mode 100644 index 00000000000..394eb7e5349 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_action_info.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000..b0907cac3bf Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 00000000000..d8ae0315497 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/tf_ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/tf_ic_launcher.png new file mode 100644 index 00000000000..06dd2a740ec Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxhdpi/tf_ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 00000000000..2c18de9e661 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 00000000000..beed3cdd2c3 Binary files /dev/null and b/lite/examples/posenet/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/lite/examples/posenet/android/app/src/main/res/values/colors.xml b/lite/examples/posenet/android/app/src/main/res/values/colors.xml new file mode 100644 index 00000000000..eed1ecdec39 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/values/colors.xml @@ -0,0 +1,23 @@ + + + + #008577 + #00574B + #D81B60 + #cc4285f4 + #66000000 + diff --git a/lite/examples/posenet/android/app/src/main/res/values/dimens.xml b/lite/examples/posenet/android/app/src/main/res/values/dimens.xml new file mode 100644 index 00000000000..1f2a4dd130c --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/values/dimens.xml @@ -0,0 +1,20 @@ + + + + 112dp + 20dp + diff --git a/lite/examples/posenet/android/app/src/main/res/values/strings.xml b/lite/examples/posenet/android/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000..a047d83b8c8 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/values/strings.xml @@ -0,0 +1,27 @@ + + + Posenet Demo + Picture + Info + This app needs camera permission. + This device doesn\'t support Camera2 API. + + + + diff --git a/lite/examples/posenet/android/app/src/main/res/values/styles.xml b/lite/examples/posenet/android/app/src/main/res/values/styles.xml new file mode 100644 index 00000000000..74f586cf184 --- /dev/null +++ b/lite/examples/posenet/android/app/src/main/res/values/styles.xml @@ -0,0 +1,18 @@ + + + + + + + diff --git a/lite/examples/posenet/android/build.gradle b/lite/examples/posenet/android/build.gradle new file mode 100644 index 00000000000..4b5160aff14 --- /dev/null +++ b/lite/examples/posenet/android/build.gradle @@ -0,0 +1,29 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + ext.kotlin_version = '1.3.40' + ext.kotlin_version = '1.3.31' + repositories { + google() + jcenter() + + } + dependencies { + classpath 'com.android.tools.build:gradle:3.4.1' + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + jcenter() + + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/lite/examples/posenet/android/gradle.properties b/lite/examples/posenet/android/gradle.properties new file mode 100644 index 00000000000..2c1318f7c18 --- /dev/null +++ b/lite/examples/posenet/android/gradle.properties @@ -0,0 +1,21 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=false +# Automatically convert third-party libraries to use AndroidX +android.enableJetifier=false +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official diff --git a/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.jar b/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000000..f6b961fd5a8 Binary files /dev/null and b/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.properties b/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..ab281bce201 --- /dev/null +++ b/lite/examples/posenet/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Jun 25 14:46:07 PDT 2019 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip diff --git a/lite/examples/posenet/android/gradlew b/lite/examples/posenet/android/gradlew new file mode 100755 index 00000000000..cccdd3d517f --- /dev/null +++ b/lite/examples/posenet/android/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/lite/examples/posenet/android/gradlew.bat b/lite/examples/posenet/android/gradlew.bat new file mode 100644 index 00000000000..f9553162f12 --- /dev/null +++ b/lite/examples/posenet/android/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lite/examples/posenet/android/posenet/build.gradle b/lite/examples/posenet/android/posenet/build.gradle new file mode 100644 index 00000000000..9d410ac1f9d --- /dev/null +++ b/lite/examples/posenet/android/posenet/build.gradle @@ -0,0 +1,56 @@ +apply plugin: 'com.android.library' + +apply plugin: 'kotlin-android' + +apply plugin: 'kotlin-android-extensions' + +android { + compileSdkVersion 29 + buildToolsVersion "29.0.0" + + defaultConfig { + minSdkVersion 21 + targetSdkVersion 23 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + + aaptOptions { + noCompress "tflite" + } + + lintOptions { + checkReleaseBuilds false + // Or, if you prefer, you can continue to check for errors in release builds, + // but continue the build even when errors are found: + abortOnError false + } +} + +// Download default models; if you wish to use your own models then +// place them in the "assets" directory and comment out this line. +apply from:'download.gradle' + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation 'com.android.support:appcompat-v7:28.0.0' + implementation 'com.android.support:design:28.0.0' + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test:runner:1.2.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + implementation 'org.tensorflow:tensorflow-lite:1.14.0' + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" +} +repositories { + mavenCentral() +} diff --git a/lite/examples/posenet/android/posenet/download.gradle b/lite/examples/posenet/android/posenet/download.gradle new file mode 100644 index 00000000000..b8288a4109d --- /dev/null +++ b/lite/examples/posenet/android/posenet/download.gradle @@ -0,0 +1,25 @@ +def targetFile = "src/main/assets/posenet_model.tflite" +def modelFloatDownloadUrl = "https://storage.googleapis.com/download.tensorflow.org/models/tflite/posenet_mobilenet_v1_100_513x513_multi_kpt_stripped.tflite" + +task downloadModelFloat(type: DownloadUrlTask) { + doFirst { + println "Downloading ${modelFloatDownloadUrl}" + } + sourceUrl = "${modelFloatDownloadUrl}" + target = file("${targetFile}") +} + +class DownloadUrlTask extends DefaultTask { + @Input + String sourceUrl + + @OutputFile + File target + + @TaskAction + void download() { + ant.get(src: sourceUrl, dest: target) + } +} + +preBuild.dependsOn downloadModelFloat diff --git a/lite/examples/posenet/android/posenet/proguard-rules.pro b/lite/examples/posenet/android/posenet/proguard-rules.pro new file mode 100644 index 00000000000..f1b424510da --- /dev/null +++ b/lite/examples/posenet/android/posenet/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/lite/examples/posenet/android/posenet/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.java b/lite/examples/posenet/android/posenet/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.java new file mode 100644 index 00000000000..c5df3e8cab0 --- /dev/null +++ b/lite/examples/posenet/android/posenet/src/androidTest/java/org/tensorflow/lite/examples/posenet/ExampleInstrumentedTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.posenet; + +import static org.junit.Assert.*; + +import android.content.Context; +import androidx.test.InstrumentationRegistry; +import androidx.test.runner.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getTargetContext(); + + assertEquals("org.tensorflow.lite.examples.posenet.test", appContext.getPackageName()); + } +} diff --git a/lite/examples/posenet/android/posenet/src/main/AndroidManifest.xml b/lite/examples/posenet/android/posenet/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..c4b7397c78e --- /dev/null +++ b/lite/examples/posenet/android/posenet/src/main/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + diff --git a/lite/examples/posenet/android/posenet/src/main/java/org/tensorflow/lite/examples/posenet/lib/Posenet.kt b/lite/examples/posenet/android/posenet/src/main/java/org/tensorflow/lite/examples/posenet/lib/Posenet.kt new file mode 100644 index 00000000000..6947e53023b --- /dev/null +++ b/lite/examples/posenet/android/posenet/src/main/java/org/tensorflow/lite/examples/posenet/lib/Posenet.kt @@ -0,0 +1,220 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.tensorflow.lite.examples.posenet.lib + +import android.content.Context +import android.graphics.Bitmap +import java.io.FileInputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.MappedByteBuffer +import java.nio.channels.FileChannel +import org.tensorflow.lite.Interpreter + +enum class BodyPart { + NOSE, + LEFT_EYE, + RIGHT_EYE, + LEFT_EAR, + RIGHT_EAR, + LEFT_SHOULDER, + RIGHT_SHOULDER, + LEFT_ELBOW, + RIGHT_ELBOW, + LEFT_WRIST, + RIGHT_WRIST, + LEFT_HIP, + RIGHT_HIP, + LEFT_KNEE, + RIGHT_KNEE, + LEFT_ANKLE, + RIGHT_ANKLE +} + +class Position { + var x: Int = 0 + var y: Int = 0 +} + +class KeyPoint { + var bodyPart: BodyPart = BodyPart.NOSE + var position: Position = Position() + var score: Float = 0.0f +} + +class Person { + var keyPoints = listOf() + var score: Float = 0.0f +} + +class Posenet(context: Context) { + /** An Interpreter for the TFLite model. */ + private var interpreter: Interpreter? = null + + init { + interpreter = Interpreter(loadModelFile("posenet_model.tflite", context)) + } + + /** + * Scale the image to a byteBuffer of [-1,1] values. + */ + private fun initInputArray(bitmap: Bitmap): ByteBuffer { + val bytesPerChannel = 4 + val inputChannels = 3 + val batchSize = 1 + val inputBuffer = ByteBuffer.allocateDirect( + batchSize * bytesPerChannel * bitmap.height * bitmap.width * inputChannels + ) + inputBuffer.order(ByteOrder.nativeOrder()) + inputBuffer.rewind() + + val mean = 128.0f + val std = 128.0f + for (row in 0 until bitmap.height) { + for (col in 0 until bitmap.width) { + val pixelValue = bitmap.getPixel(col, row) + inputBuffer.putFloat(((pixelValue shr 16 and 0xFF) - mean) / std) + inputBuffer.putFloat(((pixelValue shr 8 and 0xFF) - mean) / std) + inputBuffer.putFloat(((pixelValue and 0xFF) - mean) / std) + } + } + return inputBuffer + } + + /** Preload and memory map the model file, returning a MappedByteBuffer containing the model. */ + private fun loadModelFile(path: String, context: Context): MappedByteBuffer { + val fileDescriptor = context.assets.openFd(path) + val inputStream = FileInputStream(fileDescriptor.fileDescriptor) + return inputStream.channel.map( + FileChannel.MapMode.READ_ONLY, fileDescriptor.startOffset, fileDescriptor.declaredLength + ) + } + + /** + * Initializes an outputMap of 1 * x * y * z FloatArrays for the model processing to populate. + */ + private fun initOutputMap(interpreter: Interpreter): HashMap { + val outputMap = HashMap() + + // 1 * 17 * 17 * 17 contains heatmaps + val heatmapsShape = interpreter.getOutputTensor(0).shape() + outputMap[0] = Array(heatmapsShape[0]) { + Array(heatmapsShape[1]) { + Array(heatmapsShape[2]) { FloatArray(heatmapsShape[3]) } + } + } + + // 1 * 17 * 17 * 34 contains offsets + val offsetsShape = interpreter.getOutputTensor(1).shape() + outputMap[1] = Array(offsetsShape[0]) { + Array(offsetsShape[1]) { Array(offsetsShape[2]) { FloatArray(offsetsShape[3]) } } + } + + // 1 * 17 * 17 * 32 contains forward displacements + val displacementsFwdShape = interpreter.getOutputTensor(2).shape() + outputMap[2] = Array(offsetsShape[0]) { + Array(displacementsFwdShape[1]) { + Array(displacementsFwdShape[2]) { FloatArray(displacementsFwdShape[3]) } + } + } + + // 1 * 17 * 17 * 32 contains backward displacements + val displacementsBwdShape = interpreter.getOutputTensor(3).shape() + outputMap[3] = Array(displacementsBwdShape[0]) { + Array(displacementsBwdShape[1]) { + Array(displacementsBwdShape[2]) { FloatArray(displacementsBwdShape[3]) } + } + } + + return outputMap + } + + /** + * Estimates the pose for a single person. + * args: + * bitmap: image bitmap of frame that should be processed + * returns: + * person: a Person object containing data about keypoint locations and confidence scores + */ + fun estimateSinglePose(bitmap: Bitmap): Person { + val inputArray = arrayOf(initInputArray(bitmap)) + val outputMap = initOutputMap(interpreter!!) + interpreter!!.runForMultipleInputsOutputs(inputArray, outputMap) + + val heatmaps = outputMap[0] as Array>> + val offsets = outputMap[1] as Array>> + + val height = heatmaps[0].size + val width = heatmaps[0][0].size + val numKeypoints = heatmaps[0][0][0].size + + // Finds the (row, col) locations of where the keypoints are most likely to be. + val keypointPositions = Array(numKeypoints) { Pair(0, 0) } + for (keypoint in 0 until numKeypoints) { + var maxVal = heatmaps[0][0][0][keypoint] + var maxRow = 0 + var maxCol = 0 + for (row in 0 until height) { + for (col in 0 until width) { + if (heatmaps[0][row][col][keypoint] > maxVal) { + maxVal = heatmaps[0][row][col][keypoint] + maxRow = row + maxCol = col + } + } + } + keypointPositions[keypoint] = Pair(maxRow, maxCol) + } + + // Calculating the x and y coordinates of the keypoints with offset adjustment. + val xCoords = IntArray(numKeypoints) + val yCoords = IntArray(numKeypoints) + val confidenceScores = FloatArray(numKeypoints) + keypointPositions.forEachIndexed { idx, position -> + val positionY = keypointPositions[idx].first + val positionX = keypointPositions[idx].second + yCoords[idx] = ( + position.first / (height - 1).toFloat() * bitmap.height + + offsets[0][positionY][positionX][idx] + ).toInt() + xCoords[idx] = ( + position.second / (width - 1).toFloat() * bitmap.width + + offsets[0][positionY] + [positionX][idx + numKeypoints] + ).toInt() + confidenceScores[idx] = + ( + (heatmaps[0][positionY][positionX][idx]) / 10 + ) + } + + val person = Person() + val keypointList = Array(numKeypoints) { KeyPoint() } + var totalScore = 0.0f + enumValues().forEachIndexed { idx, it -> + keypointList[idx].bodyPart = it + keypointList[idx].position.x = xCoords[idx] + keypointList[idx].position.y = yCoords[idx] + keypointList[idx].score = confidenceScores[idx] + totalScore += confidenceScores[idx] + } + + person.keyPoints = keypointList.toList() + person.score = totalScore / numKeypoints + + return person + } +} diff --git a/lite/examples/posenet/android/posenet/src/main/res/values/strings.xml b/lite/examples/posenet/android/posenet/src/main/res/values/strings.xml new file mode 100644 index 00000000000..c7831599b73 --- /dev/null +++ b/lite/examples/posenet/android/posenet/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + posenet + diff --git a/lite/examples/posenet/android/posenetimage.png b/lite/examples/posenet/android/posenetimage.png new file mode 100644 index 00000000000..3f0722954fb Binary files /dev/null and b/lite/examples/posenet/android/posenetimage.png differ diff --git a/lite/examples/posenet/android/settings.gradle b/lite/examples/posenet/android/settings.gradle new file mode 100644 index 00000000000..2bf3052a6b0 --- /dev/null +++ b/lite/examples/posenet/android/settings.gradle @@ -0,0 +1 @@ +include ':app', ':posenet'