Python Pybind11 wrapper (CMU-Perceptual-Computing-Lab#1014)

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "3rdparty/caffe"]
 	path = 3rdparty/caffe
 	url = https://github.com/CMU-Perceptual-Computing-Lab/caffe.git
+[submodule "3rdparty/pybind11"]
+	path = 3rdparty/pybind11
+	url = https://github.com/pybind/pybind11.git

CMakeLists.txt

@@ -639,7 +639,7 @@ if (UNIX OR APPLE)
       file(GLOB CAFFE_DIR_VALID ${CMAKE_SOURCE_DIR}/3rdparty/caffe/*)
       list(LENGTH CAFFE_DIR_VALID CAFFE_DIR_VALID_LENGTH)
       if (CAFFE_DIR_VALID_LENGTH EQUAL 0)
-        execute_process(COMMAND git submodule update --init --recursive)
+        execute_process(COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/3rdparty/caffe)
       else (CAFFE_DIR_VALID_LENGTH EQUAL 0)
         message(STATUS "Caffe has already been downloaded.")
       endif (CAFFE_DIR_VALID_LENGTH EQUAL 0)
@@ -713,7 +713,7 @@ if (UNIX OR APPLE)
           -DCPU_ONLY=${CAFFE_CPU_ONLY}
           -DCMAKE_BUILD_TYPE=Release
           -DBUILD_docs=OFF
-          -DBUILD_python=${BUILD_PYTHON}
+          -DBUILD_python=OFF
           -DBUILD_python_layer=OFF
           -DUSE_LEVELDB=OFF
           -DUSE_LMDB=OFF
@@ -731,7 +731,7 @@ if (UNIX OR APPLE)
           -DCPU_ONLY=${CAFFE_CPU_ONLY}
           -DCMAKE_BUILD_TYPE=Release
           -DBUILD_docs=OFF
-          -DBUILD_python=${BUILD_PYTHON}
+          -DBUILD_python=OFF
           -DBUILD_python_layer=OFF
           -DUSE_LEVELDB=OFF
           -DUSE_LMDB=OFF
@@ -912,12 +912,24 @@ download_model("hand" ${DOWNLOAD_HAND_MODEL} hand/pose_iter_102000.caffemodel
 
 message(STATUS "Models Downloaded.")
 
+
 ### PYTHON
-if (Caffe_FOUND)
-  if(BUILD_PYTHON)
+if (BUILD_PYTHON)
+  if (WIN32)
+    execute_process(COMMAND cmd /c cd ${CMAKE_SOURCE_DIR} & git submodule update --init 3rdparty/pybind11/)
+    add_subdirectory(3rdparty/pybind11)
     add_subdirectory(python)
-  endif (BUILD_PYTHON)
-endif(Caffe_FOUND)
+  elseif (UNIX OR APPLE)
+    if (Caffe_FOUND)
+      execute_process(COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/3rdparty/pybind11/)
+      add_subdirectory(3rdparty/pybind11)
+      add_subdirectory(python)
+    endif (Caffe_FOUND)
+  else (WIN32)
+    message(FATAL_ERROR "Unknown OS.")
+  endif (WIN32)
+endif (BUILD_PYTHON)
+
 
 ### GENERATE DOCUMENTATION
 if (UNIX OR APPLE)

doc/modules/python_module.md

@@ -11,14 +11,14 @@ OpenPose Python Module and Demo
 
 
 ## Introduction
-This experimental module exposes a Python API for OpenPose. This allows you to construct an OpenPose object, pass in a numpy array for an image, and get a numpy array of the pose positions. This API also exposes an API that allows you to directly pass in heatmaps from a network and extract poses out of it (Requires Python Caffe to be installed seperately)
-
-At present the Python API only supports body pose. Hands and Face will be added in the future.
+This module exposes a Python API for OpenPose. It is effectively a wrapper that replicates most of the functionality of the [op::Wrapper class](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/include/openpose/wrapper/wrapper.hpp) and allows you to populate and retrieve data from the [op::Datum class](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/include/openpose/core/datum.hpp) using standard Python and Numpy constructs.
 
 
 
 ## Compatibility
-The OpenPose Python module is compatible with both Python 2 and Python 3. In addition, it will also run in all OpenPose compatible operating systems.
+The OpenPose Python module is compatible with both Python 2 and Python 3. In addition, it will also run in all OpenPose compatible operating systems. It uses [Pybind11](https://github.com/pybind/pybind11) for mapping between C++ and Python datatypes.
+
+To compile, enable `BUILD_PYTHON` in cmake. Pybind selects the latest version of Python by default (Python 3). To use Python 2, change `PYTHON_EXECUTABLE` and `PYTHON_LIBRARY` flags in cmake to your desired python version.
 
 
 
@@ -28,28 +28,24 @@ Check [doc/installation.md#python-module](../installation.md#python-api) for ins
 The Python API requires Numpy for array management, and OpenCV for image loading. They can be installed via:
 
 ```
-pip install numpy
-pip install opencv-python
+pip install numpy opencv-python
 ```
 
 
 
 ## Testing
-Two examples can be found in `build/examples/tutorial_api_python` in your build folder. Navigate directly to this path to run examples.
-
-- `1_extract_pose` demonstrates a simple use of the API.
-- `2_pose_from_heatmaps` demonstrates constructing pose from heatmaps from the caffe network (Requires Python Caffe to be installed seperately, only tested on Ubuntu).
+All the Python examples from the Tutorial API Python module can be found in `build/examples/tutorial_api_python` in your build folder. Navigate directly to this path to run examples.
 
 ```
 # From command line
 cd build/examples/tutorial_api_python
-python 1_extract_pose.py
+python3 1_body_from_image.py
 ```
 
 
 
 ## Exporting Python OpenPose
 Note: This step is only required if you are moving the `*.py` files outside their original location, or writting new `*.py` scripts outside `build/examples/tutorial_api_python`.
 
-- Option a, installing OpenPose: On an Ubuntu or OSX based system, you could install OpenPose by running `sudo make install`, you could then set the OpenPose path in your python scripts to the OpenPose installation path (default: `/usr/local/python`) and start using OpenPose at any location. Take a look at `build/examples/tutorial_pose/1_extract_pose.py` for an example.
-- Option b, not installing OpenPose: To move the OpenPose Python API demos to a different folder, ensure that the line `sys.path.append('{OpenPose_path}/python')` is properly set in your `*.py` files, where `{OpenPose_path}` points to your build folder of OpenPose. Take a look at `build/examples/tutorial_pose/1_extract_pose.py` for an example.
+- Option a, installing OpenPose: On an Ubuntu or OSX based system, you could install OpenPose by running `sudo make install`, you could then set the OpenPose path in your python scripts to the OpenPose installation path (default: `/usr/local/python`) and start using OpenPose at any location. Take a look at `build/examples/tutorial_pose/1_body_from_image.py` for an example.
+- Option b, not installing OpenPose: To move the OpenPose Python API demos to a different folder, ensure that the line `sys.path.append('{OpenPose_path}/python')` is properly set in your `*.py` files, where `{OpenPose_path}` points to your build folder of OpenPose. Take a look at `build/examples/tutorial_pose/1_body_from_image.py` for an example.

examples/tutorial_api_python/1_body_from_image.py

@@ -0,0 +1,67 @@
+# From Python
+# It requires OpenCV installed for Python
+import sys
+import cv2
+import os
+from sys import platform
+import argparse
+
+# Import Openpose (Windows/Ubuntu/OSX)
+dir_path = os.path.dirname(os.path.realpath(__file__))
+try:
+    # Windows Import
+    if platform == "win32":
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append(dir_path + '/../../python/openpose/Release');
+        os.environ['PATH']  = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' +  dir_path + '/../../bin;'
+        import _openpose as op
+    else:
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append('../../python');
+        # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
+        # sys.path.append('/usr/local/python')
+        from openpose import openpose as op
+except:
+    raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?')
+
+# Flags
+parser = argparse.ArgumentParser()
+parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).")
+args = parser.parse_known_args()
+
+# Custom Params (refer to include/openpose/flags.hpp for more parameters)
+params = dict()
+params["model_folder"] = "../../../models/"
+
+# Add others in path?
+for i in range(0, len(args[1])):
+    curr_item = args[1][i]
+    if i != len(args[1])-1: next_item = args[1][i+1]
+    else: next_item = "1"
+    if "--" in curr_item and "--" in next_item:
+        key = curr_item.replace('-','')
+        if key not in params:  params[key] = "1"
+    elif "--" in curr_item and "--" not in next_item:
+        key = curr_item.replace('-','')
+        if key not in params: params[key] = next_item
+
+# Construct it from system arguments
+# op.init_argv(args[1])
+# oppython = op.OpenposePython()
+
+# Starting OpenPose
+opWrapper = op.WrapperPython()
+opWrapper.configure(params)
+opWrapper.start()
+
+# Process Image
+datum = op.Datum()
+imageToProcess = cv2.imread(args[0].image_path)
+datum.cvInputData = imageToProcess
+opWrapper.emplaceAndPop([datum])
+
+# Display Image
+print("Body keypoints: \n" + str(datum.poseKeypoints))
+while 1:
+    cv2.imshow("win", datum.cvOutputData)
+    cv2.waitKey(15)

examples/tutorial_api_python/2_whole_body_from_image.py

@@ -0,0 +1,72 @@
+# From Python
+# It requires OpenCV installed for Python
+import sys
+import cv2
+import os
+from sys import platform
+import argparse
+
+# Import Openpose (Windows/Ubuntu/OSX)
+dir_path = os.path.dirname(os.path.realpath(__file__))
+try:
+    # Windows Import
+    if platform == "win32":
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append(dir_path + '/../../python/openpose/Release');
+        os.environ['PATH']  = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' +  dir_path + '/../../bin;'
+        import _openpose as op
+    else:
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append('../../python');
+        # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
+        # sys.path.append('/usr/local/python')
+        from openpose import openpose as op
+except:
+    raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?')
+
+# Flags
+parser = argparse.ArgumentParser()
+parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000241.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).")
+args = parser.parse_known_args()
+
+# Custom Params (refer to include/openpose/flags.hpp for more parameters)
+params = dict()
+params["model_folder"] = "../../../models/"
+params["face"] = True
+params["hand"] = True
+
+# Add others in path?
+for i in range(0, len(args[1])):
+    curr_item = args[1][i]
+    if i != len(args[1])-1: next_item = args[1][i+1]
+    else: next_item = "1"
+    if "--" in curr_item and "--" in next_item:
+        key = curr_item.replace('-','')
+        if key not in params:  params[key] = "1"
+    elif "--" in curr_item and "--" not in next_item:
+        key = curr_item.replace('-','')
+        if key not in params: params[key] = next_item
+
+# Construct it from system arguments
+# op.init_argv(args[1])
+# oppython = op.OpenposePython()
+
+# Starting OpenPose
+opWrapper = op.WrapperPython()
+opWrapper.configure(params)
+opWrapper.start()
+
+# Process Image
+datum = op.Datum()
+imageToProcess = cv2.imread(args[0].image_path)
+datum.cvInputData = imageToProcess
+opWrapper.emplaceAndPop([datum])
+
+# Display Image
+print("Body keypoints: \n" + str(datum.poseKeypoints))
+print("Face keypoints: \n" + str(datum.faceKeypoints))
+print("Left hand keypoints: \n" + str(datum.handKeypoints[0]))
+print("Right hand keypoints: \n" + str(datum.handKeypoints[1]))
+while 1:
+    cv2.imshow("win", datum.cvOutputData)
+    cv2.waitKey(15)

examples/tutorial_api_python/3_heatmaps_from_image.py

@@ -0,0 +1,83 @@
+# From Python
+# It requires OpenCV installed for Python
+import sys
+import cv2
+import os
+from sys import platform
+import argparse
+import numpy as np
+
+# Import Openpose (Windows/Ubuntu/OSX)
+dir_path = os.path.dirname(os.path.realpath(__file__))
+try:
+    # Windows Import
+    if platform == "win32":
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append(dir_path + '/../../python/openpose/Release');
+        os.environ['PATH']  = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' +  dir_path + '/../../bin;'
+        import _openpose as op
+    else:
+        # Change these variables to point to the correct folder (Release/x64 etc.) 
+        sys.path.append('../../python');
+        # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
+        # sys.path.append('/usr/local/python')
+        from openpose import openpose as op
+except:
+    raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?')
+
+# Flags
+parser = argparse.ArgumentParser()
+parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).")
+args = parser.parse_known_args()
+
+# Custom Params (refer to include/openpose/flags.hpp for more parameters)
+params = dict()
+params["model_folder"] = "../../../models/"
+params["heatmaps_add_parts"] = True
+params["heatmaps_add_PAFs"] = True
+
+# Add others in path?
+for i in range(0, len(args[1])):
+    curr_item = args[1][i]
+    if i != len(args[1])-1: next_item = args[1][i+1]
+    else: next_item = "1"
+    if "--" in curr_item and "--" in next_item:
+        key = curr_item.replace('-','')
+        if key not in params:  params[key] = "1"
+    elif "--" in curr_item and "--" not in next_item:
+        key = curr_item.replace('-','')
+        if key not in params: params[key] = next_item
+
+# Construct it from system arguments
+# op.init_argv(args[1])
+# oppython = op.OpenposePython()
+
+# Starting OpenPose
+opWrapper = op.WrapperPython()
+opWrapper.configure(params)
+opWrapper.start()
+
+# Process Image
+datum = op.Datum()
+imageToProcess = cv2.imread(args[0].image_path)
+datum.cvInputData = imageToProcess
+opWrapper.emplaceAndPop([datum])
+
+# Process outputs
+outputImageF = (datum.inputNetData[0].copy())[0,:,:,:] + 0.5
+outputImageF = cv2.merge([outputImageF[0,:,:], outputImageF[1,:,:], outputImageF[2,:,:]])
+outputImageF = (outputImageF*255.).astype(dtype='uint8')
+heatmaps = datum.poseHeatMaps.copy()
+heatmaps = (heatmaps*255.).astype(dtype='uint8')
+
+# Display Image
+counter = 0
+while 1:
+    num_maps = heatmaps.shape[0]
+    heatmap = heatmaps[counter, :, :].copy()
+    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
+    combined = cv2.addWeighted(outputImageF, 0.5, heatmap, 0.5, 0)
+    cv2.imshow("win", combined)
+    cv2.waitKey(-1)
+    counter += 1
+    counter = counter % num_maps

examples/tutorial_api_python/CMakeLists.txt

@@ -1,2 +1,5 @@
 ### Add Python Test
-configure_file(1_extract_pose.py 1_extract_pose.py)
+configure_file(openpose_python.py openpose_python.py)
+configure_file(1_body_from_image.py 1_body_from_image.py)
+configure_file(2_whole_body_from_image.py 2_whole_body_from_image.py)
+configure_file(3_heatmaps_from_image.py 3_heatmaps_from_image.py)