openvino: major WIP on ability to use IE backend

Signed-off-by: Ron Evans <[email protected]>

cmd/caffe-classifier/main.go

@@ -14,7 +14,12 @@
 //
 // How to run:
 //
-// 		go run ./cmd/caffe-classifier/main.go 0 [protofile] [modelfile] [descriptionsfile]
+// 		go run ./cmd/caffe-classifier/main.go 0 [modelfile] [configfile] [descriptionsfile] [backend] [device]
+//
+//
+// You can also use this sample with the Intel OpenVINO Inference Engine, if you have it installed.
+//
+// 		go run ./cmd/caffe-classifier/main.go 0 [modelfile] [configfile] [descriptionsfile] [backend] [device]
 //
 // +build example
 
@@ -49,7 +54,7 @@ func readDescriptions(path string) ([]string, error) {
 
 func main() {
 	if len(os.Args) < 5 {
-		fmt.Println("How to run:\ncaffe-classifier [camera ID] [protofile] [modelfile] [descriptionsfile]")
+		fmt.Println("How to run:\ncaffe-classifier [camera ID] [modelfile] [configfile] [descriptionsfile] ([backend] [device])")
 		return
 	}
 
@@ -64,6 +69,16 @@ func main() {
 		return
 	}
 
+	backend := gocv.NetBackendDefault
+	if len(os.Args) > 5 {
+		backend = gocv.ParseNetBackend(os.Args[5])
+	}
+
+	target := gocv.NetTargetCPU
+	if len(os.Args) > 6 {
+		target = gocv.ParseNetTarget(os.Args[6])
+	}
+
 	// open capture device
 	webcam, err := gocv.OpenVideoCapture(deviceID)
 	if err != nil {
@@ -79,12 +94,14 @@ func main() {
 	defer img.Close()
 
 	// open DNN classifier
-	net := gocv.ReadNetFromCaffe(proto, model)
+	net := gocv.ReadNet(model, proto)
 	if net.Empty() {
-		fmt.Printf("Error reading network model from : %v %v\n", proto, model)
+		fmt.Printf("Error reading network model from : %v %v\n", model, proto)
 		return
 	}
 	defer net.Close()
+	net.SetPreferableBackend(gocv.NetBackendType(backend))
+	net.SetPreferableTarget(gocv.NetTargetType(target))
 
 	status := "Ready"
 	statusColor := color.RGBA{0, 255, 0, 0}
@@ -103,10 +120,10 @@ func main() {
 		blob := gocv.BlobFromImage(img, 1.0, image.Pt(224, 224), gocv.NewScalar(104, 117, 123, 0), false, false)
 
 		// feed the blob into the classifier
-		net.SetInput(blob, "data")
+		net.SetInput(blob, "")
 
 		// run a forward pass thru the network
-		prob := net.Forward("prob")
+		prob := net.Forward("")
 
 		// reshape the results into a 1x1000 matrix
 		probMat := prob.Reshape(1, 1)

core.go

@@ -1338,3 +1338,15 @@ func toCPoints(points []image.Point) C.struct_Points {
 		length: C.int(len(points)),
 	}
 }
+
+func toCStrings(strs []string) C.struct_CStrings {
+	cStringsSlice := make([]*C.char, len(strs))
+	for i, s := range strs {
+		cStringsSlice[i] = C.CString(s)
+	}
+
+	return C.struct_CStrings{
+		strs:   (**C.char)(&cStringsSlice[0]),
+		length: C.int(len(strs)),
+	}
+}

core.h

@@ -4,10 +4,11 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-typedef struct String {
-    const char* str;
+// Wrapper for std::vector<string>
+typedef struct CStrings {
+    const char** strs;
     int length;
-} String;
+} CStrings;
 
 typedef struct ByteArray {
     char* data;

core_test.go

@@ -692,6 +692,20 @@ func TestToCPoints(t *testing.T) {
 	}
 }
 
+func TestToCStrings(t *testing.T) {
+	strs := []string{
+		"hello",
+		"fellow",
+		"CStrings",
+	}
+
+	cStrs := toCStrings(strs)
+
+	if int(cStrs.length) != len(strs) {
+		t.Error("Invalid CStrings length")
+	}
+}
+
 func TestMatBatchDistance(t *testing.T) {
 	src1 := NewMatWithSize(100, 100, MatTypeCV8U)
 	src2 := NewMatWithSize(100, 100, MatTypeCV8U)

dnn.cpp

@@ -31,6 +31,19 @@ Mat Net_Forward(Net net, const char* outputName) {
     return new cv::Mat(net->forward(outputName));
 }
 
+void Net_ForwardLayers(Net net, struct Mats outputBlobs, struct CStrings outBlobNames) {
+    std::vector< cv::Mat > blobs;
+    for (int i = 0; i < outputBlobs.length; ++i) {
+        blobs.push_back(*outputBlobs.mats[i]);
+    }
+
+    std::vector< cv::String > names;
+    for (int i = 0; i < outBlobNames.length; ++i) {
+        names.push_back(cv::String(outBlobNames.strs[i]));
+    }
+    net->forward(blobs, names);
+}
+
 void Net_SetPreferableBackend(Net net, int backend) {
     net->setPreferableBackend(backend);
 }

dnn.go

@@ -37,6 +37,24 @@ const (
 	NetBackendOpenCV NetBackendType = 3
 )
 
+// ParseNetBackend returns a valid NetBackendType given a string. Valid values are:
+// - halide
+// - openvino
+// - opencv
+// - default
+func ParseNetBackend(backend string) NetBackendType {
+	switch backend {
+	case "halide":
+		return NetBackendHalide
+	case "openvino":
+		return NetBackendOpenVINO
+	case "opencv":
+		return NetBackendOpenCV
+	default:
+		return NetBackendDefault
+	}
+}
+
 // NetTargetType is the type for the various different kinds of DNN device targets.
 type NetTargetType int
 
@@ -45,7 +63,7 @@ const (
 	NetTargetCPU NetTargetType = 0
 
 	// NetBackendFP32 is the 32-bit OpenCL backend.
-	NetBackendOpenCL NetTargetType = 1
+	NetBackendFP32 NetTargetType = 1
 
 	// NetBackendFP16 is the 16-bit OpenCL backend.
 	NetBackendFP16 NetTargetType = 2
@@ -54,6 +72,26 @@ const (
 	NetBackendVPU NetTargetType = 3
 )
 
+// ParseNetTarget returns a valid NetTargetType given a string. Valid values are:
+// - cpu
+// - fp32
+// - fp16
+// - vpu
+func ParseNetTarget(target string) NetTargetType {
+	switch target {
+	case "cpu":
+		return NetTargetCPU
+	case "fp32":
+		return NetBackendFP32
+	case "fp16":
+		return NetBackendFP16
+	case "vpu":
+		return NetBackendVPU
+	default:
+		return NetTargetCPU
+	}
+}
+
 // Close Net
 func (net *Net) Close() error {
 	C.Net_Close((C.Net)(net.p))
@@ -94,6 +132,24 @@ func (net *Net) Forward(outputName string) Mat {
 	return Mat{p: C.Net_Forward((C.Net)(net.p), cName)}
 }
 
+// ForwardLayers forward pass to compute outputs of layers listed in outBlobNames.
+//
+// For further details, please see:
+// https://docs.opencv.org/3.4.1/db/d30/classcv_1_1dnn_1_1Net.html#adb34d7650e555264c7da3b47d967311b
+//
+func (net *Net) ForwardLayers(outputBlobs []Mat, outBlobNames []string) {
+	cMatArray := make([]C.Mat, len(outputBlobs))
+	for i, r := range outputBlobs {
+		cMatArray[i] = r.p
+	}
+	cMats := C.struct_Mats{
+		mats:   (*C.Mat)(&cMatArray[0]),
+		length: C.int(len(outputBlobs)),
+	}
+
+	C.Net_ForwardLayers((C.Net)(net.p), cMats, toCStrings(outBlobNames))
+}
+
 // SetPreferableBackend ask network to use specific computation backend.
 //
 // For further details, please see:

dnn.h

@@ -26,6 +26,7 @@ void Net_Close(Net net);
 bool Net_Empty(Net net);
 void Net_SetInput(Net net, Mat blob, const char* name);
 Mat Net_Forward(Net net, const char* outputName);
+void Net_ForwardLayers(Net net, struct Mats outputBlobs, struct CStrings outBlobNames);
 void Net_SetPreferableBackend(Net net, int backend);
 void Net_SetPreferableTarget(Net net, int target);
 

imgproc.cpp

@@ -263,7 +263,7 @@ void Rectangle(Mat img, Rect r, Scalar color, int thickness) {
         cv::Point(r.x + r.width, r.y + r.height),
         c,
         thickness,
-        CV_AA
+        cv::LINE_AA
     );
 }
 

openvino/ie/README.md

@@ -4,16 +4,27 @@ The Intel OpenVINO Inference Engine is a set of libraries for executing convolut
 
 GoCV support for the Intel OpenVINO Inference Engine will be able to be found here in the "gocv.io/x/gocv/openvino/ie" package.
 
-## Roadmap
-
-Support in GoCV for the Intel OpenVINO Inference Engine is still under development, and is not yet in a usable state.
-
-- [ ] InferenceEnginePlugin - work started, can load plugin
-- [X] InferenceEngine::CNNNetReader
-- [ ] InferenceEngine::CNNNetwork  - work started
-- [ ] InferenceEngine::InputsDataMap
-- [ ] InferenceEngine::InputInfo
-- [ ] InferenceEngine::OutputsDataMap
-- [ ] InferenceEngine::ExecutableNetwork
-- [ ] InferenceEngine::InferRequest
-- [ ] InferenceEngine::Blob
+## How It Works
+
+Support in GoCV for the Intel OpenVINO Inference Engine requires version R2+ in order to work. Sinec that version is not yet released, you can install OpemVINO version R1, obtain OpenCV 3.4.2-dev and then compile as described here:
+
+https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend
+
+You will likely need to specify the modules to be pre-loaded in order to use the OpenVINO IE:
+
+    LD_PRELOAD=/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/external/mkltiny_lnx/lib/libiomp5.so:/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/external/cldnn/lib/libclDNN64.so
+
+## How to use
+
+This loads a Caffe model, and then uses OpenVINO inference engine to prepare it for execution on the GPU:
+
+```go
+net := gocv.ReadNet("/path/to/your/model.caffemodel", "/path/to/your/config.proto")
+if net.Empty() {
+    fmt.Println("Error reading network model")
+    return
+}
+
+net.SetPreferableBackend(gocv.NetBackendType("openvino"))
+net.SetPreferableTarget(gocv.NetTargetType("fp16"))
+```