Skip to content

Commit

Permalink
add iOS nanodet
Browse files Browse the repository at this point in the history
  • Loading branch information
teng committed Mar 15, 2021
1 parent 485436d commit 6c91a3d
Show file tree
Hide file tree
Showing 175 changed files with 92,537 additions and 38 deletions.
9 changes: 5 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
| model | android | iOS | from |
|-------------------|:--------:|:--------:|:--------:|
| YOLOv5s | yes | yes | [Github](https://github.com/ultralytics/yolov5) |
| NanoDet | yes | todo | [Github](https://github.com/RangiLyu/nanodet) |
| NanoDet | yes | yes | [Github](https://github.com/RangiLyu/nanodet) |

### iOS:
- 如果缺少模型请从 "android_TNN_Demo\app\src\main\assets" 复制 .tnnproto 和 .tnnmodel 文件到 "iOS_TNN_Demo\TNNDemo\res" 下。
Expand All @@ -42,14 +42,15 @@

由于TNN官方还处于开发阶段,不同时间版本可能会出现功能异常或速度差距比较大都是正常的(当前版本功能正常,但速度变慢了)。

懒人本地转换(不会上传模型): [xxxx -> tnn] https://convertmodel.com/
懒人本地转换(不会上传模型): [xxxx -> tnn](https://convertmodel.com/)

轻量级OpenCV:[opencv-mobile](https://github.com/nihui/opencv-mobile)

:art: 截图<br/>

| Android | iOS |
|:-----:|:-----:|
|<img width="324" height="145" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/Android_CPU_or_GPU.jpg"/>| None |
|<img width="324" height="145" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/Android_CPU_or_GPU.jpg"/>| <img width="320" height="166" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/iOS_CPU_or_GPU.jpg"/> |

> Android
Expand All @@ -61,7 +62,7 @@
| YOLOv5s | NanoDet |
|---------|---------|
| <img width="270" height="480" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/iOS_iPhone6sp_yolov5s_gpu.jpg"/> | None |
| <img width="270" height="480" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/iOS_iPhone6sp_yolov5s_gpu.jpg"/> | <img width="270" height="480" src="https://github.com/cmdbug/TNN_Demo/blob/main/Screenshots/iOS_iPhone5s_nanodet.jpg"/> |


感谢:<br/>
Expand Down
Binary file added Screenshots/iOS_CPU_or_GPU.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added Screenshots/iOS_iPhone5s_nanodet.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
42 changes: 34 additions & 8 deletions iOS_TNN_Demo/TNNDemo.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@
5F49BA2525ABF5A10049C8A3 /* WelcomeVC.m in Sources */ = {isa = PBXBuildFile; fileRef = 5F49BA2425ABF5A10049C8A3 /* WelcomeVC.m */; };
5F49BA2B25ABF6640049C8A3 /* ELCameraBaseCapture.m in Sources */ = {isa = PBXBuildFile; fileRef = 5F49BA2825ABF6640049C8A3 /* ELCameraBaseCapture.m */; };
5F49BA2C25ABF6640049C8A3 /* ELCameraControlCapture.m in Sources */ = {isa = PBXBuildFile; fileRef = 5F49BA2925ABF6640049C8A3 /* ELCameraControlCapture.m */; };
5F7F95E025AC2A340059E65F /* tnn.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5F7F95DF25AC2A340059E65F /* tnn.framework */; };
5F7F95E225AC2A520059E65F /* tnn.bundle in Resources */ = {isa = PBXBuildFile; fileRef = 5F7F95E125AC2A510059E65F /* tnn.bundle */; };
5F5498A025FB73C5004A72DC /* opencv2.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5F54989F25FB73C5004A72DC /* opencv2.framework */; };
5F7799EE25FB3FBD004A53A4 /* nanodet_sim_opt.tnnmodel in Resources */ = {isa = PBXBuildFile; fileRef = 5F7799EC25FB3FBD004A53A4 /* nanodet_sim_opt.tnnmodel */; };
5F7799EF25FB3FBD004A53A4 /* nanodet_sim_opt.tnnproto in Resources */ = {isa = PBXBuildFile; fileRef = 5F7799ED25FB3FBD004A53A4 /* nanodet_sim_opt.tnnproto */; };
5F7799F325FB3FDB004A53A4 /* NanoDet.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5F7799F225FB3FDB004A53A4 /* NanoDet.mm */; };
5F7F95E525AC2B510059E65F /* CoreML.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5F7F95E425AC2B510059E65F /* CoreML.framework */; };
5F7F95EF25AC33B10059E65F /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5F7F95EE25AC33B10059E65F /* Accelerate.framework */; };
5FE9786A25FEE7D200D8DF1C /* tnn.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5FE9786925FEE7D200D8DF1C /* tnn.framework */; };
5FE9786C25FEE7D900D8DF1C /* tnn.bundle in Resources */ = {isa = PBXBuildFile; fileRef = 5FE9786B25FEE7D900D8DF1C /* tnn.bundle */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
Expand All @@ -49,10 +53,15 @@
5F49BA2825ABF6640049C8A3 /* ELCameraBaseCapture.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ELCameraBaseCapture.m; sourceTree = "<group>"; };
5F49BA2925ABF6640049C8A3 /* ELCameraControlCapture.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ELCameraControlCapture.m; sourceTree = "<group>"; };
5F49BA2A25ABF6640049C8A3 /* ELCameraBaseCapture.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ELCameraBaseCapture.h; sourceTree = "<group>"; };
5F7F95DF25AC2A340059E65F /* tnn.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = tnn.framework; sourceTree = "<group>"; };
5F7F95E125AC2A510059E65F /* tnn.bundle */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.plug-in"; path = tnn.bundle; sourceTree = "<group>"; };
5F54989F25FB73C5004A72DC /* opencv2.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = opencv2.framework; sourceTree = "<group>"; };
5F7799EC25FB3FBD004A53A4 /* nanodet_sim_opt.tnnmodel */ = {isa = PBXFileReference; lastKnownFileType = file; path = nanodet_sim_opt.tnnmodel; sourceTree = "<group>"; };
5F7799ED25FB3FBD004A53A4 /* nanodet_sim_opt.tnnproto */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = nanodet_sim_opt.tnnproto; sourceTree = "<group>"; };
5F7799F125FB3FDB004A53A4 /* NanoDet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NanoDet.h; sourceTree = "<group>"; };
5F7799F225FB3FDB004A53A4 /* NanoDet.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = NanoDet.mm; sourceTree = "<group>"; };
5F7F95E425AC2B510059E65F /* CoreML.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreML.framework; path = System/Library/Frameworks/CoreML.framework; sourceTree = SDKROOT; };
5F7F95EE25AC33B10059E65F /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; };
5FE9786925FEE7D200D8DF1C /* tnn.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = tnn.framework; sourceTree = "<group>"; };
5FE9786B25FEE7D900D8DF1C /* tnn.bundle */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.plug-in"; path = tnn.bundle; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
Expand All @@ -61,8 +70,9 @@
buildActionMask = 2147483647;
files = (
5F7F95EF25AC33B10059E65F /* Accelerate.framework in Frameworks */,
5FE9786A25FEE7D200D8DF1C /* tnn.framework in Frameworks */,
5F7F95E525AC2B510059E65F /* CoreML.framework in Frameworks */,
5F7F95E025AC2A340059E65F /* tnn.framework in Frameworks */,
5F5498A025FB73C5004A72DC /* opencv2.framework in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand All @@ -72,6 +82,8 @@
5F46C48D25ABFBF60051737C /* res */ = {
isa = PBXGroup;
children = (
5F7799EC25FB3FBD004A53A4 /* nanodet_sim_opt.tnnmodel */,
5F7799ED25FB3FBD004A53A4 /* nanodet_sim_opt.tnnproto */,
5F46C48F25ABFC090051737C /* yolov5s_sim_opt.tnnmodel */,
5F46C48E25ABFC090051737C /* yolov5s_sim_opt.tnnproto */,
);
Expand All @@ -98,8 +110,9 @@
5F49BA0925ABF3730049C8A3 /* TNNDemo */ = {
isa = PBXGroup;
children = (
5F7F95E125AC2A510059E65F /* tnn.bundle */,
5F7F95DF25AC2A340059E65F /* tnn.framework */,
5FE9786B25FEE7D900D8DF1C /* tnn.bundle */,
5FE9786925FEE7D200D8DF1C /* tnn.framework */,
5F54989F25FB73C5004A72DC /* opencv2.framework */,
5F46C48D25ABFBF60051737C /* res */,
5F49BA2625ABF6640049C8A3 /* CameraCapture */,
5F49BA0A25ABF3730049C8A3 /* AppDelegate.h */,
Expand All @@ -115,6 +128,8 @@
5F49BA2425ABF5A10049C8A3 /* WelcomeVC.m */,
5F46C49625AC03A20051737C /* UIImage+Utility.h */,
5F46C49525AC03A20051737C /* UIImage+Utility.mm */,
5F7799F125FB3FDB004A53A4 /* NanoDet.h */,
5F7799F225FB3FDB004A53A4 /* NanoDet.mm */,
5F46C49325ABFC500051737C /* Yolov5.mm */,
5F46C49225ABFC500051737C /* Yolov5.h */,
);
Expand Down Expand Up @@ -200,10 +215,12 @@
files = (
5F49BA1A25ABF37C0049C8A3 /* LaunchScreen.storyboard in Resources */,
5F49BA1725ABF37C0049C8A3 /* Assets.xcassets in Resources */,
5F7F95E225AC2A520059E65F /* tnn.bundle in Resources */,
5FE9786C25FEE7D900D8DF1C /* tnn.bundle in Resources */,
5F49BA1525ABF3730049C8A3 /* Main.storyboard in Resources */,
5F46C49125ABFC090051737C /* yolov5s_sim_opt.tnnmodel in Resources */,
5F7799EF25FB3FBD004A53A4 /* nanodet_sim_opt.tnnproto in Resources */,
5F46C49025ABFC090051737C /* yolov5s_sim_opt.tnnproto in Resources */,
5F7799EE25FB3FBD004A53A4 /* nanodet_sim_opt.tnnmodel in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand All @@ -218,6 +235,7 @@
5F49BA1225ABF3730049C8A3 /* ViewController.mm in Sources */,
5F46C49725AC03A20051737C /* UIImage+Utility.mm in Sources */,
5F49BA2B25ABF6640049C8A3 /* ELCameraBaseCapture.m in Sources */,
5F7799F325FB3FDB004A53A4 /* NanoDet.mm in Sources */,
5F46C49425ABFC500051737C /* Yolov5.mm in Sources */,
5F49BA0C25ABF3730049C8A3 /* AppDelegate.m in Sources */,
5F49BA1D25ABF37C0049C8A3 /* main.m in Sources */,
Expand Down Expand Up @@ -361,12 +379,15 @@
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = 7RB5UWYX7Z;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
"$(inherited)",
"$(PROJECT_DIR)/TNNDemo",
);
HEADER_SEARCH_PATHS = "";
INFOPLIST_FILE = TNNDemo/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
Expand All @@ -378,6 +399,7 @@
);
PRODUCT_BUNDLE_IDENTIFIER = com.teng.TNNDemo;
PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = "";
TARGETED_DEVICE_FAMILY = "1,2";
VALID_ARCHS = "arm64 arm64e armv7 armv7s";
};
Expand All @@ -387,12 +409,15 @@
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = 7RB5UWYX7Z;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
"$(inherited)",
"$(PROJECT_DIR)/TNNDemo",
);
HEADER_SEARCH_PATHS = "";
INFOPLIST_FILE = TNNDemo/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
Expand All @@ -404,6 +429,7 @@
);
PRODUCT_BUNDLE_IDENTIFIER = com.teng.TNNDemo;
PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = "";
TARGETED_DEVICE_FAMILY = "1,2";
VALID_ARCHS = "arm64 arm64e armv7 armv7s";
};
Expand Down
8 changes: 4 additions & 4 deletions iOS_TNN_Demo/TNNDemo/Info.plist
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSPhotoLibraryUsageDescription</key>
<string>不让用就再见吧</string>
<key>NSCameraUsageDescription</key>
<string>不给就没的玩了</string>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
Expand All @@ -24,6 +20,10 @@
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>NSCameraUsageDescription</key>
<string>不给就没的玩了</string>
<key>NSPhotoLibraryUsageDescription</key>
<string>不让用就再见吧</string>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
<key>UIMainStoryboardFile</key>
Expand Down
103 changes: 103 additions & 0 deletions iOS_TNN_Demo/TNNDemo/NanoDet.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
//
// NanoDet.h
// TNNDemo
//
// Created by WZTENG on 2021/1/11.
// Copyright © 2021 TENG. All rights reserved.
//

#ifndef NANODET_H
#define NANODET_H

#include <stdio.h>
#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#import <UIKit/UIImage.h>
#import <functional>

#include "tnn/core/tnn.h"
#include "tnn/core/blob.h"
#include "tnn/utils/blob_converter.h"
#include "tnn/utils/mat_utils.h"

#include "UIImage+Utility.h"
#include <Metal/Metal.h>

#include "Yolov5.h"

typedef struct {
std::string cls_layer;
std::string dis_layer;
int stride;
} HeadInfo;

//typedef struct {
// float x1;
// float y1;
// float x2;
// float y2;
// float score;
// int label;
//} BoxInfo;

class NanoDet {
public:
NanoDet(bool useGPU);

~NanoDet();

std::vector<BoxInfo> detect(UIImage *image, float threshold, float nms_threshold);

std::vector<std::string> labels{"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
"fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
"elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
"sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
"potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
"microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
"hair drier", "toothbrush"};

private:
void decode_infer(const std::shared_ptr<TNN_NS::Mat>& cls_pred,
const std::shared_ptr<TNN_NS::Mat>& dis_pred,
int stride, float threshold, std::vector<std::vector<BoxInfo>> &results);

BoxInfo disPred2Bbox(const float *&dfl_det, int label, float score, int x, int y, int stride);

void nms(std::vector<BoxInfo> &input_boxes, float NMS_THRESH);

private:
std::shared_ptr<tnn::TNN> net;
std::shared_ptr<TNN_NS::Instance> instance;
TNN_NS::DeviceType device_type;

int in_n = 1;
int in_c = 3;
int in_w = 320;
int in_h = 320;

const int num_class = 80;
const int reg_max = 7;

std::vector<HeadInfo> heads_info{
// cls_pred | dis_pred | stride
{"792", "795", 8},
{"814", "817", 16},
{"836", "839", 32},
};

public:
static NanoDet *detector;
static bool hasGPU;
static bool toUseGPU;
};

template<typename _Tp>
int activation_function_softmax(const _Tp *src, _Tp *dst, int length);

inline float fast_exp(float x);

inline float sigmoid(float x);

#endif //NANODET_H
Loading

0 comments on commit 6c91a3d

Please sign in to comment.