39、使用C++ 调用腾讯开源框架NCNN调用YOLOFast,并实现视频流的自动化的labelme标注json数据

基本思想: 首先使用c++调用了腾讯的开源框架NCNN,然后使用腾讯的rapidjson实现自动化的labelme格式化json生成,实现了自动化识别和标注数据,人工稍微修正存在错误的数据集即可;(可以python调用该静态包,但是比较喜欢c++O(∩_∩)O哈哈~)该工程的源代码:

https://github.com/sxj731533730/Autolabel.git

第一步:首先下载NCNN源代码:https://github.com/Tencent/ncnn

ubuntu@ubuntu:~$ git clone https://github.com/Tencent/ncnn.git
ubuntu@ubuntu:~$ cd ncnn/
ubuntu@ubuntu:~/ncnn$ ./build.sh  

然后在目录/home/ubuntu/ncnn/build-host-gcc-linux/src/ 会生成一个静态包libncnn.a

第二步:下载android版本的对应包:https://github.com/Tencent/ncnn/releases/download/20200916/ncnn-android-lib.zip

将头文件include拖出来,放入clion工程中;同时将上述编译的静态包也拖入到新建的libs文件夹中,目录结构如下:

ubuntu@ubuntu:~/CLionProjects/untitled3$ tree -L 2
.
├── cmake-build-debug
│   ├── CMakeCache.txt
│   ├── CMakeFiles
│   ├── cmake_install.cmake
│   ├── Makefile
│   └── untitled3.cbp
├── CMakeLists.txt
├── include
│   └── ncnn
├── libs
│   └── libncnn.a
└── main.cpp

5 directories, 7 files

修改CmakeLIst.txt文件和对应的代码如下:

cmake_minimum_required(VERSION 3.17)
project(untitled3)


set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp")

include_directories(${CMAKE_SOURCE_DIR}/include)
#导入ncnn
add_library(libncnn STATIC IMPORTED)
set_target_properties(libncnn PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/libs/libncnn.a)
find_package(OpenCV REQUIRED)
set(CMAKE_CXX_STANDARD 11)

add_executable(untitled3 main.cpp)
target_link_libraries(untitled3 libncnn ${OpenCV_LIBS})

修改代码和指定模型路径;参考链接代码链接:https://blog.csdn.net/sxj731533730/article/details/108616226

#include "ncnn/benchmark.h"
#include "ncnn/cpu.h"
#include "ncnn/datareader.h"
#include "ncnn/net.h"
#include "ncnn/gpu.h"

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <vector>
#include <algorithm>
using namespace cv;
using namespace std;
using namespace ncnn;
int demo(cv::Mat& image, ncnn::Net &detector, int detector_size_width, int detector_size_height)
{

    static const char* class_names[] = {"background",
                                        "aeroplane", "bicycle", "bird", "boat",
                                        "bottle", "bus", "car", "cat", "chair",
                                        "cow", "diningtable", "dog", "horse",
                                        "motorbike", "person", "pottedplant",
                                        "sheep", "sofa", "train", "tvmonitor"
    };

    cv::Mat bgr = image.clone();
    int img_w = bgr.cols;
    int img_h = bgr.rows;

    ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB,\
                                                 bgr.cols, bgr.rows, detector_size_width, detector_size_height);

    //数据预处理
    const float mean_vals[3] = {0.f, 0.f, 0.f};
    const float norm_vals[3] = {1/255.f, 1/255.f, 1/255.f};
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = detector.create_extractor();
    ex.set_num_threads(8);
    ex.input("data", in);
    ncnn::Mat out;
    ex.extract("output", out);

    for (int i = 0; i < out.h; i++)
    {
        int label;
        float x1, y1, x2, y2, score;
        float pw,ph,cx,cy;
        const float* values = out.row(i);

        x1 = values[2] * img_w;
        y1 = values[3] * img_h;
        x2 = values[4] * img_w;
        y2 = values[5] * img_h;

        score = values[1];
        label = values[0];

        //处理坐标越界问题
        if(x1<0) x1=0;
        if(y1<0) y1=0;
        if(x2<0) x2=0;
        if(y2<0) y2=0;

        if(x1>img_w) x1=img_w;
        if(y1>img_h) y1=img_h;
        if(x2>img_w) x2=img_w;
        if(y2>img_h) y2=img_h;
        cv::rectangle (image, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(255, 255, 0), 1, 1, 0);

        char text[256];
        sprintf(text, "%s %.1f%%", class_names[label], score * 100);
        int baseLine = 0;
        cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
        cv::putText(image, text, cv::Point(x1, y1 + label_size.height),
                    cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));
    }
    return 0;
}

//摄像头测试
int test_cam()
{
    //定义yolo-fastest VOC检测器
    ncnn::Net detector;
    detector.load_param("/home/ubuntu/CLionProjects/untitled3/model/yolo-fastest.param");
    detector.load_model("/home/ubuntu/CLionProjects/untitled3/model/yolo-fastest.bin");
    int detector_size_width  = 320;
    int detector_size_height = 320;

    cv::Mat frame;
    cv::VideoCapture cap(0);

    while (true)
    {
        cap >> frame;
        double start = ncnn::get_current_time();
        demo(frame, detector, detector_size_width, detector_size_height);
        double end = ncnn::get_current_time();
        double time = end - start;
        printf("Time:%7.2f \n",time);
        cv::imshow("demo", frame);
        cv::waitKey(1);
    }
    return 0;
}
int main()
{
    test_cam();
    return 0;
}

完成了c++ 调用小企鹅的NCNN框架,然后在使用小马哥的rapidjson组件的功能,拼凑labelme标注的标注json格式数据;

下载 https://github.com/Tencent/rapidjson  或者使用命令安装,但是在clion工程中仍然使用头文件

sudo apt-get install rapidjson-dev

 

ubuntu@ubuntu:~$ git clone https://github.com/Tencent/rapidjson.git
ubuntu@ubuntu:~$ cd rapidjson/
ubuntu@ubuntu:~/rapidjson$ mkdir -p build
ubuntu@ubuntu:~/rapidjson$ cd build/
ubuntu@ubuntu:~/rapidjson/build$ cmake ..
ubuntu@ubuntu:~/rapidjson/build$ sudo make install

然后,将工程中include文件拷贝到CLion工程,整个工程目录为:

.
├── cmake-build-debug
│   ├── CMakeCache.txt
│   ├── CMakeFiles
│   ├── cmake_install.cmake
│   ├── Makefile
│   ├── untitled3
│   └── untitled3.cbp
├── CMakeLists.txt
├── include
│   ├── ncnn
│   └── rapidjson
├── libs
│   └── libncnn.a
├── main.cpp
└── model
    ├── yolo-fastest.bin
    └── yolo-fastest.param

7 directories, 10 files

修改代码为检测图片的目录和生成对应的检测json文件(增加生成json模块);

#include "ncnn/benchmark.h"
#include "ncnn/cpu.h"
#include "ncnn/datareader.h"
#include "ncnn/net.h"
#include "ncnn/gpu.h"
#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <vector>
#include <fstream>

using namespace cv;
using namespace std;
using namespace ncnn;
using namespace rapidjson;

struct LabelInfo {
    int label;
    float x_min;
    float y_min;
    float x_max;
    float y_max;
};
LabelInfo labelinfo;

vector<LabelInfo> vec_info;

ncnn::Net detector;

int
detect(cv::Mat &image, ncnn::Net &detector, int detector_size_width, int detector_size_height, char *class_names[]) {


    cv::Mat bgr = image.clone();
    int img_w = bgr.cols;
    int img_h = bgr.rows;

    ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, \
                                                 bgr.cols, bgr.rows, detector_size_width, detector_size_height);

    //数据预处理
    const float mean_vals[3] = {0.f, 0.f, 0.f};
    const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f};
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = detector.create_extractor();
    ex.set_num_threads(8);
    ex.input("data", in);
    ncnn::Mat out;
    ex.extract("output", out);

    for (int i = 0; i < out.h; i++) {
        int label;
        float x1, y1, x2, y2, score;
        float pw, ph, cx, cy;
        const float *values = out.row(i);

        x1 = values[2] * img_w;
        y1 = values[3] * img_h;
        x2 = values[4] * img_w;
        y2 = values[5] * img_h;

        score = values[1];
        label = values[0];

        //处理坐标越界问题
        if (x1 < 0) x1 = 0;
        if (y1 < 0) y1 = 0;
        if (x2 < 0) x2 = 0;
        if (y2 < 0) y2 = 0;

        if (x1 > img_w) x1 = img_w;
        if (y1 > img_h) y1 = img_h;
        if (x2 > img_w) x2 = img_w;
        if (y2 > img_h) y2 = img_h;
        //cv::rectangle(image, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(255, 255, 0), 1, 1, 0);

        char text[256];
        sprintf(text, "%s %.1f%%", class_names[label], score * 100);
        int baseLine = 0;
        //cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
        //cv::putText(image, text, cv::Point(x1, y1 + label_size.height),cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));

        labelinfo.label = label;
        labelinfo.x_min = x1;
        labelinfo.y_min = y1;
        labelinfo.x_max = x2;
        labelinfo.y_max = y2;
        vec_info.push_back(labelinfo);


    }

}


int init_detect(char *detect_param, char *detect_bin) {
    //定义yolo-fastest VOC检测器

    detector.load_param(detect_param);
    detector.load_model(detect_bin);

}

// 帧号 目的json文件 帧 图片目的 生成json文件 图片的实际宽和高 文件的基本名字
void
makejson(string frame_name, cv::Mat &frame, string image_destion, string json_destion, int imageWidth, int imageHeight, char *class_names[]) {
    if (vec_info.size()) {

        rapidjson::Document doc;
        doc.SetObject();
        rapidjson::Document::AllocatorType &allocator = doc.GetAllocator();
        //成员1
        rapidjson::Value str_version(rapidjson::kStringType);
        str_version.SetString("4.5.6");
        rapidjson::Value str_flags(rapidjson::kStringType);
        str_flags.SetObject();
        rapidjson::Value str_imageData(rapidjson::kStringType);
        str_imageData.SetNull();
        rapidjson::Value str_imageWidth(rapidjson::kStringType);
        str_imageWidth.SetInt(imageWidth);
        rapidjson::Value str_imageHeight(rapidjson::kStringType);
        str_imageHeight.SetInt(imageHeight);
        rapidjson::Value str_imagePath(rapidjson::kStringType);
        string image_frame_name = frame_name + ".jpg";
        str_imagePath.SetString(image_frame_name.c_str(), image_frame_name.length(), allocator);
        rapidjson::Value ary(rapidjson::kArrayType);
        for(int i=0;i<vec_info.size();i++)
        {
        // 嵌套成员2对象
        rapidjson::Document sub_doc;
        sub_doc.SetObject();
        rapidjson::Document::AllocatorType &sub_allocator = sub_doc.GetAllocator();
        rapidjson::Value sub_str_shape_type(rapidjson::kStringType);
        sub_str_shape_type.SetString("rectangle");
        rapidjson::Value sub_str_flags(rapidjson::kStringType);
        sub_str_flags.SetObject();
        rapidjson::Value sub_str_group_id(rapidjson::kStringType);
        sub_str_group_id.SetNull();
        rapidjson::Value sub_str_label(rapidjson::kStringType);
        int labelid=vec_info[i].label;
        string labelname=class_names[labelid];
        sub_str_label.SetString(labelname.c_str(), labelname.length(), allocator);
        // 嵌套坐标点
        Value sub_array0(kArrayType);
        Value sub_array1(kArrayType);
        Value sub_point(kArrayType);

        float x_min=vec_info[i].x_min;
        float y_min=vec_info[i].y_min;
        float x_max=vec_info[i].x_max;
        float y_max=vec_info[i].y_max;

        sub_array0.PushBack(x_min, allocator).PushBack(y_min, allocator);
        sub_array1.PushBack(x_max, allocator).PushBack(y_max, allocator);
        sub_point.PushBack(sub_array0, allocator);
        sub_point.PushBack(sub_array1, allocator);
        sub_doc.AddMember("points", sub_point, allocator);

        // 嵌套坐标点完成

        sub_doc.AddMember("shape_type", sub_str_shape_type, allocator);
        sub_doc.AddMember("flags", sub_str_flags, allocator);
        sub_doc.AddMember("group_id", sub_str_group_id, allocator);
        sub_doc.AddMember("label", sub_str_label, allocator);

        ary.PushBack(sub_doc, allocator);
        //成员2完成
        }
        //加入doc中
        doc.AddMember("version", str_version, allocator);
        doc.AddMember("flags", str_flags, allocator);
        doc.AddMember("imageData", str_imageData, allocator);
        doc.AddMember("imageWidth", imageWidth, allocator);
        doc.AddMember("imageHeight", imageHeight, allocator);
        doc.AddMember("imagePath", str_imagePath, allocator);

        doc.AddMember("shapes", ary, allocator);
        //转化为string
        rapidjson::StringBuffer buffer;
        rapidjson::Writer<rapidjson::StringBuffer> write(buffer);
        doc.Accept(write);
        std::string json = buffer.GetString();

        // Output {"project":"rapidjson","stars":11}
        std::cout << json << std::endl;
        ofstream fout;
        string destination_name=json_destion+"/"+frame_name+".json";

        fout.open(destination_name);    //可以使绝对和相对路径,用\\隔开目录,test, test.json, test.txt 都行,不局限于文件格式后缀,只要是文本文档
        fout<<buffer.GetString();
        fout.close();
        string destination_image=image_destion+"/"+frame_name+".jpg";
        imwrite(destination_image,frame);

    }
    vec_info.clear();
    vector<LabelInfo>().swap(vec_info);
}

void detect_object(char *video_src, int detector_size_width, int detector_size_height, char *class_names[],
                   string image_destion, string json_destion) {


    char *base_name = basename(video_src);
    string label_ext = base_name;
    string file_name = label_ext.substr(0, label_ext.rfind("."));

    cv::Mat frame;
    cv::VideoCapture cap(video_src);
    int frame_num = 0;
    while (cap.read(frame)) {

        double start = ncnn::get_current_time();
        detect(frame, detector, detector_size_width, detector_size_height, class_names);
        double end = ncnn::get_current_time();
        double time = end - start;
        string frame_name = file_name + to_string(frame_num);
        makejson(frame_name, frame, image_destion, json_destion, frame.rows, frame.cols,class_names);
        printf("Time:%7.2f \n", time);
        cv::imshow("demo", frame);
        cv::waitKey(1);
        frame_num++;
    }
    cap.release();
    destroyAllWindows();
}

int main() {

    char *video_src = "/home/ubuntu/MOT16Labels/MOT16-06-raw.webm";
    char *detect_param = "/home/ubuntu/CLionProjects/untitled3/model/yolo-fastest.param";
    char *detect_bin = "/home/ubuntu/CLionProjects/untitled3/model/yolo-fastest.bin";
    int detector_size_width = 320;
    int detector_size_height = 320;
    char *class_names[] = {"background",
                           "aeroplane", "bicycle", "bird", "boat",
                           "bottle", "bus", "car", "cat", "chair",
                           "cow", "diningtable", "dog", "horse",
                           "motorbike", "person", "pottedplant",
                           "sheep", "sofa", "train", "tvmonitor"
    };

    string image_destion = "/home/ubuntu/Downloads/A";
    string json_destion = "/home/ubuntu/Downloads/A";
    init_detect(detect_param, detect_bin);
    detect_object(video_src, detector_size_width, detector_size_height, class_names, image_destion, json_destion);

    return 0;
}

 

上一篇:html标签反转义


下一篇:水质生物毒性检测-应急常规项目39项