一、大体流程图
上图是rockx+rv1126的大体流程，首先要初始化模块包括VI模块、VENC模块、并启动VI模块采集视频流、rockx模块的初始化。初始化模块后，就要分两个线程处理了。
主线程是负责rockx对VI视频流的处理，并用OPENCV对人脸进行画框，最后把处理后的VI数据传输到VENC模块里面。
第二个线程rockx_face_detect_venc_thread，从VENC模块获取到H264的编码码流数据，并把VENC码流数据保存。
二．代码
#include <assert.h> #include <cstddef> #include <fcntl.h> #include <getopt.h> #include <opencv2/imgproc.hpp> #include <opencv2/imgproc/imgproc_c.h> #include <pthread.h> #include <signal.h> #include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <time.h> #include <unistd.h> // #include "common/sample_common.h" #include "rkmedia_api.h" #include "rknn_rockx_include/rockx_type.h" #include "rockx.h" #include <opencv2/core.hpp> // #include <opencv2/imgoroc.hpp> #include <opencv2/highgui.hpp> #include <opencv2/opencv.hpp> #define CAMERA_PATH "rkispp_scale0" #define CAMERA_ID 0 #define CAMERA_CHN 0 #define VENC_CHN 0 #define WIDTH 1920 #define HEIGHT 1080 using namespace cv; void * rockx_face_detect_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_detect_h264 = fopen("face_detect_venc.h264", "w+"); MEDIA_BUFFER mb = NULL; while(1) { mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf("Get Rockx_Venc Data berek...\n"); break; } printf("Get Rockx_Venc Data Success...\n"); fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; } int main(int argc, char **argv) { int ret; VI_CHN_ATTR_S vi_chn_attr; vi_chn_attr.pcVideoNode = CAMERA_PATH; // Path vi_chn_attr.u32Width = WIDTH; // Width vi_chn_attr.u32Height = HEIGHT; // Height vi_chn_attr.enPixFmt = IMAGE_TYPE_NV12; // ImageType vi_chn_attr.enBufType = VI_CHN_BUF_TYPE_MMAP; // BufType vi_chn_attr.u32BufCnt = 3; // Cnt vi_chn_attr.enWorkMode = VI_WORK_MODE_NORMAL; // Mode ret = RK_MPI_VI_SetChnAttr(CAMERA_ID, CAMERA_CHN, &vi_chn_attr); if (ret) { printf("Vi Set Attr Failed.....\n"); return 0; } else { printf("Vi Set Attr Success.....\n"); } ret = RK_MPI_VI_EnableChn(CAMERA_ID, CAMERA_CHN); if (ret) { printf("Vi Enable Attr Failed.....\n"); return 0; } else { printf("Vi Enable Attr Success.....\n"); } VENC_CHN_ATTR_S venc_chn_attr; memset(&venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S)); venc_chn_attr.stVencAttr.u32PicWidth = WIDTH; venc_chn_attr.stVencAttr.u32PicHeight = HEIGHT; venc_chn_attr.stVencAttr.u32VirWidth = WIDTH; venc_chn_attr.stVencAttr.u32VirHeight = HEIGHT; venc_chn_attr.stVencAttr.imageType = IMAGE_TYPE_NV12; venc_chn_attr.stVencAttr.enType = RK_CODEC_TYPE_H264; venc_chn_attr.stVencAttr.u32Profile = 66; venc_chn_attr.stRcAttr.enRcMode = VENC_RC_MODE_H264CBR; venc_chn_attr.stRcAttr.stH264Cbr.u32Gop = 25; venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate = WIDTH * HEIGHT * 3; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen = 1; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum = 25; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen = 1; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum = 25; ret = RK_MPI_VENC_CreateChn(VENC_CHN, &venc_chn_attr); if (ret) { printf("ERROR: Create venc failed!\n"); exit(0); } ret = RK_MPI_VI_StartStream(CAMERA_ID, CAMERA_CHN); if (ret) { printf("RK_MPI_VI_StartStream Failed.....\n"); return 0; } else { printf("RK_MPI_VI_StartStream Success.....\n"); } pthread_t pid; pthread_create(&pid, NULL, rockx_face_detect_venc_thread, NULL); rockx_config_t * face_detect_config = rockx_create_config(); rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, "/userdata/rockx_data"); rockx_handle_t face_detect_handle; rockx_ret_t face_detect_ret; rockx_module_t face_detect_module = ROCKX_MODULE_FACE_DETECTION_V2; face_detect_ret = rockx_create(&face_detect_handle, face_detect_module, face_detect_config, 0); if(face_detect_ret != ROCKX_RET_SUCCESS) { printf("rockx_creat face_detect failed...\n"); return -1; } rockx_release_config(face_detect_config); rockx_image_t rv1126_rockx_image; memset(&rv1126_rockx_image, 0 ,sizeof(rv1126_rockx_image)); rv1126_rockx_image.width = WIDTH; rv1126_rockx_image.height = HEIGHT; rv1126_rockx_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12; MEDIA_BUFFER mb; while(1) { mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1); if(!mb) { printf("Get Vi Stream break....\n"); break; } rv1126_rockx_image.data = (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_rockx_image.size = RK_MPI_MB_GetSize(mb); rockx_object_array_t face_detect_array; face_detect_ret = rockx_face_detect(face_detect_handle, &rv1126_rockx_image, &face_detect_array, NULL); if(face_detect_ret != ROCKX_RET_SUCCESS) { printf("face_detect failed....\n"); } Mat rv1126_rockx_mat = Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data); for(int i = 0; i < face_detect_array.count; i++) { int left = face_detect_array.object[i].box.left; int top = face_detect_array.object[i].box.top; int w = face_detect_array.object[i].box.right - face_detect_array.object[i].box.left; int h = face_detect_array.object[i].box.bottom - face_detect_array.object[i].box.top; Rect boundingRect(left, top, w, h); rectangle(rv1126_rockx_mat, boundingRect, Scalar(255,255,0)); } RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb); } RK_MPI_VENC_DestroyChn(VENC_CHN); RK_MPI_VI_DisableChn(CAMERA_ID, CAMERA_CHN); return 0; }2.1. RV1126模块初始化并启动VI工作
RV1126模块的初始化，包括VI模块、VENC模块的初始化，初始化上述模块后，则调用RK_MPI_VI_StartStream启动VI开始采集摄像头的视频流。关于VI模块、VENC模块的初始化参数这里就不阐述了，因为之前的课程里面已经讲了很多次。
2.2. rockx人脸检测模块的初始化
rockx_config_t * face_detect_config = rockx_create_config(); rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, "/userdata/rockx_data"); rockx_handle_t face_detect_handle; rockx_ret_t face_detect_ret; rockx_module_t face_detect_module = ROCKX_MODULE_FACE_DETECTION_V2; face_detect_ret = rockx_create(&face_detect_handle, face_detect_module, face_detect_config, 0); if(face_detect_ret != ROCKX_RET_SUCCESS) { printf("rockx_creat face_detect failed...\n"); return -1; } rockx_release_config(face_detect_config);这段代码是初始化rockx的模块，首先要使用rockx_create_config分配rockx_config_t结构体，并使用rockx_add_config把对应的rockx路径配置进去，在我们的板子里面在/userdata/rockx_data里面,并使用rockx_create创建rockx_handle_t句柄，rockx_create的传参第一个参数rockx_handle_t结构体指针、 第二个参数rockx_module_t是ROCKX_MODULE_FACE_DETECTION_V2，ROCKX_MODULE_FACE_DETECTION_V2是人脸检测的Version2模块、第三个参数是rockx_config_t结构体指针、第四个参数默认是0。
2.3.使用rockx对VI模块的数据进行人脸检测处理
rockx_image_t rv1126_rockx_image; memset(&rv1126_rockx_image, 0 ,sizeof(rv1126_rockx_image)); rv1126_rockx_image.width = WIDTH; rv1126_rockx_image.height = HEIGHT; rv1126_rockx_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;MEDIA_BUFFER mb; while(1) { mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1); if(!mb) { printf("Get Vi Stream break....\n"); break; } rv1126_rockx_image.data = (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_rockx_image.size = RK_MPI_MB_GetSize(mb); rockx_object_array_t face_detect_array; face_detect_ret = rockx_face_detect(face_detect_handle, &rv1126_rockx_image, &face_detect_array, NULL); if(face_detect_ret != ROCKX_RET_SUCCESS) { printf("face_detect failed....\n"); }这部分代码是整个DEMO的核心，也是ROCKX检测VI视频数据的核心。第一段是初始化rockx_image_t结构体，初始化需要传三个值分别是width = WIDTH(1920)、height = HEIGHT(1080)、pixel_format=ROCKX_PIXEL_FORMAT_YUV420SP_NV12。这三个值都需要和VI模块的配置是一样的。
初始化rockx_image_t后，则需要通过RK_MPI_SYS_GetMediaBuffer获取每一帧VI模块的数据，并把每一帧VI模块的缓冲区和长度传输给rockx_image_t。具体的代码是rv1126_rockx_image.data = (uint8_t *)RK_MPI_MB_GetPtr(mb)(把每一帧VI缓冲区数据赋值到rockx_image_t的data)、rv1126_rockx_image.size = RK_MPI_MB_GetSize(mb)(把每一帧VI大小赋值到rockx_image_t的size)
赋值到rockx_image_t后，则调用rockx_face_detect对每一帧的rockx_image_t图像进行人脸检测，并把人脸检测的结果输出到rockx_object_array_t。rockx_object_array_t的内容主要存储的是人脸检测数量和人脸检测区域信息(如：left、top、right、bottom的坐标信息)
2.4.使用opencv对人脸检测的结果进行画框
Mat rv1126_rockx_mat = Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data); for(int i = 0; i < face_detect_array.count; i++) { int left = face_detect_array.object[i].box.left; int top = face_detect_array.object[i].box.top; int w = face_detect_array.object[i].box.right - face_detect_array.object[i].box.left; int h = face_detect_array.object[i].box.bottom - face_detect_array.object[i].box.top; Rect boundingRect(left, top, w, h); rectangle(rv1126_rockx_mat, boundingRect, Scalar(255,255,0)); }检测完每一帧人脸数据后就需要对每个人脸区域进行画框了，这里画框是用opencv进行处理。首先要先创建OPENCV的Mat矩阵，Mat rv1126_image_mat = Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data)。创建完Mat之后，则需要根据rockx_object_array_t的坐标信息进行画框，先循环遍历人脸的数量(rockx_object_array_t.count)，然后获取每一帧人脸的坐标信息,主要是left、top、right、bottom, 最后使用OPENCV的rectangle函数把坐标信息描绘出一个矩形表现出来。
2.5.把处理后的数据发送到VENC模块
RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb);把上述的数据处理完成之后则把每一帧数据传输给VENC模块，这里使用的API是RK_MPI_SYS_SendMediaBuffer。此时此刻VENC模块就有VENC码流数据了
2.6.创建rockx_face_detect_venc _thread线程保存每一帧H264的编码码流数据
pthread_t pid; pthread_create(&pid, NULL, rockx_face_detect_venc_thread, NULL);void * rockx_face_detect_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_detect_h264 = fopen("face_detect_venc.h264", "w+"); MEDIA_BUFFER mb = NULL; while(1) { mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf("Get Rockx_Venc Data berek...\n"); break; } printf("Get Rockx_Venc Data Success...\n"); fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; }通过pthread_create创建venc码流线程，这个线程的名字是rockx_face_detect_venc thread。在这个线程里面，通过RK_MPI_SYS_GetMediaBuffer获取每一帧通过rockx人脸检测处理后的VENC码流数据，并用fwrite保存起来(fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264))。
最终输出的结果是在视频中检测出对应的人脸并用opencv画矩形出来。