要想将摄像头采集的视频流和桌面采集的视频流混成一股流，需要再次借助ffmpeg的avfilter功能库。

技术简介

借助fmpeg的filter功能，将两股视频合成一股视频流，从而实现两股视频的画中画效果

使用模块（库）

使用ffmpeg的avfilter库

主要流程和代码

1、初始化视频混流器。最需要关注的是filterDesc，即"[in0]setpts=PTS-STARTPTS,scale=%dx%d[main];[in1]setpts=PTS-STARTPTS,scale=%dx%d[over];[main][over]overlay=x=W-w:y=H-h:format=0[out]"，因为这个最关键，这句话的意思是将[in0]设置从0开始的时间戳、scale视频大小并输出为标签[main]，同理，[in1]输出为标签[over]，然后将[over]在[main]上面使用overlay设置左上角位置、输出format为yuv420p并输出标签[out]

int VideoMixer::init(const VIDEO_FILTER_CTX& outCtx, const VIDEO_FILTER_CTX* inCtx0, const VIDEO_FILTER_CTX* inCtx1)
{
	int err = ERROR_CODE_OK;

	if (m_inited) {
		return err;
	}

	if (inCtx0 == nullptr || inCtx1 == nullptr) {
		err = ERROR_CODE_PARAMS_ERROR;
		return err;
	}

	do {
		m_filterGraph = avfilter_graph_alloc();
		if (m_filterGraph == nullptr) {
			err = ERROR_CODE_FILTER_ALLOC_GRAPH_FAILED;
			break;
		}

		m_filterInCtxs = new VIDEO_FILTER_CTX[VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE];
		memcpy_s(&m_filterInCtxs[0], sizeof(VIDEO_FILTER_CTX), inCtx0, sizeof(VIDEO_FILTER_CTX));
		memcpy_s(&m_filterInCtxs[1], sizeof(VIDEO_FILTER_CTX), inCtx1, sizeof(VIDEO_FILTER_CTX));
		m_filterOutCtx = outCtx;

		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			m_filterInCtxs[i].filterInout = avfilter_inout_alloc();
		}
		m_filterOutCtx.filterInout = avfilter_inout_alloc();

		char filterInArgs[2][512] = { 0 };
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			sprintf_s(filterInArgs[i], sizeof(filterInArgs[i]), "video_size=%dx%d:pix_fmt=%d:frame_rate=%d:time_base=%d/%d:pixel_aspect=%d/%d",
				m_filterInCtxs[i].width, m_filterInCtxs[i].height, m_filterInCtxs[i].pixelFmt, m_filterInCtxs[i].framerate,
				m_filterInCtxs[i].timebase.num, m_filterInCtxs[i].timebase.den, m_filterInCtxs[i].pixelAspect.num, m_filterInCtxs[i].pixelAspect.den);
		}

		int ret = 0;
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			char filterName[4] = { 0 };
			sprintf_s(filterName, sizeof(filterName), "in%d", i);
			ret = avfilter_graph_create_filter(&m_filterInCtxs[i].filterCtx, avfilter_get_by_name("buffer"), filterName, filterInArgs[i], nullptr, m_filterGraph);
			if (ret < 0) {
				err = ERROR_CODE_FILTER_CREATE_FILTER_FAILED;
				break;
			}
		}
		if (err != ERROR_CODE_OK) {
			break;
		}

		ret = avfilter_graph_create_filter(&m_filterOutCtx.filterCtx, avfilter_get_by_name("buffersink"), "out", nullptr, nullptr, m_filterGraph);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_CREATE_FILTER_FAILED;
			break;
		}

		av_opt_set_bin(m_filterOutCtx.filterCtx, "pix_fmts", (uint8_t*)&m_filterOutCtx.pixelFmt, sizeof(m_filterOutCtx.pixelFmt), AV_OPT_SEARCH_CHILDREN);

		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			char filterName[4] = { 0 };
			sprintf_s(filterName, sizeof(filterName), "in%d", i);
			m_filterInCtxs[i].filterInout->name = av_strdup(filterName);
			m_filterInCtxs[i].filterInout->filter_ctx = m_filterInCtxs[i].filterCtx;
			m_filterInCtxs[i].filterInout->pad_idx = 0;
			if (i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE - 1) {
				m_filterInCtxs[i].filterInout->next = m_filterInCtxs[i + 1].filterInout;
			}
			else {
				m_filterInCtxs[i].filterInout->next = nullptr;
			}
		}

		m_filterOutCtx.filterInout->name = av_strdup("out");
		m_filterOutCtx.filterInout->filter_ctx = m_filterOutCtx.filterCtx;
		m_filterOutCtx.filterInout->pad_idx = 0;
		m_filterOutCtx.filterInout->next = nullptr;

		AVFilterInOut* filterInOutputs[VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE];
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			filterInOutputs[i] = m_filterInCtxs[i].filterInout;
		}

		char filterDesc[256] = { 0 };
		int smallWidth, smallHeight;
		getSmallResolutionByLarge(m_filterOutCtx.width, m_filterOutCtx.height, smallWidth, smallHeight);
		sprintf_s(filterDesc, sizeof(filterDesc), "[in0]setpts=PTS-STARTPTS,scale=%dx%d[main];[in1]setpts=PTS-STARTPTS,scale=%dx%d[over];[main][over]overlay=x=W-w:y=H-h:format=0[out]",
			m_filterOutCtx.width, m_filterOutCtx.height, smallWidth, smallHeight);
		ret = avfilter_graph_parse_ptr(m_filterGraph, filterDesc, &m_filterOutCtx.filterInout, filterInOutputs, nullptr);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_PARSE_PTR_FAILED;
			break;
		}

		ret = avfilter_graph_config(m_filterGraph, nullptr);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_CONFIG_FAILED;
			break;
		}

		m_inited = true;
	} while (0);

	if (err != ERROR_CODE_OK) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] init mixer, error: %s", __FUNCTION__, HCMDR_GET_ERROR_DESC(err));
		cleanup();
	}

	return err;
}

2、启动视频filter线程

int VideoMixer::start()
{
	int err = ERROR_CODE_OK;

	if (!m_inited) {
		err = ERROR_CODE_UNINITIALIZED;
		return err;
	}

	if (m_running) {
		return err;
	}

	m_running = true;
	m_thread = std::thread(std::bind(&VideoMixer::mixProcess, this));

	return err;
}

视频混流线程函数，从buffersink队列中获取一帧传给下一层

void VideoMixer::mixProcess()
{
	AVFrame* frame = av_frame_alloc();

	while (m_running) {
		std::unique_lock<std::mutex> lock(m_mutex);

		while (m_running && !m_conditionFlag) {
			m_condition.wait_for(lock, std::chrono::milliseconds(VIDEO_MIXER_WAIT_FOR_FRAME_TIMEOUT));
		}

		while (m_running && m_conditionFlag) {
			int ret = av_buffersink_get_frame(m_filterOutCtx.filterCtx, frame);
			if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
				break;
			}

			if (ret < 0) {
				LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] get sink frame, error: %s", __FUNCTION__, ret);
				if (m_onVideoFilterError != nullptr) {
					m_onVideoFilterError(ret);
				}
				break;
			}

			if (m_onVideoFilterData != nullptr) {
				m_onVideoFilterData(frame);
			}

			av_frame_unref(frame);
		}

		m_conditionFlag = false;
	}

	av_frame_free(&frame);
}

3、视频流输入，根据index可以将对应视频输入源的视频帧加入buffersrc队列

int VideoMixer::addFrame(AVFrame* frame, int index)
{
	int err = ERROR_CODE_OK;

	if (frame == nullptr) {
		err = ERROR_CODE_PARAMS_ERROR;
		return err;
	}
	if (0 > index || index >= VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE) {
		err = ERROR_CODE_FILTER_INVALID_FILTER;
		return err;
	}

	std::unique_lock<std::mutex> lock(m_mutex);

	do {
		AVFilterContext* filterCtx = m_filterInCtxs[index].filterCtx;
		int ret = av_buffersrc_add_frame_flags(filterCtx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_ADD_FRAME_FAILED;
			break;
		}
	} while (0);

	if (err != ERROR_CODE_OK) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] add frame, error: %s", __FUNCTION__, HCMDR_GET_ERROR_DESC(err));
	}

	m_conditionFlag = true;
	m_condition.notify_all();

	return err;
}

4、停止视频混流

int VideoMixer::stop()
{
	int err = ERROR_CODE_OK;

	if (!m_inited) {
		err = ERROR_CODE_UNINITIALIZED;
		return err;
	}

	if (!m_running) {
		return err;
	}

	m_running = false;

	m_conditionFlag = true;
	m_condition.notify_all();

	if (m_thread.joinable()) {
		m_thread.join();
	}

	return err;
}

这样，两个视频合成一股视频了。

补充：需要在FfmpegMuxer类（可以关注文章《【音视频】保存同步的音视频文件（九）》）中需要添加一个处理VideoFilter的数据回调，如：

void FfmpegMuxer::onVideoFilterData(AVFrame* frame)
{
	if (frame == nullptr) {
		return;
	}
	if (!m_running || m_paused || m_videoStream == nullptr || m_videoStream->vEncoder == nullptr) {
		return;
	}

	int err = ERROR_CODE_OK;
	AVFrame* dstFrame = frame;
	uint8_t* dstData = nullptr;
	uint32_t dstDataLen = 0;
	if (dstFrame->format == AV_PIX_FMT_YUV420P) {
		int dstOffset = 0;
		dstDataLen = dstFrame->width * dstFrame->height * 3 / 2;
		dstData = new uint8_t[dstDataLen];
		for (int i = 0; i < dstFrame->height; i++) {
			memcpy_s(dstData + i * dstFrame->width, dstFrame->width, dstFrame->data[0] + i * dstFrame->linesize[0], dstFrame->width);
		}
		dstOffset += dstFrame->width * dstFrame->height;
		for (int i = 0; i < dstFrame->height / 2; i++) {
			memcpy_s(dstData + dstOffset + i * dstFrame->width / 2, dstFrame->width / 2, dstFrame->data[1] + i * dstFrame->linesize[1], dstFrame->width / 2);
		}
		dstOffset += dstFrame->width * dstFrame->height / 4;
		for (int i = 0; i < dstFrame->height / 2; i++) {
			memcpy_s(dstData + dstOffset + i * dstFrame->width / 2, dstFrame->width / 2, dstFrame->data[2] + i * dstFrame->linesize[2], dstFrame->width / 2);
		}
	}
	if (dstData != nullptr && dstDataLen > 0) {
		err = m_videoStream->vEncoder->addFrame(dstData, dstDataLen, dstFrame);
		if (m_previewed && m_onMuxData != nullptr) {
			m_onMuxData((AVPixelFormat)dstFrame->format, dstData, dstDataLen, 0, 0);
		}
		delete[] dstData;
	}
}

这里需要注意的是YUV数据的处理，我这里混流之后的视频格式是YUV420P，看起来很麻烦的yuv数据拷贝其实是必须的，因为YUV420P原始数据Y占据height*linesize[0]，U占据(height/2)*linesize[1]，V占据(height/2)*linesize[2]，而目标数据Y占据height*width，U和V各占据(height/2)*(width/2)

总结：需要研究一下ffmpeg中filter的使用，另外需要研究YUV数据格式。