androidcamera采集、H264编码与Rtmp推流

MediaPlus是基于FFmpeg从零开发的android多媒体组件&＃xff0c;主要包括&＃xff1a;采集&＃xff0c;编码&＃xff0c;同步&＃xff0c;推流&＃xff0c;滤镜及直播及短视频比较通用的功能等&＃xff0c;后续功能的新增都会有相应文档更新&＃xff0c;感谢关注。

android相机的视频采集格式比较多 &＃xff0c;如&＃xff1a;NV21,NV12,YV12等。他们之间的区别就是U,V排列顺序不一致&＃xff0c;具体YUV相关内容可以看看其他详细的文档&＃xff0c;如&＃xff1a;[总结]FFMPEG视音频编解码零基础学习方法。

需要了解的就是&＃xff1a;YUV采样&＃xff0c;数据分布及空间大小计算。
YUV采样&＃xff1a;

24439730_13282389538k8V.jpg

YUV420P YUV排序如下图&＃xff1a;

1346422959_6364.png

NV12,NV21,YV12,I420都属于YUV420,但是YUV420 又分为YUV420P&＃xff0c;YUV420SP&＃xff0c;P与SP区别就是&＃xff0c;前者YUV420P UV顺序存储&＃xff0c;而YUV420SP则是UV交错存储&＃xff0c;这是最大的区别&＃xff0c;具体的yuv排序就是这样的&＃xff1a;
I420: YYYYYYYY UU VV ->YUV420P
YV12: YYYYYYYY VV UU ->YUV420P
NV12: YYYYYYYY UVUV ->YUV420SP
NV21: YYYYYYYY VUVU ->YUV420SP

那么H264编码&＃xff0c;为什么需要把android 相机采集的NV21数据转换成YUV420P?
刚开始对这些颜色格式也很模糊&＃xff0c;后来找到了真理:因为H264编码必须要用 I420, 所以这里必须要处理色彩格式转换。
MediaPlus采集视频数据为NV21格式,以下描述如何获取android camera采集的每一帧数据&＃xff0c;并处理色彩格式转换&＃xff0c;代码如下:

获取相机采集数据&＃xff1a;

mCamera &＃61; Camera.open(Camera.CameraInfo.CAMERA_FACING_BACK);mParams &＃61; mCamera.getParameters();setCameraDisplayOrientation(this, Camera.CameraInfo.CAMERA_FACING_BACK, mCamera);mParams.setPreviewSize(SRC_FRAME_WIDTH, SRC_FRAME_HEIGHT);mParams.setPreviewFormat(ImageFormat.NV21); //preview format&＃xff1a;NV21 mParams.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO);m_camera.setDisplayOrientation(90);mCamera.setParameters(mParams); // setting camera parametersm_camera.addCallbackBuffer(m_nv21);m_camera.setPreviewCallbackWithBuffer(this);m_camera.startPreview();&＃64;Overridepublic void onPreviewFrame(byte[] data, Camera camera) {// TODO Auto-generated method stub//data这里就是获取到的NV21数据m_camera.addCallbackBuffer(m_nv21);//这里要添加一次缓冲&＃xff0c;否则onPreviewFrame可能不会再被回调}

因为NV21数据的所需空间大小(字节)&＃xff1d;宽 x 高 x 3 / 2 (y&＃61;WxH,u&＃61;WxH/4,v&＃61;WxH/4)&＃xff1b;所以我们需要建立一个byte数组&＃xff0c;作为采集视频数据的缓冲区.
MediaPlus>>app.mobile.nativeapp.com.libmedia.core.streamer.RtmpPushStreamer 类主要采集音视频数据&＃xff0c;并交由底层处理&＃xff1b;有两个线程分别用于处理音视频&＃xff0c;AudioThread 、VideoThread.

首先看下VideoThread

/*** 视频采集线程*/class VideoThread extends Thread {public volatile boolean m_bExit &＃61; false;byte[] m_nv21Data &＃61; new byte[mVideoSizeConfig.srcFrameWidth* mVideoSizeConfig.srcFrameHeight * 3 / 2];byte[] m_I420Data &＃61; new byte[mVideoSizeConfig.srcFrameWidth* mVideoSizeConfig.srcFrameHeight * 3 / 2];byte[] m_RotateData &＃61; new byte[mVideoSizeConfig.srcFrameWidth* mVideoSizeConfig.srcFrameHeight * 3 / 2];byte[] m_MirrorData &＃61; new byte[mVideoSizeConfig.srcFrameWidth* mVideoSizeConfig.srcFrameHeight * 3 / 2];&＃64;Overridepublic void run() {// TODO Auto-generated method stubsuper.run();VideoCaptureInterface.GetFrameDataReturn ret;while (!m_bExit) {try {Thread.sleep(1, 10);if (m_bExit) {break;}} catch (InterruptedException e) {e.printStackTrace();}ret &＃61; mVideoCapture.GetFrameData(m_nv21Data,m_nv21Data.length);if (ret &＃61;&＃61; VideoCaptureInterface.GetFrameDataReturn.RET_SUCCESS) {frameCount&＃43;&＃43;;LibJniVideoProcess.NV21TOI420(mVideoSizeConfig.srcFrameWidth, mVideoSizeConfig.srcFrameHeight, m_nv21Data, m_I420Data);if (curCameraType &＃61;&＃61; VideoCaptureInterface.CameraDeviceType.CAMERA_FACING_FRONT) {LibJniVideoProcess.MirrorI420(mVideoSizeConfig.srcFrameWidth, mVideoSizeConfig.srcFrameHeight, m_I420Data, m_MirrorData);LibJniVideoProcess.RotateI420(mVideoSizeConfig.srcFrameWidth, mVideoSizeConfig.srcFrameHeight, m_MirrorData, m_RotateData, 90);} else if (curCameraType &＃61;&＃61; VideoCaptureInterface.CameraDeviceType.CAMERA_FACING_BACK) {LibJniVideoProcess.RotateI420(mVideoSizeConfig.srcFrameWidth, mVideoSizeConfig.srcFrameHeight, m_I420Data, m_RotateData, 90);}encodeVideo(m_RotateData, m_RotateData.length);}}}public void stopThread() {m_bExit &＃61; true;}}

为什么要旋转&＃xff1f;
实际上android camera采集的时候&＃xff0c;不管手机是纵向还是横向&＃xff0c;视频都是横向进行采集&＃xff0c;这样当手机纵向的时候&＃xff0c;就会有角度差异&＃xff1b;前置需要旋转270°&＃xff0c;后置旋转90°&＃xff0c;这样就能保证采集到的图像和手机方向是一致的。

处理镜像的原因是因为前置相机采集的图像默认就是镜像的&＃xff0c;再做一次镜像&＃xff0c;将图像还原回去。
MediaPlus中&＃xff0c;使用libyuv来处理转换、旋转、镜像等。
MediaPlus>>app.mobile.nativeapp.com.libmedia.core.jni.LibJniVideoProcess 提供应用层接口

package app.mobile.nativeapp.com.libmedia.core.jni;import app.mobile.nativeapp.com.libmedia.core.config.MediaNativeInit;/*** 色彩空间处理* Created by android on 11/16/17.*/public class LibJniVideoProcess {static {MediaNativeInit.InitMedia();}/*** NV21转换I420** &＃64;param in_width 输入宽度* &＃64;param in_height 输入高度* &＃64;param srcData 源数据* &＃64;param dstData 目标数据* &＃64;return*/public static native int NV21TOI420(int in_width, int in_height,byte[] srcData,byte[] dstData);/*** 镜像I420* &＃64;param in_width 输入宽度* &＃64;param in_height 输入高度* &＃64;param srcData 源数据* &＃64;param dstData 目标数据* &＃64;return*/public static native int MirrorI420(int in_width, int in_height,byte[] srcData,byte[] dstData);/*** 指定角度旋转I420* &＃64;param in_width 输入宽度* &＃64;param in_height 输入高度* &＃64;param srcData 源数据* &＃64;param dstData 目标数据*/public static native int RotateI420(int in_width, int in_height,byte[] srcData,byte[] dstData, int rotationValue);}

libmedia/src/cpp/jni/jni_Video_Process.cpp 图像处理JNI层&＃xff0c;libyuv比较强大&＃xff0c;包括了所有YUV的转换等其他处理&＃xff0c;简单描述下函数参数&＃xff0c;如&＃xff1a;

LIBYUV_API int NV21ToI420(const uint8* src_y, int src_stride_y,const uint8* src_vu, int src_stride_vu,uint8* dst_y, int dst_stride_y,uint8* dst_u, int dst_stride_u,uint8* dst_v, int dst_stride_v,int width, int height);

src_y :y分量存储空间
src_stride_y :y分量宽度数据长度
src_vu:uv分量存储空间
src_stride_uv:uv分量宽度数据长度
dst_y :目标y分量存储空间
dst_u :目标u分量存储空间
dst_v :目标v分量存储空间
dst_stride_y:目标y分量宽度数据长度
dst_stride_u:目标v分量宽度数据长度
dst_stride_v:目标u分量宽度数据长度
width: 视频宽
height:视频高
假设&＃xff0c;一个8(宽)x6(高)的图像,函数参数如下&＃xff1a;

int width&＃61;8; int height&＃61;6; //源数据存储空间 uint8_t *srcNV21Data; //目标存储空间 uint8_t *dstI420Data;src_y&＃61;srcNV21Data; src_uv&＃61;srcNV21Data &＃43; (widthxheight); src_stride_y&＃61;width; src_stride_uv&＃61;width/2;dst_y&＃61;dstI420Data; dst_u&＃61;dstI420Data&＃43;(widthxheight); dst_v&＃61;dstI420Data&＃43;(widthxheightx5/4); dst_stride_y&＃61;width; dst_stride_u&＃61;width/2; dst_stride_v&＃61;width/2;

以下是调用libyuv完成图像转换、旋转、镜像的代码&＃xff1a;

// // Created by developer on 11/16/17. //#include "jni_Video_Process.h"#ifdef __cplusplus extern "C" { #endifJNIEXPORT jint JNICALL Java_app_mobile_nativeapp_com_libmedia_core_jni_LibJniVideoProcess_NV21TOI420(JNIEnv *env,class type,jin in_width,jin in_height,jbyteArray srcData_,jbyteArray dstData_) {jbyte *srcData &＃61; env->GetByteArrayElements(srcData_, NULL);jbyte *dstData &＃61; env->GetByteArrayElements(dstData_, NULL);VideoProcess::NV21TOI420(in_width, in_height, (const uint8_t *) srcData,(uint8_t *) dstData);return 0; }JNIEXPORT jint JNICALL Java_app_mobile_nativeapp_com_libmedia_core_jni_LibJniVideoProcess_MirrorI420(JNIEnv *env,class type,jin in_width,jin in_height,jbyteArray srcData_,jbyteArray dstData_) {jbyte *srcData &＃61; env->GetByteArrayElements(srcData_, NULL);jbyte *dstData &＃61; env->GetByteArrayElements(dstData_, NULL);VideoProcess::MirrorI420(in_width, in_height, (const uint8_t *) srcData,(uint8_t *) dstData);return 0; }JNIEXPORT jint JNICALL Java_app_mobile_nativeapp_com_libmedia_core_jni_LibJniVideoProcess_RotateI420(JNIEnv *env,class type,jin in_width,jin in_hegith,jbyteArray srcData_,jbyteArray dstData_,jint rotationValue) {jbyte *srcData &＃61; env->GetByteArrayElements(srcData_, NULL);jbyte *dstData &＃61; env->GetByteArrayElements(dstData_, NULL);return VideoProcess::RotateI420(in_width, in_hegith, (const uint8_t *) srcData,(uint8_t *) dstData, rotationValue); }#ifdef __cplusplus } #endif

以上代码完成NV21转换为I420等处理&＃xff0c;接下来将数据传入底层&＃xff0c;就可以使用FFmpeg进行H264编码了,下图是底层C&＃43;&＃43;封装类图:

类图说明了&＃xff0c;MediaEncoder依赖于MediaCapture,MediaPushStreamer依赖MediaEncoder的相互关系。VideoCapture接收视频数据缓存至videoCaptureframeQueue&＃xff0c;AudioCapture接收音频数据缓存至audioCaptureframeQueue&＃xff0c;这样RtmpPushStreamer就可以调用MediaEncoder完成音视频编码&＃xff0c;并推流。

MediaPlus>>app.mobile.nativeapp.com.libmedia.core.streamer.RtmpPushStreamer,InitNative()中调用了 initCapture()用于初始化接收音视频数据的两个类及initEncoder()初始化音视频编码器&＃xff0c;当调用startPushStream开始直播推流时&＃xff0c;经JNI方法LiveJniMediaManager.StartPush(pushUrl)开始底层编码推流。

/*** 初始化底层采集与编码器*/private boolean InitNative() {if (!initCapture()) {return false;}if (!initEncoder()) {return false;}Log.d("initNative", "native init success!");nativeInt &＃61; true;return nativeInt;}/*** 开启推流* &＃64;param pushUrl* &＃64;return*/private boolean startPushStream(String pushUrl) {if (nativeInt) {int ret &＃61; 0;ret &＃61; LiveJniMediaManager.StartPush(pushUrl);if (ret <0) {Log.d("initNative", "native push failed!");return false;}return true;}return false;}

以下是开启推流时的JNI层调用&＃xff1a;

*** 开始推流*/ JNIEXPORT jint JNICALL Java_app_mobile_nativeapp_com_libmedia_core_jni_LiveJniMediaManager_StartPush(JNIEnv *env,jclass type,jstring url_) {mMutex.lock();if (videoCaptureInit && audioCaptureInit) {startStream &＃61; true;isClose &＃61; false;videoCapture->StartCapture();audioCapture->StartCapture();const char *url &＃61; env->GetStringUTFChars(url_, 0);rtmpStreamer &＃61; RtmpStreamer::Get();//初始化推流器if (rtmpStreamer->InitStreamer(url) !&＃61; 0) {LOG_D(DEBUG, "jni initStreamer success!");mMutex.unlock();return -1;}rtmpStreamer->SetVideoEncoder(videoEncoder);rtmpStreamer->SetAudioEncoder(audioEncoder);if (rtmpStreamer->StartPushStream() !&＃61; 0) {LOG_D(DEBUG, "jni push stream failed!");videoCapture->CloseCapture();audioCapture->CloseCapture();rtmpStreamer->ClosePushStream();mMutex.unlock();return -1;}LOG_D(DEBUG, "jni push stream success!");env->ReleaseStringUTFChars(url_, url);}mMutex.unlock();return 0; }

AudioCapture\VideoCapture用于接收应用层传入的音视频数据及采集参数&＃xff0c;libyuv转换的I420&＃xff0c;LiveJniMediaManager.StartPush(pushUrl)调用后&＃xff0c; videoCapture->StartCapture() VideoCapture就可以接收到上层传入音视频数据&＃xff0c;

LiveJniMediaManager.EncodeH264(videoBuffer, length);JNIEXPORT jint JNICALL Java_app_mobile_nativeapp_com_libmedia_core_jni_LiveJniMediaManager_EncodeH264(JNIEnv *env,jclass type,jbyteArray videoBuffer_,jint length) {if (videoCaptureInit && !isClose) {jbyte *videoSrc &＃61; env->GetByteArrayElements(videoBuffer_, 0);uint8_t *videoDstData &＃61; (uint8_t *) malloc(length);memcpy(videoDstData, videoSrc, length);OriginData *videoOriginData &＃61; new OriginData();videoOriginData->size &＃61; length;videoOriginData->data &＃61; videoDstData;videoCapture->PushVideoData(videoOriginData);env->ReleaseByteArrayElements(videoBuffer_, videoSrc, 0);}return 0; }

VideoCapture接收到数据后缓存至同步队列&＃xff1a;

/*** 往队列中添加视频数据*/ int VideoCapture::PushVideoData(OriginData *originData) {if (ExitCapture) {return 0;}originData->pts &＃61; av_gettime();LOG_D(DEBUG,"video capture pts :%lld",originData->pts);videoCaputureframeQueue.push(originData);return originData->size; }

libmedia/src/main/cpp/core/VideoEncoder.cpp
libmedia/src/main/cpp/core/RtmpStreamer.cpp
这两个类是核心&＃xff0c;前者负责编码视频&＃xff0c;后者用于Rtmp推流,从前面的JNI调用开始推流 rtmpStreamer->SetVideoEncoder(videoEncoder)&＃xff0c;可以看出来RtmpStreamer依赖VideoEncoder类&＃xff0c;接下来说明下相互间如何完成编码及推流&＃xff1a;

/** * 视频编码任务 */ void *RtmpStreamer::PushVideoStreamTask(void *pObj) {RtmpStreamer *rtmpStreamer &＃61; (RtmpStreamer *) pObj;rtmpStreamer->isPushStream &＃61; true;if (NULL &＃61;&＃61; rtmpStreamer->videoEncoder) {return 0;}VideoCapture *pVideoCapture &＃61; rtmpStreamer->videoEncoder->GetVideoCapture();AudioCapture *pAudioCapture &＃61; rtmpStreamer->audioEncoder->GetAudioCapture();if (NULL &＃61;&＃61; pVideoCapture) {return 0;}int64_t beginTime &＃61; av_gettime();int64_t lastAudioPts &＃61; 0;while (true) {if (!rtmpStreamer->isPushStream ||pVideoCapture->GetCaptureState()) {break;}OriginData *pVideoData &＃61; pVideoCapture->GetVideoData(); // OriginData *pAudioData &＃61; pAudioCapture->GetAudioData();//h264 encodeif (pVideoData !&＃61; NULL && pVideoData->data) { // if(pAudioData&&pAudioData->pts>pVideoData->pts){ // int64_t overValue&＃61;pAudioData->pts-pVideoData->pts; // pVideoData->pts&＃43;&＃61;overValue&＃43;1000; // LOG_D(DEBUG, "synchronized video audio pts videoPts:%lld audioPts:%lld", pVideoData->pts,pAudioData->pts); // }pVideoData->pts &＃61; pVideoData->pts - beginTime;LOG_D(DEBUG, "before video encode pts:%lld", pVideoData->pts);rtmpStreamer->videoEncoder->EncodeH264(&pVideoData);LOG_D(DEBUG, "after video encode pts:%lld", pVideoData->avPacket->pts);}if (pVideoData !&＃61; NULL && pVideoData->avPacket->size > 0) {rtmpStreamer->SendFrame(pVideoData, rtmpStreamer->videoStreamIndex);}}return 0; }int RtmpStreamer::StartPushStream() {videoStreamIndex &＃61; AddStream(videoEncoder->videoCodecContext);audioStreamIndex &＃61; AddStream(audioEncoder->audioCodecContext);pthread_create(&t3, NULL, RtmpStreamer::WriteHead, this);pthread_join(t3, NULL);VideoCapture *pVideoCapture &＃61; videoEncoder->GetVideoCapture();AudioCapture *pAudioCapture &＃61; audioEncoder->GetAudioCapture();pVideoCapture->videoCaputureframeQueue.clear();pAudioCapture->audioCaputureframeQueue.clear();if(writeHeadFinish) {pthread_create(&t1, NULL, RtmpStreamer::PushAudioStreamTask, this);pthread_create(&t2, NULL, RtmpStreamer::PushVideoStreamTask, this);}else{return -1;} // pthread_create(&t2, NULL, RtmpStreamer::PushStreamTask, this); // pthread_create(&t2, NULL, RtmpStreamer::PushStreamTask, this);return 0; }

rtmpStreamer->StartPushStream()调用了&＃xff0c;RtmpStreamer::StartPushStream();
在RtmpStreamer::StartPushStream()中&＃xff0c;开起新的线程:

pthread_create(&t1, NULL, RtmpStreamer::PushAudioStreamTask, this);pthread_create(&t2, NULL, RtmpStreamer::PushVideoStreamTask, this);

在PushVideoStreamTask主要有以下调用:

从VideoCapture队列中获取缓存的数据pVideoCapture->GetVideoData().
计算PTS&＃xff1a;pVideoData->pts &＃61; pVideoData->pts - beginTime.
编码器完成编码:rtmpStreamer->videoEncoder->EncodeH264(&pVideoData).
rtmpStreamer->SendFrame(pVideoData, rtmpStreamer->videoStreamIndex) 完成推流.

这样就完成了编码与推流的整个流程&＃xff0c;那么是如何完成编码的?
因为在开启推流之前&＃xff0c;就已经初始化了编码器&＃xff0c;所以RtmpStreamer只需要调用VideoEncoder编码&＃xff0c;其实VideoCapture,RtmpStreamer二者就是生产者与消费者的模式。
VideoEncoder::EncodeH264();正是完成了推流前的重要部分-视频编码。

int VideoEncoder::EncodeH264(OriginData **originData) {av_image_fill_arrays(outputYUVFrame->data,outputYUVFrame->linesize, (*originData)->data,AV_PIX_FMT_YUV420P, videoCodecContext->width,videoCodecContext->height, 1);outputYUVFrame->pts &＃61; (*originData)->pts;int ret &＃61; 0;ret &＃61; avcodec_send_frame(videoCodecContext, outputYUVFrame);if (ret !&＃61; 0) { #ifdef SHOW_DEBUG_INFOLOG_D(DEBUG, "avcodec video send frame failed"); #endif}av_packet_unref(&videoPacket);ret &＃61; avcodec_receive_packet(videoCodecContext, &videoPacket);if (ret !&＃61; 0) { #ifdef SHOW_DEBUG_INFOLOG_D(DEBUG, "avcodec video recieve packet failed"); #endif}(*originData)->Drop();(*originData)->avPacket &＃61; &videoPacket; #ifdef SHOW_DEBUG_INFOLOG_D(DEBUG, "encode video packet size:%d pts:%lld", (*originData)->avPacket->size,(*originData)->avPacket->pts);LOG_D(DEBUG, "Video frame encode success!"); #endif(*originData)->avPacket->size;return videoPacket.size; }

以上就是H264编码的核心代码了&＃xff0c;填充AVFrame&＃xff0c;再完成编码&＃xff0c;AVFrame data中存储的是编码前的数据&＃xff0c;经编码后AVPacket data中存储的是压缩编码后的数据&＃xff0c;再通过 RtmpStreamer::SendFrame()将编码后的数据发送出去。发送过程中&＃xff0c;需要转换PTS&＃xff0c;DTS时间基数&＃xff0c;将本地编码器的时间基数&＃xff0c;转换为AVStream中的时间基数。

int RtmpStreamer::SendFrame(OriginData *pData, int streamIndex) {std::lock_guard lk(mut1);AVRational stime;AVRational dtime;AVPacket *packet &＃61; pData->avPacket;packet->stream_index &＃61; streamIndex;LOG_D(DEBUG, "write packet index:%d index:%d pts:%lld", packet->stream_index, streamIndex,packet->pts);//判断是音频还是视频if (packet->stream_index &＃61;&＃61; videoStreamIndex) {stime &＃61; videoCodecContext->time_base;dtime &＃61; videoStream->time_base;}else if (packet->stream_index &＃61;&＃61; audioStreamIndex) {stime &＃61; audioCodecContext->time_base;dtime &＃61; audioStream->time_base;}else {LOG_D(DEBUG, "unknow stream index");return -1;}packet->pts &＃61; av_rescale_q(packet->pts, stime, dtime);packet->dts &＃61; av_rescale_q(packet->dts, stime, dtime);packet->duration &＃61; av_rescale_q(packet->duration, stime, dtime);int ret &＃61; av_interleaved_write_frame(iAvFormatContext, packet);if (ret &＃61;&＃61; 0) {if (streamIndex &＃61;&＃61; audioStreamIndex) {LOG_D(DEBUG, "---------->write &＃64;&＃64;&＃64;&＃64;&＃64;&＃64;&＃64;&＃64;&＃64; frame success------->!");} else if (streamIndex &＃61;&＃61; videoStreamIndex) {LOG_D(DEBUG, "---------->write ######### frame success------->!");}} else {char buf[1024] &＃61; {0};av_strerror(ret, buf, sizeof(buf));LOG_D(DEBUG, "stream index %d writer frame failed! :%s", streamIndex, buf);}return 0; }