0.引言
阅读本文前,可以先阅读前面的文章,能够帮助你更好理解本篇文章。文章列表如下:
SRS流媒体服务器之HLS源码分析(1)
SRS流媒体服务器之HLS源码分析(2)
SRS流媒体服务器之HLS配置、测试和技术选型
SRS流媒体服务器集群之Edge模式(3)
SRS流媒体服务器集群之Edge模式(2)
SRS流媒体服务器集群之Edge模式(1)
SRS流媒体服务器集群之Forward模式(2)
SRS流媒体服务器集群之Forward模式(1)
SRS流媒体服务器之HTTP-FLV框架分析(1)
SRS流媒体服务器之RTMP推流消息处理(1)
SRS流媒体服务器之RTMP协议分析(2)
SRS流媒体框架分析(1)
SRS流媒体之RTMP推流框架分析(2)
SRS流媒体之RTMP拉流框架分析(3)
SRS流媒体服务器之RTMP协议分析(1)
简述SRS流媒体服务器相关技术
流媒体推拉流实战之RTMP协议分析(BAT面试官推荐)
流媒体服务器架构与应用分析
手把手搭建流媒体服务器详细步骤
手把手搭建FFmpeg的Windows环境
超详细手把手搭建在ubuntu系统的FFmpeg环境
本篇文章主要讲解的是在推流时,音频数据是如何通过RTMP协议转化为HLS的ts封装,视频也是类似。以源码的方式进行分析。
1.源码分析
SrsSharedPtrMessage是rtmp的消息格式。
srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg){ srs_error_t err &#61; srs_success; bool is_aac_sequence_header &#61; SrsFlvAudio::sh(msg->payload, msg->size); bool is_sequence_header &#61; is_aac_sequence_header; // whether consumer should drop for the duplicated sequence header. bool drop_for_reduce &#61; false; if (is_sequence_header && meta->previous_ash() && _srs_config->get_reduce_sequence_header(req->vhost)) { if (meta->previous_ash()->size &#61;&#61; msg->size) { drop_for_reduce &#61; srs_bytes_equals(meta->previous_ash()->payload, msg->payload, msg->size); srs_warn("drop for reduce sh audio, size&#61;%d", msg->size); } } // copy to all consumer if (!drop_for_reduce) { for (int i &#61; 0; i <(int)consumers.size(); i&#43;&#43;) { SrsConsumer* consumer &#61; consumers.at(i); if ((err &#61; consumer->enqueue(msg, atc, jitter_algorithm)) !&#61; srs_success) { return srs_error_wrap(err, "consume message"); } } } // Copy to hub to all utilities. if ((err &#61; hub->on_audio(msg)) !&#61; srs_success) { return srs_error_wrap(err, "consume audio"); } // cache the sequence header of aac, or first packet of mp3. // for example, the mp3 is used for hls to write the "right" audio codec. // TODO: FIXME: to refine the stream info system. if (is_aac_sequence_header || !meta->ash()) { if ((err &#61; meta->update_ash(msg)) !&#61; srs_success) { return srs_error_wrap(err, "meta consume audio"); } } // when sequence header, donot push to gop cache and adjust the timestamp. if (is_sequence_header) { return err; } // cache the last gop packets if ((err &#61; gop_cache->cache(msg)) !&#61; srs_success) { return srs_error_wrap(err, "gop cache consume audio"); } // if atc, update the sequence header to abs time. if (atc) { if (meta->ash()) { meta->ash()->timestamp &#61; msg->timestamp; } if (meta->data()) { meta->data()->timestamp &#61; msg->timestamp; } } return err;}
把Rtmp封装好的数据&#xff0c;转为音视频裸数据。
srs_error_t SrsOriginHub::on_audio(SrsSharedPtrMessage* shared_audio){ srs_error_t err &#61; srs_success; SrsSharedPtrMessage* msg &#61; shared_audio; //转换封装格式 if ((err &#61; format->on_audio(msg)) !&#61; srs_success) { return srs_error_wrap(err, "format consume audio"); } // cache the sequence header if aac // donot cache the sequence header to gop_cache, return here. if (format->is_aac_sequence_header()) { srs_assert(format->acodec); SrsAudioCodecConfig* c &#61; format->acodec; static int flv_sample_sizes[] &#61; {8, 16, 0}; static int flv_sound_types[] &#61; {1, 2, 0}; // when got audio stream info. SrsStatistic* stat &#61; SrsStatistic::instance(); if ((err &#61; stat->on_audio_info(req, SrsAudioCodecIdAAC, c->sound_rate, c->sound_type, c->aac_object)) !&#61; srs_success) { return srs_error_wrap(err, "stat audio"); } srs_trace("%dB audio sh, codec(%d, profile&#61;%s, %dchannels, %dkbps, %dHZ), flv(%dbits, %dchannels, %dHZ)", msg->size, c->id, srs_aac_object2str(c->aac_object).c_str(), c->aac_channels, c->audio_data_rate / 1000, srs_aac_srates[c->aac_sample_rate], flv_sample_sizes[c->sound_size], flv_sound_types[c->sound_type], srs_flv_srates[c->sound_rate]); } if ((err &#61; hls->on_audio(msg, format)) !&#61; srs_success) { // apply the error strategy for hls. // &#64;see https://github.com/ossrs/srs/issues/264 std::string hls_error_strategy &#61; _srs_config->get_hls_on_error(req->vhost); if (srs_config_hls_is_on_error_ignore(hls_error_strategy)) { srs_warn("hls: ignore audio error %s", srs_error_desc(err).c_str()); hls->on_unpublish(); srs_error_reset(err); } else if (srs_config_hls_is_on_error_continue(hls_error_strategy)) { if (srs_hls_can_continue(srs_error_code(err), source->meta->ash(), msg)) { srs_error_reset(err); } else { return srs_error_wrap(err, "hls: audio"); } } else { return srs_error_wrap(err, "hls: audio"); } } if ((err &#61; dash->on_audio(msg, format)) !&#61; srs_success) { srs_warn("dash: ignore audio error %s", srs_error_desc(err).c_str()); srs_error_reset(err); dash->on_unpublish(); } if ((err &#61; dvr->on_audio(msg, format)) !&#61; srs_success) { srs_warn("dvr: ignore audio error %s", srs_error_desc(err).c_str()); srs_error_reset(err); dvr->on_unpublish(); } #ifdef SRS_AUTO_HDS if ((err &#61; hds->on_audio(msg)) !&#61; srs_success) { srs_warn("hds: ignore audio error %s", srs_error_desc(err).c_str()); srs_error_reset(err); hds->on_unpublish(); }#endif // copy to all forwarders. if (true) { std::vector::iterator it; for (it &#61; forwarders.begin(); it !&#61; forwarders.end(); &#43;&#43;it) { SrsForwarder* forwarder &#61; *it; if ((err &#61; forwarder->on_audio(msg)) !&#61; srs_success) { return srs_error_wrap(err, "forward: audio"); } } } return err;}
转换封装格式&#xff0c;提取音视频裸流数据&#xff0c;源码如下:
srs_error_t SrsRtmpFormat::on_audio(SrsSharedPtrMessage* shared_audio){ SrsSharedPtrMessage* msg &#61; shared_audio; //主要是取出msg的数据部分 char* data &#61; msg->payload; int size &#61; msg->size; return SrsFormat::on_audio(msg->timestamp, data, size);}
由这种SrsSharedPtrMessage格式的数据转换为新的SrsFormat格式的数据。
srs_error_t SrsFormat::on_audio(int64_t timestamp, char* data, int size){ srs_error_t err &#61; srs_success; if (!data || size <&#61; 0) { srs_trace("no audio present, ignore it."); return err; } SrsBuffer* buffer &#61; new SrsBuffer(data, size); SrsAutoFree(SrsBuffer, buffer); // We already checked the size is positive and data is not NULL. srs_assert(buffer->require(1)); // &#64;see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 uint8_t v &#61; buffer->read_1bytes(); SrsAudioCodecId codec &#61; (SrsAudioCodecId)((v >> 4) & 0x0f); if (codec !&#61; SrsAudioCodecIdMP3 && codec !&#61; SrsAudioCodecIdAAC) { return err; } if (!acodec) { acodec &#61; new SrsAudioCodecConfig(); } if (!audio) { audio &#61; new SrsAudioFrame(); } if ((err &#61; audio->initialize(acodec)) !&#61; srs_success) { return srs_error_wrap(err, "init audio"); } // Parse by specified codec. buffer->skip(-1 * buffer->pos()); if (codec &#61;&#61; SrsAudioCodecIdMP3) { return audio_mp3_demux(buffer, timestamp); } return audio_aac_demux(buffer, timestamp);}
以aac距离&#xff0c;解封装&#xff0c;并加入aac头&#xff0c;写入cache中。
srs_error_t SrsFrame::add_sample(char* bytes, int size){ srs_error_t err &#61; srs_success; if (nb_samples >&#61; SrsMaxNbSamples) { return srs_error_new(ERROR_HLS_DECODE_ERROR, "Frame samples overflow"); } SrsSample* sample &#61; &samples[nb_samples&#43;&#43;]; sample->bytes &#61; bytes; sample->size &#61; size; return err;}
添加aac或其他格式数据。
srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts){ srs_error_t err &#61; srs_success; // write audio to cache. if ((err &#61; tsmc->cache_audio(frame, pts)) !&#61; srs_success) { return srs_error_wrap(err, "hls: cache audio"); } // reap when current source is pure audio. // it maybe changed when stream info changed, // for example, pure audio when start, audio/video when publishing, // pure audio again for audio disabled. // so we reap event when the audio incoming when segment overflow. // &#64;see https://github.com/ossrs/srs/issues/151 // we use absolutely overflow of segment to make jwplayer/ffplay happy // &#64;see https://github.com/ossrs/srs/issues/151#issuecomment-71155184 if (tsmc->audio && muxer->is_segment_absolutely_overflow()) { if ((err &#61; reap_segment()) !&#61; srs_success) { return srs_error_wrap(err, "hls: reap segment"); } } // for pure audio, aggregate some frame to one. // TODO: FIXME: Check whether it&#39;s necessary. if (muxer->pure_audio() && tsmc->audio) { if (pts - tsmc->audio->start_pts
先写数据到cache中&#xff0c;最后把cache的数据合并写入ts文件中。
srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts){ srs_error_t err &#61; srs_success; // write audio to cache. if ((err &#61; tsmc->cache_audio(frame, pts)) !&#61; srs_success) { return srs_error_wrap(err, "hls: cache audio"); } // reap when current source is pure audio. // it maybe changed when stream info changed, // for example, pure audio when start, audio/video when publishing, // pure audio again for audio disabled. // so we reap event when the audio incoming when segment overflow. // &#64;see https://github.com/ossrs/srs/issues/151 // we use absolutely overflow of segment to make jwplayer/ffplay happy // &#64;see https://github.com/ossrs/srs/issues/151#issuecomment-71155184 if (tsmc->audio && muxer->is_segment_absolutely_overflow()) { if ((err &#61; reap_segment()) !&#61; srs_success) { return srs_error_wrap(err, "hls: reap segment"); } } // for pure audio, aggregate some frame to one. // TODO: FIXME: Check whether it&#39;s necessary. if (muxer->pure_audio() && tsmc->audio) { if (pts - tsmc->audio->start_pts
写音频数据到cache中。
srs_error_t SrsTsMessageCache::cache_audio(SrsAudioFrame* frame, int64_t dts){ srs_error_t err &#61; srs_success; // create the ts audio message. if (!audio) { audio &#61; new SrsTsMessage(); audio->write_pcr &#61; false; audio->dts &#61; audio->pts &#61; audio->start_pts &#61; dts; } // TODO: FIXME: refine code. //audio->dts &#61; dts; //audio->pts &#61; audio->dts; audio->sid &#61; SrsTsPESStreamIdAudioCommon; // must be aac or mp3 SrsAudioCodecConfig* acodec &#61; frame->acodec(); srs_assert(acodec->id &#61;&#61; SrsAudioCodecIdAAC || acodec->id &#61;&#61; SrsAudioCodecIdMP3); // write video to cache. if (acodec->id &#61;&#61; SrsAudioCodecIdAAC) { if ((err &#61; do_cache_aac(frame)) !&#61; srs_success) { return srs_error_wrap(err, "ts: cache aac"); } } else { if ((err &#61; do_cache_mp3(frame)) !&#61; srs_success) { return srs_error_wrap(err, "ts: cache mp3"); } } return err;}
以aac举例&#xff0c;写cache时&#xff0c;注意写ADTS头:
srs_error_t SrsTsContextWriter::write_audio(SrsTsMessage* audio){ srs_error_t err &#61; srs_success; srs_info("hls: write audio pts&#61;%" PRId64 ", dts&#61;%" PRId64 ", size&#61;%d", audio->pts, audio->dts, audio->PES_packet_length); if ((err &#61; context->encode(writer, audio, vcodec, acodec)) !&#61; srs_success) { return srs_error_wrap(err, "ts: write audio"); } srs_info("hls encode audio ok"); return err;}
按照ts的封装格式&#xff0c;写入文件:
// SrsTsMessage* msg指向的数据&#xff0c;存储的是裸流数据&#xff0c;这里的AAC(带ADTS头)srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, SrsVideoCodecId vc, SrsAudioCodecId ac){ srs_error_t err &#61; srs_success; SrsTsStream vs, as; int16_t video_pid &#61; 0, audio_pid &#61; 0; switch (vc) { case SrsVideoCodecIdAVC: vs &#61; SrsTsStreamVideoH264; video_pid &#61; TS_VIDEO_AVC_PID; break; case SrsVideoCodecIdDisabled: vs &#61; SrsTsStreamReserved; break; case SrsVideoCodecIdReserved: case SrsVideoCodecIdReserved1: case SrsVideoCodecIdReserved2: case SrsVideoCodecIdSorensonH263: case SrsVideoCodecIdScreenVideo: case SrsVideoCodecIdOn2VP6: case SrsVideoCodecIdOn2VP6WithAlphaChannel: case SrsVideoCodecIdScreenVideoVersion2: case SrsVideoCodecIdHEVC: case SrsVideoCodecIdAV1: vs &#61; SrsTsStreamReserved; break; } switch (ac) { case SrsAudioCodecIdAAC: as &#61; SrsTsStreamAudioAAC; audio_pid &#61; TS_AUDIO_AAC_PID; break; case SrsAudioCodecIdMP3: as &#61; SrsTsStreamAudioMp3; audio_pid &#61; TS_AUDIO_MP3_PID; break; case SrsAudioCodecIdDisabled: as &#61; SrsTsStreamReserved; break; case SrsAudioCodecIdReserved1: case SrsAudioCodecIdLinearPCMPlatformEndian: case SrsAudioCodecIdADPCM: case SrsAudioCodecIdLinearPCMLittleEndian: case SrsAudioCodecIdNellymoser16kHzMono: case SrsAudioCodecIdNellymoser8kHzMono: case SrsAudioCodecIdNellymoser: case SrsAudioCodecIdReservedG711AlawLogarithmicPCM: case SrsAudioCodecIdReservedG711MuLawLogarithmicPCM: case SrsAudioCodecIdReserved: case SrsAudioCodecIdSpeex: case SrsAudioCodecIdReservedMP3_8kHz: case SrsAudioCodecIdReservedDeviceSpecificSound: case SrsAudioCodecIdOpus: as &#61; SrsTsStreamReserved; break; } if (as &#61;&#61; SrsTsStreamReserved && vs &#61;&#61; SrsTsStreamReserved) { return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec&#61;%d, acodec&#61;%d", vc, ac); } // when any codec changed, write PAT/PMT table. //PAT和PMT表 if (vcodec !&#61; vc || acodec !&#61; ac) { vcodec &#61; vc; acodec &#61; ac; if ((err &#61; encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) !&#61; srs_success) { return srs_error_wrap(err, "ts: encode PAT/PMT"); } } // encode the media frame to PES packets over TS. if (msg->is_audio()) { //编码为PES包 return encode_pes(writer, msg, audio_pid, as, vs &#61;&#61; SrsTsStreamReserved); } else { return encode_pes(writer, msg, video_pid, vs, vs &#61;&#61; SrsTsStreamReserved); }}
使用上下文SrsTsContext的方法进行封装&#xff0c;这里的源码就是按照ts格式进行写入&#xff0c;比如这里先封装成pes数据。也可以参考关于ts的文章以及spec。
// SrsTsMessage* msg指向的数据&#xff0c;存储的是裸流数据&#xff0c;这里的AAC(带ADTS头)srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, SrsVideoCodecId vc, SrsAudioCodecId ac){ srs_error_t err &#61; srs_success; SrsTsStream vs, as; int16_t video_pid &#61; 0, audio_pid &#61; 0; switch (vc) { case SrsVideoCodecIdAVC: vs &#61; SrsTsStreamVideoH264; video_pid &#61; TS_VIDEO_AVC_PID; break; case SrsVideoCodecIdDisabled: vs &#61; SrsTsStreamReserved; break; case SrsVideoCodecIdReserved: case SrsVideoCodecIdReserved1: case SrsVideoCodecIdReserved2: case SrsVideoCodecIdSorensonH263: case SrsVideoCodecIdScreenVideo: case SrsVideoCodecIdOn2VP6: case SrsVideoCodecIdOn2VP6WithAlphaChannel: case SrsVideoCodecIdScreenVideoVersion2: case SrsVideoCodecIdHEVC: case SrsVideoCodecIdAV1: vs &#61; SrsTsStreamReserved; break; } switch (ac) { case SrsAudioCodecIdAAC: as &#61; SrsTsStreamAudioAAC; audio_pid &#61; TS_AUDIO_AAC_PID; break; case SrsAudioCodecIdMP3: as &#61; SrsTsStreamAudioMp3; audio_pid &#61; TS_AUDIO_MP3_PID; break; case SrsAudioCodecIdDisabled: as &#61; SrsTsStreamReserved; break; case SrsAudioCodecIdReserved1: case SrsAudioCodecIdLinearPCMPlatformEndian: case SrsAudioCodecIdADPCM: case SrsAudioCodecIdLinearPCMLittleEndian: case SrsAudioCodecIdNellymoser16kHzMono: case SrsAudioCodecIdNellymoser8kHzMono: case SrsAudioCodecIdNellymoser: case SrsAudioCodecIdReservedG711AlawLogarithmicPCM: case SrsAudioCodecIdReservedG711MuLawLogarithmicPCM: case SrsAudioCodecIdReserved: case SrsAudioCodecIdSpeex: case SrsAudioCodecIdReservedMP3_8kHz: case SrsAudioCodecIdReservedDeviceSpecificSound: case SrsAudioCodecIdOpus: as &#61; SrsTsStreamReserved; break; } if (as &#61;&#61; SrsTsStreamReserved && vs &#61;&#61; SrsTsStreamReserved) { return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec&#61;%d, acodec&#61;%d", vc, ac); } // when any codec changed, write PAT/PMT table. //PAT和PMT表 if (vcodec !&#61; vc || acodec !&#61; ac) { vcodec &#61; vc; acodec &#61; ac; if ((err &#61; encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) !&#61; srs_success) { return srs_error_wrap(err, "ts: encode PAT/PMT"); } } // encode the media frame to PES packets over TS. if (msg->is_audio()) { //编码为PES包 return encode_pes(writer, msg, audio_pid, as, vs &#61;&#61; SrsTsStreamReserved); } else { return encode_pes(writer, msg, video_pid, vs, vs &#61;&#61; SrsTsStreamReserved); }}
最终到hls流时&#xff0c;都是已经经过处理过的数据。
断点调试&#xff0c;输入如下命令:
b SrsFrame::add_sample()
c
如下界面:
查看调用栈(调用关系是从下到上&#xff0c;即15到0&#xff0c;运行到断点处)&#xff0c;如下界面:
0 SrsFrame::add_sample at src/kernel/srs_kernel_codec.cpp:4521 SrsFormat::audio_aac_demux at src/kernel/srs_kernel_codec.cpp:1252 in SrsFormat::on_audio (this&#61;0xa3b560, frame&#61;0xa41640, pts&#61;3060) at src/kernel/srs_kernel_codec.cpp:5873 in SrsRtmpFormat::on_audio (this&#61;0xa3b030, shared_audio&#61;0x7ffff7ee8b10, format&#61;0xa3bad0) at src/protocol/srs_protocol_format.cpp:534 0x00000000004deeaf in SrsOriginHub::on_audio (this&#61;0xa3b670, shared_audio&#61;0x7ffff7ee8b10) at src/app/srs_app_source.cpp:9695 0x00000000004e579a in SrsSource::on_audio_imp (this&#61;0xa3b1e0, msg&#61;0x7ffff7ee8b10) at src/app/srs_app_source.cpp:21916 0x00000000004e539d in SrsSource::on_audio (this&#61;0xa3b1e0, shared_audio&#61;0xa3dfa0) at src/app/srs_app_source.cpp:21417 0x00000000004d8f28 in SrsRtmpConn::process_publish_message (this&#61;0xa30d60, source&#61;0xa3b1e0, msg&#61;0xa3dfa0) at src/app/srs_app_rtmp_conn.cpp:10148 0x00000000004d8dce in SrsRtmpConn::handle_publish_message (this&#61;0xa30d60, source&#61;0xa3b1e0, msg&#61;0xa3dfa0) at src/app/srs_app_rtmp_conn.cpp:9939 0x00000000005810b6 in SrsPublishRecvThread::consume (this&#61;0x7ffff7f42800, msg&#61;0xa3dfa0) at src/app/srs_app_recv_thread.cpp:38910 0x000000000057fbd4 in SrsRecvThread::do_cycle (this&#61;0x7ffff7f42808)---Type to continue, or q to quit--- at src/app/srs_app_recv_thread.cpp:14611 0x000000000057fa25 in SrsRecvThread::cycle (this&#61;0x7ffff7f42808) at src/app/srs_app_recv_thread.cpp:11512 0x0000000000509c88 in SrsSTCoroutine::cycle (this&#61;0xa3de00) at src/app/srs_app_st.cpp:19813 0x0000000000509cfd in SrsSTCoroutine::pfn (arg&#61;0xa3de00) at src/app/srs_app_st.cpp:21314 _st_thread_main() at sched.c:33715 st_thread_create at sched.c:616
在hls处理音视频均打上断点&#xff0c;输入命令:
b SrsHls::on_video()
b SrsHls::on_audio()
c
界面如下:
打印音频封装格式&#xff0c;输入如下命令&#xff0c;如下界面:
print *format
表示一组采样数量&#xff0c;可能有多个帧:
print format->audio->nb_samples
通过一层层的源码分析&#xff0c;封装格式的变化&#xff0c;都没有使用深拷贝&#xff0c;都是通过指针的方式&#xff0c;进行浅拷贝。
2.总结
本篇文章主要是讲解在SRS源码中是如何将音频的转封装格式进行转换&#xff0c;视频也是类似&#xff0c;希望能够帮到大家。欢迎关注&#xff0c;转发&#xff0c;点赞&#xff0c;收藏&#xff0c;分享&#xff0c;评论区讨论。
后期关于项目的知识&#xff0c;会在微信公众号上更新&#xff0c;如果想要学习项目&#xff0c;可以关注微信公众号“记录世界 from antonio”