详解RTP协议之H264封包和解包实战

0.引言

为了更好理解本篇文章，可以先阅读前面几篇文章，文章列表如下:

详解RTP协议之H264封包细节(1)

详细解析RTSP框架和数据包分析(1)

手把手搭建RTSP流媒体服务器

RTP协议

HLS实战之Wireshark抓包分析

HTTP实战之Wireshark抓包分析

建议:阅读本文前，一定要阅读前面的文章，只有理解了原理，才能够正真读懂代码。

1.RTP实战源码框架

在win上主要是支持qt，使用2015编译器。linux上支持cmake编译。主要是支持跨平台支持。

使用函数int get_annexb_nalu (nalu_t *nalu, FILE *bits)，一开始是读取本地h264文件，解析出不带startcode的nalu，接着是函数static void rtp_h264_pack_get_info(void* pack, uint16_t* seq, uint32_t* timestamp)，经过函数就会得到RTP包，处理RTP包就有2个流程，第一个是通过网络发送出去，然后播放。另外一个流程是通过RTP_unpack去解码，生成nalu，再加上h264的start code，就可以存储在本地文件，然后再播放。框架如下图所示:

main.c是用户自定义发送和写文件，测试函数。

rtp-packet.c是RTP Packet序列化和反序列化功能。

rtp-payload.c主要是RTP封包和拆包的接口层。

rtp-h264-packet.c主要是H264封包RTP的实现层。

rtp-h264-unpacket.c主要是H264拆包RTP的实现层。

这里以视频数据为例子，整个源码的数据对象的关系是，在封包流程中，nalu指向rtp结构，再指向rtp序列化，最终才是sendto的数据。在解封装的流程中，recv的数据指向rtp反序列化，再到rtp结构，最终解析出nalu，加上startcode，存在本地文件，保存为h264，可以播放。

2.RTP实战测试效果

(1)播放命令

ffplay h264.sdp -protocol_whitelist "file,http,https,rtp,udp,tcp,tls"

界面如下:

(2)注意，需要在如下的目录下，准备好输入h264文件，即把h264文件放到这个debug目录，因为代码中需要打开输入的h264文件。

(3)h264.sdp是代码生成，整个日志打印过程，其界面如下:

3.源码详解

3.1 初始化参数和回调函数注册

(1)rtp header头部定义。按照前面文章描述的头部结构来定义。封包流程中，注册函数，源码如下:

/**
 * @brief rtp_payload_encode_input 这里是通用的接口
 * @param encoder
 * @param data      data具体是什么媒体类型的数据，接口不关注，具体由ctx->encoder->input去处理
 * @param bytes
 * @param timestamp
 * @return
 */
int rtp_payload_encode_input(void* encoder, const void* data, int bytes, uint32_t timestamp)
{
    struct rtp_payload_delegate_t* ctx;
    ctx = (struct rtp_payload_delegate_t*)encoder;
    return ctx->encoder->input(ctx->packer, data, bytes, timestamp);
}

RTP Header固定长度为12个字节。rtp packet(包括header+payload)的数据结构定义如下:

void* rtp_payload_decode_create(int payload, const char* name, struct rtp_payload_t *handler, void* cbparam)
{
    struct rtp_payload_delegate_t* ctx;
    ctx = calloc(1, sizeof(*ctx));
    if (ctx)
    {
        if (rtp_payload_find(payload, name, ctx) < 0
            || NULL == (ctx->packer = ctx->decoder->create(handler, cbparam)))
        {
            free(ctx);
            return NULL;
        }
    }
    return ctx;
}

(2)网络发送到本地，这里使用的UDP发送。

int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

3.2 详解 封装RTP包流程源码

(1)函数rtp_payload_encode_create(int payload, const char* name, uint16_t seq, uint32_t ssrc, struct rtp_payload_t *handler, void* cbparam)接收用户参数，并设置回调函数。源码如下:

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(2)通过如下代码可以看出，ctx->encoder->create指向的函数是 rtp_h264_pack_create，也就是正真的实现体。源码如下:

struct rtp_payload_encode_t *rtp_h264_encode()
{
    static struct rtp_payload_encode_t packer = {
        rtp_h264_pack_create,
        rtp_h264_pack_destroy,
        rtp_h264_pack_get_info,
        rtp_h264_pack_input,
    };

    return &packer;
}

(3)函数rtp_h264_pack_create，用来存储外部设置的参数。源码如下:

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(4)封包流程中，也是使用相同的回调函数，源码如下:

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(5)通过网络发送到本地。源码如下:

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(6)获取nalu的sps、pps，并封装成RTP包。源码如下:

int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(7)rtp-payload.c的函数都是接口层，正真实现体都在rtp-h264-pack.c和rtp-h264-unpack.c里。

int rtp_packet_serialize(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;

    hdrlen = rtp_packet_serialize_header(pkt, data, bytes);
    if (hdrlen < RTP_FIXED_HEADER || hdrlen + pkt->payloadlen > bytes)
        return -1;

    memcpy(((uint8_t*)data) + hdrlen, pkt->payload, pkt->payloadlen);
    return hdrlen + pkt->payloadlen;
}

(8)接口ctx->encoder->input(ctx->packer, data, bytes, timestamp)执行的实现体rtp_h264_pack_input(void* pack, const void* h264, int bytes, uint32_t timestamp)，通过如下源码，可以看出。

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(9)从这里就可以实现打包，源码如下:

int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(10)函数rtp_h264_pack_nalu(packer, p1, (int)nalu_size)意思是，正如前面文章所讲，是直接打包，就是没有切片，一个nalu打包为一个RTP包。源码如下:

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(11)函数调用rtp_packet_serialize(&packer->pkt, rtp, n)，就是一个打包的过程，就是严格按照RTP的SPEC来实现。源码如下:

// 拿到一帧RTP序列化后的数据
static int rtp_encode_packet(void* param, const void *packet, int bytes, uint32_t timestamp, int flags)
{
    struct rtp_h264_test_t* ctx = (struct rtp_h264_test_t*)param;//拿到用户传递进来的参数
    int ret = 0;
    ret = sendto(
                ctx->fd,
                (void*)packet,
                bytes,
                0,
                (struct sockaddr*)&ctx->addr,
                ctx->addr_size);
    uint8_t *nalu = (uint8_t *)packet;
    printf("rtp send packet -> nalu_type:%d,0x%02x,0x%02x, bytes:%d, timestamp:%u\n",
           nalu[12]&0x1f,  nalu[12],  nalu[13], bytes, timestamp);

    ret = rtp_payload_decode_input(ctx->decoder_h264, packet, bytes);       // 重新又


解封装

    return 0;
}

// 把可读RTP packet封装成要发送出去的数据 序列化
int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(12)把可读RTP packet封装成要发送出去的数据，与上面的函数是反向关系。这里把头部和数据分开处理。从packet读取一个个字节，然后封包去处理。

int rtp_packet_serialize_header(const struct rtp_packet_t *pkt, void* data, int bytes)
{
    int hdrlen;
    uint32_t i;
    uint8_t* ptr;

    if (RTP_VERSION != pkt->rtp.v || 0 != (pkt->extlen % 4))
    {
        assert(0); // RTP version field must equal 2 (p66)
        return -1;
    }

    // RFC3550 5.1 RTP Fixed Header Fields(p12) 是否要扩展
    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4 + (pkt->rtp.x ? 4 : 0);
    if (bytes < hdrlen + pkt->extlen)
        return -1;

    ptr = (uint8_t *)data;
    nbo_write_rtp_header(ptr, &pkt->rtp);
    ptr += RTP_FIXED_HEADER;

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++, ptr += 4)
    {
        nbo_w32(ptr, pkt->csrc[i]);     // csrc列表封装到头部
    }

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        // 5.3.1 RTP Header Extension
        assert(0 == (pkt->extlen % 4));
        nbo_w16(ptr, pkt->reserved);
        nbo_w16(ptr + 2, pkt->extlen / 4);
        memcpy(ptr + 4, pkt->extension, pkt->extlen);   // extension封装到头部
        ptr += pkt->extlen + 4;
    }

    return hdrlen + pkt->extlen;
}

(13)RTP打包H264函数，把带startcode的nalu也传递进来。如果nalu_size + RTP_FIXED_HEADER <= (size_t)packer->size，这就是前面所说的打包的一种方式，正好吻合起来，如下:

static int rtp_h264_pack_fu_a(struct rtp_encode_h264_t *packer, const uint8_t* nalu, int bytes)
{
    int r, n;
    unsigned char *rtp;

    // RFC6184 5.3. NAL Unit Header Usage: Table 2 (p15)
    // RFC6184 5.8. Fragmentation Units (FUs) (p29)
    uint8_t fu_indicator = (*nalu & 0xE0) | 28; // FU-A 固定的逻辑
    uint8_t fu_header = *nalu & 0x1F;           // 设置为s置位1

    r = 0;
    nalu += 1; // skip NAL Unit Type byte
    bytes -= 1;
    assert(bytes > 0);

    // FU-A start
    for (fu_header |= FU_START; 0 == r && bytes > 0; ++packer->pkt.rtp.seq)
    {
        if (bytes + RTP_FIXED_HEADER <= packer->size - N_FU_HEADER)
        {
            assert(0 == (fu_header & FU_START));
            fu_header = FU_END | (fu_header & 0x1F); // FU-A end  整个nalu结束了
            packer->pkt.payloadlen = bytes;
        }
        else
        {
            packer->pkt.payloadlen = packer->size - RTP_FIXED_HEADER - N_FU_HEADER;
        }

        packer->pkt.payload = nalu;
        n = RTP_FIXED_HEADER + N_FU_HEADER + packer->pkt.payloadlen;
        rtp = (uint8_t*)packer->handler.alloc(packer->cbparam, n);
        if (!rtp) return -ENOMEM;

        packer->pkt.rtp.m = (FU_END & fu_header) ? 1 : 0; // set marker flag
        n = rtp_packet_serialize_header(&packer->pkt, rtp, n);
        if (n != RTP_FIXED_HEADER)
        {
            assert(0);
            return -1;
        }

        /*fu_indicator + fu_header*/
        rtp[n + 0] = fu_indicator;
        rtp[n + 1] = fu_header;
        memcpy(rtp + n + N_FU_HEADER, packer->pkt.payload, packer->pkt.payloadlen);
        // packer->cbparam用户的参数
        r = packer->handler.packet(packer->cbparam, rtp, n + N_FU_HEADER + packer->pkt.payloadlen, packer->pkt.rtp.timestamp, 0);
        packer->handler.free(packer->cbparam, rtp);

        bytes -= packer->pkt.payloadlen;
        nalu += packer->pkt.payloadlen;
        fu_header &= 0x1F; // clear flags
    }

    return r;
}

(14)把一个比较大的nalu，分多次进行打包，使用FU的方式。代码的思想，如同前面的文章所描述。代码如下:

static int rtp_h264_pack_fu_a(struct rtp_encode_h264_t *packer, const uint8_t* nalu, int bytes)
{
    int r, n;
    unsigned char *rtp;

    // RFC6184 5.3. NAL Unit Header Usage: Table 2 (p15)
    // RFC6184 5.8. Fragmentation Units (FUs) (p29)
    uint8_t fu_indicator = (*nalu & 0xE0) | 28; // FU-A 固定的逻辑
    uint8_t fu_header = *nalu & 0x1F;           // 设置为s置位1

    r = 0;
    nalu += 1; // skip NAL Unit Type byte
    bytes -= 1;
    assert(bytes > 0);

    // FU-A start
    for (fu_header |= FU_START; 0 == r && bytes > 0; ++packer->pkt.rtp.seq)
    {
        if (bytes + RTP_FIXED_HEADER <= packer->size - N_FU_HEADER)
        {
            assert(0 == (fu_header & FU_START));
            fu_header = FU_END | (fu_header & 0x1F); // FU-A end  整个nalu结束了
            packer->pkt.payloadlen = bytes;
        }
        else
        {
            packer->pkt.payloadlen = packer->size - RTP_FIXED_HEADER - N_FU_HEADER;
        }

        packer->pkt.payload = nalu;
        n = RTP_FIXED_HEADER + N_FU_HEADER + packer->pkt.payloadlen;
        rtp = (uint8_t*)packer->handler.alloc(packer->cbparam, n);
        if (!rtp) return -ENOMEM;

        packer->pkt.rtp.m = (FU_END & fu_header) ? 1 : 0; // set marker flag
        n = rtp_packet_serialize_header(&packer->pkt, rtp, n);
        if (n != RTP_FIXED_HEADER)
        {
            assert(0);
            return -1;
        }

        /*fu_indicator + fu_header*/
        rtp[n + 0] = fu_indicator;
        rtp[n + 1] = fu_header;
        memcpy(rtp + n + N_FU_HEADER, packer->pkt.payload, packer->pkt.payloadlen);
        // packer->cbparam用户的参数
        r = packer->handler.packet(packer->cbparam, rtp, n + N_FU_HEADER + packer->pkt.payloadlen, packer->pkt.rtp.timestamp, 0);
        packer->handler.free(packer->cbparam, rtp);

        bytes -= packer->pkt.payloadlen;
        nalu += packer->pkt.payloadlen;
        fu_header &= 0x1F; // clear flags
    }

    return r;
}

(15)函数rtp_h264_pack_fu_a(struct rtp_encode_h264_t *packer, const uint8_t* nalu, int bytes)表示一个大的nalu分多次打包的过程，这个原理前面的文章，已经讲过了。源码如下:

static int rtp_h264_pack_fu_a(struct rtp_encode_h264_t *packer, const uint8_t* nalu, int bytes)
{
    int r, n;
    unsigned char *rtp;

    // RFC6184 5.3. NAL Unit Header Usage: Table 2 (p15)
    // RFC6184 5.8. Fragmentation Units (FUs) (p29)
    uint8_t fu_indicator = (*nalu & 0xE0) | 28; // FU-A 固定的逻辑
    uint8_t fu_header = *nalu & 0x1F;           // 设置为s置位1

    r = 0;
    nalu += 1; // skip NAL Unit Type byte
    bytes -= 1;
    assert(bytes > 0);

    // FU-A start
    for (fu_header |= FU_START; 0 == r && bytes > 0; ++packer->pkt.rtp.seq)
    {
        if (bytes + RTP_FIXED_HEADER <= packer->size - N_FU_HEADER)
        {
            assert(0 == (fu_header & FU_START));
            fu_header = FU_END | (fu_header & 0x1F); // FU-A end  整个nalu结束了
            packer->pkt.payloadlen = bytes;
        }
        else
        {
            packer->pkt.payloadlen = packer->size - RTP_FIXED_HEADER - N_FU_HEADER;
        }

        packer->pkt.payload = nalu;
        n = RTP_FIXED_HEADER + N_FU_HEADER + packer->pkt.payloadlen;
        rtp = (uint8_t*)packer->handler.alloc(packer->cbparam, n);
        if (!rtp) return -ENOMEM;
        //视频拆包时,nalu的结束标志
        packer->pkt.rtp.m = (FU_END & fu_header) ? 1 : 0; // set marker flag
        n = rtp_packet_serialize_header(&packer->pkt, rtp, n);
        if (n != RTP_FIXED_HEADER)
        {
            assert(0);
            return -1;
        }

        /*fu_indicator + fu_header*/
        rtp[n + 0] = fu_indicator;
        rtp[n + 1] = fu_header;
        memcpy(rtp + n + N_FU_HEADER, packer->pkt.payload, packer->pkt.payloadlen);
        // packer->cbparam用户的参数，回调用户定义的函数，前面已介绍过
        r = packer->handler.packet(packer->cbparam, rtp, n + N_FU_HEADER + packer->pkt.payloadlen, packer->pkt.rtp.timestamp, 0);
        packer->handler.free(packer->cbparam, rtp);

        bytes -= packer->pkt.payloadlen;
        nalu += packer->pkt.payloadlen;
        fu_header &= 0x1F; // clear flags
    }

    return r;
}

(16)前面的源码已经分析过，函数rtp_encode_packet(void* param, const void *packet, int bytes, uint32_t timestamp, int flags)是用户定义的函数，主要是用作发送数据和自定义封装函数(把这个两个函数写在一起不是很合适)。函数packer->handler.packet(packer->cbparam, rtp, n, packer->pkt.rtp.timestamp, 0)，这里会回调用户定义的rtp_encode_packet(void* param, const void *packet, int bytes, uint32_t timestamp, int flags)，就通过网络去发送数据到指定的IP上，我这里就是到本机。用户的参数和定义的函数就是通过 struct rtp_payload_t handler_rtp_decode_h264和struct rtp_h264_test_t ctx来完成。再回顾下这个源码，如下:

// 拿到一帧RTP序列化后的数据
static int rtp_encode_packet(void* param, const void *packet, int bytes, uint32_t timestamp, int flags)
{
    struct rtp_h264_test_t* ctx = (struct rtp_h264_test_t*)param;//拿到用户传递进来的参数
    int ret = 0;
    ret = sendto(
                ctx->fd,
                (void*)packet,
                bytes,
                0,
                (struct sockaddr*)&ctx->addr,
                ctx->addr_size);
    uint8_t *nalu = (uint8_t *)packet;
    printf("rtp send packet -> nalu_type:%d,0x%02x,0x%02x, bytes:%d, timestamp:%u\n",
           nalu[12]&0x1f,  nalu[12],  nalu[13], bytes, timestamp);
    ret = rtp_payload_decode_input(ctx->decoder_h264, packet, bytes);       // 重新又解封装
    return 0;
}

3.3 解封RTP包流程源码

(1)函数rtp_payload_decode_create(int payload, const char* name, struct rtp_payload_t *handler, void* cbparam)接收外部参数。源码如下:

void* rtp_payload_decode_create(int payload, const char* name, struct rtp_payload_t *handler, void* cbparam)
{
    struct rtp_payload_delegate_t* ctx;
    ctx = calloc(1, sizeof(*ctx));
    if (ctx)
    {
        if (rtp_payload_find(payload, name, ctx) < 0
            || NULL == (ctx->packer = ctx->decoder->create(handler, cbparam)))
        {
            free(ctx);
            return NULL;
        }
    }
    return ctx;
}

(2)通过如下代码可以看出，ctx->decoder->create指向的函数是rtp_h264_unpack_create(struct rtp_payload_t *handler, void* param)，也就是正真的实现体。源码如下:

struct rtp_payload_decode_t *rtp_h264_decode()
{
    static struct rtp_payload_decode_t unpacker = {
        rtp_h264_unpack_create,
        rtp_h264_unpack_destroy,
        rtp_h264_unpack_input,
    };

    return &unpacker;
}

static void* rtp_h264_unpack_create(struct rtp_payload_t *handler, void* param)
{
    struct rtp_decode_h264_t *unpacker;
    unpacker = (struct rtp_decode_h264_t *)calloc(1, sizeof(*unpacker));
    if(!unpacker)
        return NULL;

    memcpy(&unpacker->handler, handler, sizeof(unpacker->handler));
    unpacker->cbparam = param;
    unpacker->flags = -1;
    return unpacker;
}

(3)函数rtp_payload_decode_input(ctx->decoder_h264, packet, bytes)就是处在接口层，正真的实现是由ctx->decoder->input指向的函数rtp_h264_unpack_input(void* p, const void* packet, int bytes)(该函数才是正真的实现体)，根据packer不同的类型去实现解封装，源码如下:

//解封装
static int rtp_h264_unpack_input(void* p, const void* packet, int bytes)
{
    int r;
    uint8_t nalt;
    struct rtp_packet_t pkt;
    struct rtp_decode_h264_t *unpacker;

    unpacker = (struct rtp_decode_h264_t *)p;
    // 反序列化
    if(!unpacker || 0 != rtp_packet_deserialize(&pkt, packet, bytes) || pkt.payloadlen < 1)
        return -EINVAL;

    if (-1 == unpacker->flags)
    {
        unpacker->flags = 0;
        unpacker->seq = (uint16_t)(pkt.rtp.seq - 1); // disable packet lost
    }

    if ((uint16_t)pkt.rtp.seq != (uint16_t)(unpacker->seq + 1))
    {
        unpacker->flags = RTP_PAYLOAD_FLAG_PACKET_LOST;
        unpacker->size = 0; // discard previous packets
    }
    unpacker->seq = (uint16_t)pkt.rtp.seq;
    //就是payload的第一个字节
    nalt = ((unsigned char *)pkt.payload)[0];
    switch(nalt & 0x1F)
    {
    case 0: // reserved
    case 31: // reserved
        // 这里最好是报错 然后返回错误值
        assert(0);
        return 0; // packet discard
    //对应不同的类型,就做不同的解释
    case 24: // STAP-A
        return rtp_h264_unpack_stap(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 0);
    case 25: // STAP-B
        return rtp_h264_unpack_stap(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 1);
    case 26: // MTAP16
        return rtp_h264_unpack_mtap(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 2);
    case 27: // MTAP24
        return rtp_h264_unpack_mtap(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 3);
    case 28: // FU-A
        return rtp_h264_unpack_fu(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 0);
    case 29: // FU-B
        return rtp_h264_unpack_fu(unpacker, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, 1);

    default: // 1-23 NAL unit
        r = unpacker->handler.packet(unpacker->cbparam, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, unpacker->flags);
        unpacker->flags = 0;
        unpacker->size = 0;
        return 0 == r ? 1 : r; // packet handled
    }
}

(4)如果是1对1的RTP包，直接就反序列化即可。如收到一个UDP包，有1000字节，就可以解析出这个udp包的rtp信息。通过收到的数据，解析出来可读的RTP Packet。

// RFC3550 RTP: A Transport Protocol for Real-Time Applications
// 5.1 RTP Fixed Header Fields (p12)
/*
 0               1               2               3
 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|V=2|P|X|   CC  |M|     PT      |      sequence number          |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                           timestamp                           |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                synchronization source (SSRC) identifier       |
+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
|                 contributing source (CSRC) identifiers        |
|                               ....                            |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
// 通过收到的数据，解析出来可读的RTP packet 反序列化
int rtp_packet_deserialize(struct rtp_packet_t *pkt, const void* data, int bytes)
{
    uint32_t i, v;
    int hdrlen;
    const uint8_t *ptr;

    if (bytes < RTP_FIXED_HEADER) // RFC3550 5.1 RTP Fixed Header Fields(p12)
        return -1;
    ptr = (const unsigned char *)data;
    memset(pkt, 0, sizeof(struct rtp_packet_t));

    // pkt header
    v = nbo_r32(ptr);
    pkt->rtp.v = RTP_V(v);
    pkt->rtp.p = RTP_P(v);
    pkt->rtp.x = RTP_X(v);
    pkt->rtp.cc = RTP_CC(v);
    pkt->rtp.m = RTP_M(v);
    pkt->rtp.pt = RTP_PT(v);
    pkt->rtp.seq = RTP_SEQ(v);
    pkt->rtp.timestamp = nbo_r32(ptr + 4);
    pkt->rtp.ssrc = nbo_r32(ptr + 8);
    assert(RTP_VERSION == pkt->rtp.v);      // 调试的时候用

    hdrlen = RTP_FIXED_HEADER + pkt->rtp.cc * 4;    // 解析带csrc时的总长度
    if (RTP_VERSION != pkt->rtp.v || bytes < hdrlen + (pkt->rtp.x ? 4 : 0) + (pkt->rtp.p ? 1 : 0))
        return -1;      // 报错

    // pkt contributing source
    for (i = 0; i < pkt->rtp.cc; i++)
    {
        pkt->csrc[i] = nbo_r32(ptr + 12 + i * 4);
    }

    assert(bytes >= hdrlen);
    pkt->payload = (uint8_t*)ptr + hdrlen;      // 跳过头部 拿到payload
    pkt->payloadlen = bytes - hdrlen;           // payload长度

    // pkt header extension
    if (1 == pkt->rtp.x)
    {
        const uint8_t *rtpext = ptr + hdrlen;
        assert(pkt->payloadlen >= 4);
        pkt->extension = rtpext + 4;
        pkt->reserved = nbo_r16(rtpext);
        pkt->extlen = nbo_r16(rtpext + 2) * 4;
        if (pkt->extlen + 4 > pkt->payloadlen)
        {
            assert(0);
            return -1;
        }
        else
        {
            //有扩展
            pkt->payload = rtpext + pkt->extlen + 4;
            pkt->payloadlen -= pkt->extlen + 4;
        }
    }

    // padding
    if (1 == pkt->rtp.p)
    {
        uint8_t padding = ptr[bytes - 1];
        if (pkt->payloadlen < padding)
        {
            assert(0);
            return -1;
        }
        else
        {
            pkt->payloadlen -= padding;
        }
    }

    return 0;
}

(5)将视频RTP包解封装为nalu，其原理就是封包的逆向过程，通过不同类型的数据包采用不同的打拆包方式，1对多的RTP拆包，源码如下:

// 5.8. Fragmentation Units (FUs) (p29)
/*
 0               1               2               3
 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|  FU indicator |   FU header   |              DON              |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|
|                                                               |
|                          FU payload                           |
|                                                               |
|                               +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                               :   ...OPTIONAL RTP padding     |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
static int rtp_h264_unpack_fu(struct rtp_decode_h264_t *unpacker, const uint8_t* ptr, int bytes, uint32_t timestamp, int fu_b)
{
    int r, n;
    uint8_t fuheader;
    //uint16_t don;

    r = 0;
    n = fu_b ? 4 : 2;   // payload前面有几个字节是供fu_a或fu_b使用， fu_a只有2个字节; fu_b是4个字节
    if (bytes < n || unpacker->size + bytes - n > RTP_PAYLOAD_MAX_SIZE)
    {
        assert(0);
        return -EINVAL; // error
    }

    if (unpacker->size + bytes - n + 1 /*NALU*/ > unpacker->capacity)
    {
        void* p = NULL;
        int size = unpacker->size + bytes + 1;
        size += size / 4 > 128000 ? size / 4 : 128000;
        p = realloc(unpacker->ptr, size);
        if (!p)
        {
            // set packet lost flag
            unpacker->flags = RTP_PAYLOAD_FLAG_PACKET_LOST;
            unpacker->size = 0;
            return -ENOMEM; // error
        }
        unpacker->ptr = (uint8_t*)p;
        unpacker->capacity = size;
    }

    fuheader = ptr[1];
    //don = nbo_r16(ptr + 2);
    if (FU_START(fuheader))//头
    {
#if 0
        if (unpacker->size > 0)
        {
            unpacker->flags |= RTP_PAYLOAD_FLAG_PACKET_CORRUPT;
            unpacker->handler.packet(unpacker->cbparam, unpacker->ptr, unpacker->size, unpacker->timestamp, unpacker->flags);
            unpacker->flags = 0;
            unpacker->size = 0; // reset
        }
#endif

        unpacker->size = 1; // NAL unit type byte
        unpacker->ptr[0] = (ptr[0]/*indicator*/ & 0xE0) | (fuheader & 0x1F);
        assert(H264_NAL(unpacker->ptr[0]) > 0 && H264_NAL(unpacker->ptr[0]) < 24);
    }
    else
    {
        if (0 == unpacker->size)//中间包
        {
            unpacker->flags = RTP_PAYLOAD_FLAG_PACKET_LOST;
            return 0; // packet discard
        }
        assert(unpacker->size > 0);
    }

    unpacker->timestamp = timestamp;
    if (bytes > n)
    {
        assert(unpacker->capacity >= unpacker->size + bytes - n);
        memmove(unpacker->ptr + unpacker->size, ptr + n, bytes - n);
        unpacker->size += bytes - n;
    }

    if(FU_END(fuheader))//中间过程
    {
        if(unpacker->size > 0)      // 多次传入数据后等到FU_END的时候难道一个完整的nalu
            r = unpacker->handler.packet(unpacker->cbparam, unpacker->ptr, unpacker->size, timestamp, unpacker->flags);
        unpacker->flags = 0;
        unpacker->size = 0; // reset
    }

    return 0 == r ? 1 : r; // packet handled
}

(6)函数unpacker->handler.packet(unpacker->cbparam, (const uint8_t*)pkt.payload, pkt.payloadlen, pkt.rtp.timestamp, unpacker->flags)，也会回调用户自定义的函数去写入文件，注意：写文件前，需要加上startcode。源码如下:

static int rtp_decode_packet(void* param, const void *packet, int bytes, uint32_t timestamp, int flags)
{
    static const uint8_t start_code[4] = { 0, 0, 0, 1 };
    struct rtp_h264_test_t* ctx = (struct rtp_h264_test_t*)param;

    static uint8_t buffer[2 * 1024 * 1024];
    assert(bytes + 4 < sizeof(buffer));
    assert(0 == flags);

    size_t size = 0;
    if (0 == strcmp("H264", ctx->encoding) || 0 == strcmp("H265", ctx->encoding))
    {
        memcpy(buffer, start_code, sizeof(start_code));//添加startcode
        size += sizeof(start_code);
    }
    else if (0 == strcasecmp("mpeg4-generic", ctx->encoding))
    {
        int len = bytes + 7;
        uint8_t profile = 2;
        uint8_t sampling_frequency_index = 4;
        uint8_t channel_configuration = 2;
        buffer[0] = 0xFF; /* 12-syncword */
        buffer[1] = 0xF0 /* 12-syncword */ | (0 << 3)/*1-ID*/ | (0x00 << 2) /*2-layer*/ | 0x01 /*1-protection_absent*/;
        buffer[2] = ((profile - 1) << 6) | ((sampling_frequency_index & 0x0F) << 2) | ((channel_configuration >> 2) & 0x01);
        buffer[3] = ((channel_configuration & 0x03) << 6) | ((len >> 11) & 0x03); /*0-original_copy*/ /*0-home*/ /*0-copyright_identification_bit*/ /*0-copyright_identification_start*/
        buffer[4] = (uint8_t)(len >> 3);
        buffer[5] = ((len & 0x07) << 5) | 0x1F;
        buffer[6] = 0xFC | ((len / 1024) & 0x03);
        size = 7;
    }
    memcpy(buffer + size, packet, bytes);
    size += bytes;
    printf("nalu get -> bytes:%d, timestamp:%u\n", size, timestamp);
    // TODO:
    // check media file
    fwrite(buffer, 1, size, ctx->out_file);
}

4.总结

本篇文章主要通过实战过程中源码的方式，对RTP打包和拆包进行了详细分析，并用代码实现了前面文章讲的原理。结合前面的文章，能够更好的理解整个过程，希望能够帮助到大家。欢迎关注，收藏，转发，分享。

后期关于项目知识，也会更新在微信公众号“记录世界 from antonio”，欢迎关注