最近这两天在写一个ios demo，用 VideoToolBox 硬解码从网络上实时接收过来的原始h264 nalu 数据（裸数据）。
网络裸数据如下：
后面的原始数据帧类似以上。。。
（1）开始用了一个正常思路的方式：
循环获取原始数据 -----> 分割大包中的nalu并单个送入硬解码 (nalu的前四个字节需要做调整：
由00 00 00 01 变为大端的 nalu 的长度(不包括头四个字节))
发现一个奇异现象：在模拟器中基本可以正常显示，偶尔有点水波现象，但是基本是正常的，不影响观看，然后部署到真机（iphone6－ios 9.3）中，出现绿屏或者出现一半正常显示一半出现绿屏。
大致代码如下：
/** Find the beginning and end of a NAL (Network Abstraction Layer) unit in
a byte buffer containing H264 bitstream data.
@param[in] buf the buffer
@param[in] size the size of the buffer
@param[out] nal_start the beginning offset of the nal
@param[out] nal_end the end offset of the nal
@return the length of the nal, or 0 if did not find start of nal, or -1 if did not find end of nal */
static int findNalUnit(uint8_t* buf, int size, int* nal_start, int* nal_end)
{
int i;
// find start
*nal_start = 0;
*nal_end = 0;
i = 0;
while ( //( next_bits( 24 ) != 0x000001 && next_bits( 32 ) != 0x00000001 )
(buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0x01) &&
(buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0 || buf[i+3] != 0x01)
)
{
i++; // skip leading zero
if (i+4 >= size)
{
return 0;
} // did not find nal start
}
if (buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0x01) // ( next_bits( 24 ) != 0x000001 )
{
i++;
}
if (buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0x01)
{
/* error, should never happen */
return 0;
}
i+= 3;
*nal_start = i;
while ( //( next_bits( 24 ) != 0x000000 && next_bits( 24 ) != 0x000001 )
(buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0) &&
(buf[i] != 0 || buf[i+1] != 0 || buf[i+2] != 0x01)
)
{
i++;
// FIXME the next line fails when reading a nal that ends exactly at the end of the data
if (i+3 >= size)
{
*nal_end = size;
return (*nal_end - *nal_start);//return -1;
} // did not find nal end, stream ended first
}
*nal_end = i;
return (*nal_end - *nal_start);
}
- (BOOL)decodeNalu:(uint32_t)frame withSize(uint32_t)frameSize {
// LOGD(@">>>>>>>>>>开始解码");
if (frame == NULL || frameSize == 0)
return NO;
uint8_t* p = frame, *pf;
size_t sz = frameSize;
int nal_start, nal_end;
while (![[NSThread currentThread] isCancelled] && findNalUnit(p, sz, &nal_start, &nal_end) > 0) {
CVPixelBufferRef pixelBuffer = NULL;
int nalu_type = p[nal_start] & 0x1f;
int nal_len = nal_end - nal_start;
uint8_t *pnal_size = (uint8_t*)(&nal_len);
//{(uint8_t)(nal_len >> 24), (uint8_t)(nal_len >> 16), (uint8_t)(nal_len >> 8), (uint8_t)nal_len};
if (nal_start == 3) { //big-endian
p[-1] = *(pnal_size + 3);
p[0] = *(pnal_size + 2);
p[1] = *(pnal_size + 1);
p[2] = *(pnal_size);
pf = p - 1;
}
else if (nal_start == 4) {
p[0] = *(pnal_size + 3);
p[1] = *(pnal_size + 2);
p[2] = *(pnal_size + 1);
p[3] = *(pnal_size);
pf = p;
}
switch (nalu_type)
{
case 0x05:
LOGD(@"nalu_type:%d Nal type is IDR frame", nalu_type);
if ([self initH264Decoder]) {
pixelBuffer = [self decode:pf withSize:(nal_len + 4)];
}
break;
case 0x07:
LOGD(@"nalu_type:%d Nal type is SPS", nalu_type);
if (_sps == NULL) {
_spsSize = nal_len;
_sps = (uint8_t*)malloc(_spsSize);
memcpy(_sps, &pf[4], _spsSize);
}
break;
case 0x08:
LOGD(@"nalu_type:%d Nal type is PPS", nalu_type);
if (_pps == NULL) {
_ppsSize = nal_len;
_pps = (uint8_t*)malloc(_ppsSize);
memcpy(_pps, &pf[4], _ppsSize);
}
break;
default:
LOGD(@"nalu_type:%d Nal type is B/P frame", nalu_type);
if ([self initH264Decoder]) {
pixelBuffer = [self decode:pf withSize:(nal_len + 4)];
}
break;
}
p += nal_start;
p += nal_len;
sz -= nal_end;
}
（2）另外一种方式：
循环获取原始数据 -----> 重新打包大包中的nalu (每个nalu的头部改为当前nalu的长度)
使用这种方式在模拟器和真机上都是正常显示的，没有花屏和绿屏现象。所以，对于获取到的一帧数据可能被分成了多个nalu，解码的时候不需要再拆分成单个nalu单独去解码，这样硬解码器内部认为此单nalu不是一个完整的帧，导致花屏。
大致代码如下：
//
// WBH264Play.m
// wenba_rtc
//
// Created by zhouweiwei on 16/11/20.
// Copyright © 2016年 zhouweiwei. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "WBH264Play.h"
#define kH264outputWidth 160
#define kH264outputHeight 120
static const uint8_t *avc_find_startcode_internal(const uint8_t *p, const uint8_t *end)
{
const uint8_t *a = p + 4 - ((intptr_t)p & 3);
for (end -= 3; p < a && p < end; p++) {
if (p[0] == 0 && p[1] == 0 && p[2] == 1)
return p;
}
for (end -= 3; p < end; p += 4) {
uint32_t x = *(const uint32_t*)p;
// if ((x - 0x01000100) & (~x) & 0x80008000) // little endian
// if ((x - 0x00010001) & (~x) & 0x00800080) // big endian
if ((x - 0x01010101) & (~x) & 0x80808080) { // generic
if (p[1] == 0) {
if (p[0] == 0 && p[2] == 1)
return p;
if (p[2] == 0 && p[3] == 1)
return p+1;
}
if (p[3] == 0) {
if (p[2] == 0 && p[4] == 1)
return p+2;
if (p[4] == 0 && p[5] == 1)
return p+3;
}
}
}
for (end += 3; p < end; p++) {
if (p[0] == 0 && p[1] == 0 && p[2] == 1)
return p;
}
return end + 3;
}
const uint8_t *avc_find_startcode(const uint8_t *p, const uint8_t *end)
{
const uint8_t *out= avc_find_startcode_internal(p, end);
if(p<out && out<end && !out[-1]) out--;
return out;
}
@interface H264HwDecoder()
{
NSThread *thread;
uint8_t* _vdata;
size_t _vsize;
uint8_t *_buf_out; // 原始接收的重组数据包
uint8_t *_sps;
size_t _spsSize;
uint8_t *_pps;
size_t _ppsSize;
VTDecompressionSessionRef _deocderSession;
CMVideoFormatDescriptionRef _decoderFormatDescription;
}
@end
@implementation H264HwDecoder
//解码回调函数
static void didDecompress(void *decompressionOutputRefCon, void *sourceFrameRefCon, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration ) {
if (status != noErr || imageBuffer == nil) {
LOGE(@"Error decompresssing frame at time: %.3f error: %d infoFlags: %u",
presentationTimeStamp.value/presentationTimeStamp.timescale, status, infoFlags);
return;
}
if (kVTDecodeInfo_FrameDropped & infoFlags) {
LOGW(@"video frame droped");
return;
}
// int i,j;
// if (CVPixelBufferIsPlanar(imageBuffer)) {
// i = (int)CVPixelBufferGetWidthOfPlane(imageBuffer, 0);
// j = (int)CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
// } else {
// i = (int)CVPixelBufferGetWidth(imageBuffer);
// j = (int)CVPixelBufferGetHeight(imageBuffer);
// }
__weak H264HwDecoder *decoder = (__bridge H264HwDecoder *)decompressionOutputRefCon;
if (decoder.delegate != nil) {
CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
*outputPixelBuffer = CVPixelBufferRetain(imageBuffer);
[decoder.delegate displayDecodedFrame:decoder.uid imageBuffer:imageBuffer];
}
}
- (BOOL)open:width:(uint16_t)width height:(uint16_t)height (NSObject<IWBH264HwDecoderDelegate>*)displayDelegate {
[self close];
if (width == 0 || height == 0) {
_out_width = kH264outputWidth;
_out_height = kH264outputHeight;
}
else {
_out_width = width;
_out_height = height;
}
_vsize = _out_width * _out_height * 3;
_vdata = (uint8_t*)malloc(_vsize * sizeof(uint8_t));
_buf_out = (uint8_t*)malloc(_out_width * _out_height * sizeof(uint8_t));
self.delegate = displayDelegate;
thread = [[NSThread alloc] initWithTarget:self selector:@selector(run) object:nil];
//thread.name = @"Thread";
[thread start];
return YES;
}
- (void)setH264DecoderInterface:(NSObject<IWBH264HwDecoderDelegate>*)displayDelegate {
self.delegate = displayDelegate;
}
- (void)run {
size_t out_size = 0;
while (![[NSThread currentThread] isCancelled]) {
/*这里从网路端循环获取视频数据*/
if (api_video_get(_uid, _vdata, &out_size) == 0 && out_size > 0) {
if ([self decodeNalu:_vdata withSize:out_size]) {
}
}
[NSThread sleepForTimeInterval:0.005];
}
}
- (void)stop {
LOGD(@"uid:%u decoder stop", _uid);
if (_thread != nil) {
if (!_thread.isCancelled) {
[_thread cancel];
LOGD(@"uid:%u thread cancel", _uid);
}
}
LOGD(@"uid:%u decoder stoped", _uid);
if (_decoderFormatDesc != nil) {
CFRelease(_decoderFormatDesc);
_decoderFormatDesc = nil;
}
if (_deocderSession != nil) {
VTDecompressionSessionWaitForAsynchronousFrames(_deocderSession);
VTDecompressionSessionInvalidate(_deocderSession);
CFRelease(_deocderSession);
_deocderSession = nil;
}
_uid = 0;
_out_width = kH264outputWidth;
_out_height = kH264outputHeight;
if (_vdata != NULL) {
free(_vdata);
_vdata = NULL;
_vsize = 0;
}
if (_sps != NULL) {
free(_sps);
_sps = NULL;
_spsSize = 0;
}
if (_pps != NULL) {
free(_pps);
_pps = NULL;
_ppsSize = 0;
}
if (_buf_out != NULL) {
free(_buf_out);
_buf_out = NULL;
}
self.delegate = nil;
}
- (void)close {
[self stop];
_thread = nil;
LOGD(@"uid:%u decoder close", _uid);
}
-(BOOL)initH264Decoder {
if (_deocderSession) {
return YES;
}
if (!_sps || !_pps || _spsSize == 0 || _ppsSize == 0) {
return NO;
}
const uint8_t* const parameterSetPointers[2] = { _sps, _pps };
const size_t parameterSetSizes[2] = { _spsSize, _ppsSize };
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,
2, //param count
parameterSetPointers,
parameterSetSizes,
4, //nal start code size
&_decoderFormatDescription);
if (status == noErr) {
NSDictionary* destinationPixelBufferAttributes = @{
(id)kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]
//硬解必须是 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange 或者是kCVPixelFormatType_420YpCbCr8Planar
//因为iOS是nv12 其他是nv21
, (id)kCVPixelBufferWidthKey : [NSNumber numberWithInt:kH264outputWidth]
, (id)kCVPixelBufferHeightKey : [NSNumber numberWithInt:kH264outputHeight]
//, (id)kCVPixelBufferBytesPerRowAlignmentKey : [NSNumber numberWithInt:kH264outputWidth*2]
, (id)kCVPixelBufferOpenGLCompatibilityKey : [NSNumber numberWithBool:NO]
, (id)kCVPixelBufferOpenGLESCompatibilityKey : [NSNumber numberWithBool:YES]
};
VTDecompressionOutputCallbackRecord callBackRecord;
callBackRecord.decompressionOutputCallback = didDecompress;
callBackRecord.decompressionOutputRefCon = (__bridge void *)self;
status = VTDecompressionSessionCreate(kCFAllocatorDefault,
_decoderFormatDescription,
NULL,
(__bridge CFDictionaryRef)destinationPixelBufferAttributes,
&callBackRecord,
&_deocderSession);
VTSessionSetProperty(_deocderSession, kVTDecompressionPropertyKey_ThreadCount, (__bridge CFTypeRef)[NSNumber numberWithInt:1]);
VTSessionSetProperty(_deocderSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
} else {
LOGE(@"reset decoder session failed status=%d", status);
return NO;
}
return YES;
}
- (BOOL)resetH264Decoder {
if(_deocderSession) {
VTDecompressionSessionWaitForAsynchronousFrames(_deocderSession);
VTDecompressionSessionInvalidate(_deocderSession);
CFRelease(_deocderSession);
_deocderSession = NULL;
}
return [self initH264Decoder];
}
- (CVPixelBufferRef)decode:(uint8_t *)frame withSize:(uint32_t)frameSize {
if (frame == NULL || _deocderSession == nil)
return NULL;
CVPixelBufferRef outputPixelBuffer = NULL;
CMBlockBufferRef blockBuffer = NULL;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(NULL,
(void *)frame,
frameSize,
kCFAllocatorNull,
NULL,
0,
frameSize,
FALSE,
&blockBuffer);
if(status == kCMBlockBufferNoErr) {
CMSampleBufferRef sampleBuffer = NULL;
const size_t sampleSizeArray[] = {frameSize};
// status = CMSampleBufferCreateReady(kCFAllocatorDefault,
// blockBuffer,
// _decoderFormatDescription ,
// 1, 0, NULL, 1, sampleSizeArray,
// &sampleBuffer);
status = CMSampleBufferCreate(NULL, blockBuffer, TRUE, 0, 0, _decoderFormatDescription, 1, 0, NULL, 0, NULL, &sampleBuffer);
if (status == kCMBlockBufferNoErr && sampleBuffer) {
VTDecodeFrameFlags flags = 0;
VTDecodeInfoFlags flagOut = 0;
status = VTDecompressionSessionDecodeFrame(_deocderSession,
sampleBuffer,
flags,
&outputPixelBuffer,
&flagOut);
if (status == kVTInvalidSessionErr) {
LOGE(@"Invalid session, reset decoder session");
[self resetH264Decoder];
} else if(status == kVTVideoDecoderBadDataErr) {
LOGE(@"decode failed status=%d(Bad data)", status);
} else if(status != noErr) {
LOGE(@"decode failed status=%d", status);
}
}
if (sampleBuffer != NULL)
CFRelease(sampleBuffer);
}
if (blockBuffer != NULL)
CFRelease(blockBuffer);
return outputPixelBuffer;
}
- (BOOL)decodeNalu:(uint8_t *)frame withSize:(uint32_t)frameSize {
// LOGD(@">>>>>>>>>>开始解码");
if (frame == NULL || frameSize == 0)
return NO;
int size = frameSize;
const uint8_t *p = frame;
const uint8_t *end = p + size;
const uint8_t *nal_start, *nal_end;
int nal_len, nalu_type;
size = 0;
nal_start = avc_find_startcode(p, end);
while (![[NSThread currentThread] isCancelled]) {
while (![[NSThread currentThread] isCancelled] && nal_start < end && !*(nal_start++));
if (nal_start == end)
break;
nal_end = avc_find_startcode(nal_start, end);
nal_len = nal_end - nal_start;
nalu_type = nal_start[0] & 0x1f;
if (nalu_type == 0x07) {
if (_sps == NULL) {
_spsSize = nal_len;
_sps = (uint8_t*)malloc(_spsSize);
memcpy(_sps, nal_start, _spsSize);
}
}
else if (nalu_type == 0x08) {
if (_pps == NULL) {
_ppsSize = nal_len;
_pps = (uint8_t*)malloc(_ppsSize);
memcpy(_pps, nal_start, _ppsSize);
}
}
else {
_buf_out[size + 0] = (uint8_t)(nal_len >> 24);
_buf_out[size + 1] = (uint8_t)(nal_len >> 16);
_buf_out[size + 2] = (uint8_t)(nal_len >> 8 );
_buf_out[size + 3] = (uint8_t)(nal_len);
memcpy(_buf_out + 4 + size, nal_start, nal_len);
size += 4 + nal_len;
}
nal_start = nal_end;
}
if ([self initH264Decoder]) {
CVPixelBufferRef pixelBuffer = NULL;
pixelBuffer = [self decode:_buf_out withSize:size];
}
return size > 0 ? YES : NO;
}
@end
注意：ios 的视频部分坑很多，需要自己亲身实践才能得之精髓，另外需要结合Apple 的官方例子进行测试可以事半功倍。
下一篇打算讲解下《ios硬编码h264视频设置帧率的坑》，此部分网络上基本没有正确的解答，比如采集摄像头
30fps，设置成12fps，基本都是不生效的，这里面有点坑。
发私信要demo代码的太多了，为了服务更好，可以合作。
并提供各种推拉流、编解码、播放相关等一体化的流媒体直播、点播解决方案。
