Android FFmpeg音频播放
本文介绍了使用opensl es和FFmpeg在Android平台上实现音频解码播放功能的方法。
opensl es简介
Android NDK中包含了平台特有的opensl es。它包含了一系列底层的音频接口，允许开发者使用纯底层代码实现音频播放，录制等功能。相比于AudioTrack，opensl es具有低延迟，高性能等多项优点。
准备工作
- 搭建Android Studio NDK开发环境
- 编译FFmpeg库并将之集成到Android Studio中
- 在工程的AndroidManifest中加入权限
<!-- RECORD_AUDIO is needed to create an audio recorder -->
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<!-- MODIFY_AUDIO_SETTINGS is needed to use audio effects such as environmental reverb -->
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<!-- INTERNET is needed to use a URI-based audio player, depending on the URI -->
<uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
- CMakeLists.txt中加入链接库opensl es
target_link_libraries(native-lib
log
android
OpenSLES
avcodec-57_lib
avformat-57_lib
avutil-55_lib
swresample-2_lib
swscale-4_lib)
FFmpeg音频解码
- 在native-lib中加入相关的库
extern "C"{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libswresample/swresample.h"
#include "libavutil/opt.h"
};
- 声明全局变量
static void *nextBuffer;
static int nextSize;
static AVPacket *packet;
static AVFrame *pFrame;
static AVCodecContext *pCodecCtx;
static SwrContext *swr;
static AVFormatContext *pFormatCtx;
static int audioindex;uint8_t *outputBuffer;
- 初始化FFmpeg，读取音频文件，创建解码器和输出缓存
extern "C"
int Java_cn_jx_audiotest_MainActivity_play(JNIEnv* env, jclass clazz, jstring url) {
int i;
AVCodec *pCodec;
char input_str[500]={0};
//读取输入的音频文件地址
sprintf(input_str, "%s", env->GetStringUTFChars(url, NULL));
//初始化
av_register_all();
//分配一个AVFormatContext结构
pFormatCtx = avformat_alloc_context();
//打开文件
if(avformat_open_input(&pFormatCtx,input_str,NULL,NULL)!=0) {
LOGD("Couldn't open input stream.\n");
return -1;
}
//查找文件的流信息
if(avformat_find_stream_info(pFormatCtx,NULL)<0) {
LOGD("Couldn't find stream information.\n");
return -1;
}
//在流信息中找到音频流
audioindex = -1;
for(i=0; i<pFormatCtx->nb_streams; i++) {
if (pFormatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_AUDIO) {
audioindex = i;
break;
}
}
if(audioindex == -1){
LOGD("Couldn't find a video stream.\n");
return -1;
}
//获取相应音频流的解码器
pCodecCtx=pFormatCtx->streams[audioindex]->codec;
pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
assert(pCodec != NULL);
//分配一个帧指针，指向解码后的原始帧
pFrame = av_frame_alloc();
packet=(AVPacket *)av_malloc(sizeof(AVPacket));
//打开解码器
if(avcodec_open2(pCodecCtx, pCodec,NULL)<0){
LOGD("Couldn't open codec.\n");
return -1;
}
//设置格式转换
swr = swr_alloc();
av_opt_set_int(swr, "in_channel_layout", pCodecCtx->channel_layout, 0);
av_opt_set_int(swr, "out_channel_layout", pCodecCtx->channel_layout, 0);
av_opt_set_int(swr, "in_sample_rate", pCodecCtx->sample_rate, 0);
av_opt_set_int(swr, "out_sample_rate", pCodecCtx->sample_rate, 0);
av_opt_set_sample_fmt(swr, "in_sample_fmt", pCodecCtx->sample_fmt, 0);
av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
swr_init(swr);
//分配输入缓存
int outputBufferSize = 8196;
outputBuffer = (uint8_t *) malloc(sizeof(uint8_t) * outputBufferSize);
//解码音频文件
getPCM();
//将解码后的buffer使用opensl es播放
SLresult result;
if (nextSize > 0) {
result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, nextBuffer, nextSize);
if (SL_RESULT_SUCCESS != result) {
pthread_mutex_unlock(&audioEngineLock);
return -1;
}
}
return 0;
}
};
- 解码音频文件
int getPCM(){
while(av_read_frame(pFormatCtx, packet)>=0) {
if (packet->stream_index == audioindex) {
int ret = avcodec_send_packet(pCodecCtx, packet);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
return -1;
ret = avcodec_receive_frame(pCodecCtx, pFrame);
if (ret < 0 && ret != AVERROR_EOF)
return -1;
//处理不同的格式
if (pCodecCtx->sample_fmt == AV_SAMPLE_FMT_S16P) {
nextSize = av_samples_get_buffer_size(pFrame->linesize, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
}else {
av_samples_get_buffer_size(&nextSize, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
}
// 音频格式转换
swr_convert(swr, &outputBuffer, nextSize,
(uint8_t const **) (pFrame->extended_data),
pFrame->nb_samples);
nextBuffer = outputBuffer;
return 0;
}
}
}
opensl es音频播放
- 定义相关全局变量
// engine interfaces
static SLObjectItf engineObject = NULL;
static SLEngineItf engineEngine;
static SLObjectItf outputMixObject = NULL;
static SLEnvironmentalReverbItf outputMixEnvironmentalReverb = NULL;
static SLObjectItf bqPlayerObject = NULL;
static SLPlayItf bqPlayerPlay;
static SLAndroidSimpleBufferQueueItf bqPlayerBufferQueue;
static SLEffectSendItf bqPlayerEffectSend;
static SLMuteSoloItf bqPlayerMuteSolo;
static SLVolumeItf bqPlayerVolume;
static SLmilliHertz bqPlayerSampleRate = 0;
static jint bqPlayerBufSize = 0;
static short *resampleBuf = NULL;
static pthread_mutex_t audioEngineLock = PTHREAD_MUTEX_INITIALIZER;
// aux effect on the output mix, used by the buffer queue player
static const SLEnvironmentalReverbSettings reverbSettings =
SL_I3DL2_ENVIRONMENT_PRESET_STONECORRIDOR;
- 创建引擎
// create the engine and output mix objects
extern "C"
void
Java_cn_jx_audiotest_MainActivity_createEngine(JNIEnv* env, jclass clazz) {
SLresult result;
// create engine
result = slCreateEngine(&engineObject, 0, NULL, 0, NULL, NULL);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// realize the engine
result = (*engineObject)->Realize(engineObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// get the engine interface, which is needed in order to create other objects
result = (*engineObject)->GetInterface(engineObject, SL_IID_ENGINE, &engineEngine);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// create output mix, with environmental reverb specified as a non-required interface
const SLInterfaceID ids[1] = {SL_IID_ENVIRONMENTALREVERB};
const SLboolean req[1] = {SL_BOOLEAN_FALSE};
result = (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObject, 1, ids, req);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// realize the output mix
result = (*outputMixObject)->Realize(outputMixObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// get the environmental reverb interface
// this could fail if the environmental reverb effect is not available,
// either because the feature is not present, excessive CPU load, or
// the required MODIFY_AUDIO_SETTINGS permission was not requested and granted
result = (*outputMixObject)->GetInterface(outputMixObject, SL_IID_ENVIRONMENTALREVERB,
&outputMixEnvironmentalReverb);
if (SL_RESULT_SUCCESS == result) {
result = (*outputMixEnvironmentalReverb)->SetEnvironmentalReverbProperties(
outputMixEnvironmentalReverb, &reverbSettings);
(void)result;
}
// ignore unsuccessful result codes for environmental reverb, as it is optional for this example
}
- 创建BufferQueueAudioPlayer。注意到代码中注册的bqPlayerCallback，每次buffer被处理后，将会调用这个callback。我们在这个callback中再次解码一帧数据塞给BufferQueueAudioPlayer处理。
// create buffer queue audio player
extern "C"
void
Java_cn_jx_audiotest_MainActivity_createBufferQueueAudioPlayer(JNIEnv* env, jclass clazz, jint sampleRate, jint bufSize)
{
SLresult result;
if (sampleRate >= 0 && bufSize >= 0 ) {
bqPlayerSampleRate = sampleRate * 1000;
/*
* device native buffer size is another factor to minimize audio latency, not used in this
* sample: we only play one giant buffer here
*/
bqPlayerBufSize = bufSize;
}
// configure audio source
SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2};
SLDataFormat_PCM format_pcm = {SL_DATAFORMAT_PCM, 1, SL_SAMPLINGRATE_8,
SL_PCMSAMPLEFORMAT_FIXED_16, SL_PCMSAMPLEFORMAT_FIXED_16,
SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN};
/*
* Enable Fast Audio when possible: once we set the same rate to be the native, fast audio path
* will be triggered
*/
if(bqPlayerSampleRate) {
format_pcm.samplesPerSec = bqPlayerSampleRate; //sample rate in mili second
}
SLDataSource audioSrc = {&loc_bufq, &format_pcm};
// configure audio sink
SLDataLocator_OutputMix loc_outmix = {SL_DATALOCATOR_OUTPUTMIX, outputMixObject};
SLDataSink audioSnk = {&loc_outmix, NULL};
/*
* create audio player:
* fast audio does not support when SL_IID_EFFECTSEND is required, skip it
* for fast audio case
*/
const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND,
/*SL_IID_MUTESOLO,*/};
const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
/*SL_BOOLEAN_TRUE,*/ };
result = (*engineEngine)->CreateAudioPlayer(engineEngine, &bqPlayerObject, &audioSrc, &audioSnk,
bqPlayerSampleRate? 2 : 3, ids, req);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// realize the player
result = (*bqPlayerObject)->Realize(bqPlayerObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// get the play interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_PLAY, &bqPlayerPlay);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// get the buffer queue interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_BUFFERQUEUE,
&bqPlayerBufferQueue);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// register callback on the buffer queue
result = (*bqPlayerBufferQueue)->RegisterCallback(bqPlayerBufferQueue, bqPlayerCallback, NULL);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// get the effect send interface
bqPlayerEffectSend = NULL;
if( 0 == bqPlayerSampleRate) {
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_EFFECTSEND,
&bqPlayerEffectSend);
assert(SL_RESULT_SUCCESS == result);
(void)result;
}
#if 0 // mute/solo is not supported for sources that are known to be mono, as this is
// get the mute/solo interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_MUTESOLO, &bqPlayerMuteSolo);
assert(SL_RESULT_SUCCESS == result);
(void)result;
#endif
// get the volume interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_VOLUME, &bqPlayerVolume);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// set the player's state to playing
result = (*bqPlayerPlay)->SetPlayState(bqPlayerPlay, SL_PLAYSTATE_PLAYING);
assert(SL_RESULT_SUCCESS == result);
(void)result;
}
- 实现bqPlayerCallback
// this callback handler is called every time a buffer finishes playing
void
bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context)
{
assert(bq == bqPlayerBufferQueue);
assert(NULL == context);
// for streaming playback, replace this test by logic to find and fill the next buffer
getPCM();
if ( NULL != nextBuffer && 0 != nextSize) {
SLresult result;
// enqueue another buffer
result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, nextBuffer, nextSize);
// the most likely other result is SL_RESULT_BUFFER_INSUFFICIENT,
// which for this code example would indicate a programming error
if (SL_RESULT_SUCCESS != result) {
pthread_mutex_unlock(&audioEngineLock);
}
(void)result;
} else {
releaseResampleBuf();
pthread_mutex_unlock(&audioEngineLock);
}
}
JAVA层调用
我们在JAVA层定义好JNI接口，并按照顺序调用即可开始播放音频文件了。
- 加载so库，定义JNI接口
static {
System.loadLibrary("native-lib");
}
public static native void createEngine();
public static native boolean createBufferQueueAudioPlayer(int sampleRate, int samplesPerBuf);
public native void play(String url);
- 依次调用接口，创建引擎开始播放
createEngine();
int sampleRate = 0;
int bufSize = 0;
/*
* retrieve fast audio path sample rate and buf size; if we have it, we pass to native
* side to create a player with fast audio enabled [ fast audio == low latency audio ];
* IF we do not have a fast audio path, we pass 0 for sampleRate, which will force native
* side to pick up the 8Khz sample rate.
*/
if(Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) {
AudioManager myAudioMgr = (AudioManager) getSystemService(Context.AUDIO_SERVICE);
String nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE);
sampleRate = Integer.parseInt(nativeParam);
nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER);
bufSize = Integer.parseInt(nativeParam);
}
createBufferQueueAudioPlayer(sampleRate, bufSize);
//获取文件地址
String folderurl = Environment.getExternalStorageDirectory().getPath();
String inputurl = folderurl+"/background.mp3";
play(inputurl);
遇到的坑
- 不同的音视频文件解析出来的音频采样数据格式不一样，有的MP3文件解出来是AV_SAMPLE_FMT_S16P，有的MP4文件是AV_SAMPLE_FMT_FLTP，这两种文件的outputBuffer大小设置不太一样。没处理好就会有大量杂音，处理方式详见代码getPCM中的解码部分。（处理格式如有遗漏欢迎举报）
