技术背景
我们在做Android平台RTSP、RTMP播放器的时候，遇到这样的技术诉求，开发者除了希望低延迟的播放外，还想把数据回调上来，然后做视觉算法分析。
单纯地回调数据，不难，需要保证的是，在不影响播放、录像、快照等常规功能的前提下，尽可能高效的数据回调。
技术实现
以大牛直播SDK的SmartPlayer为例，点开始播放之前，初始化参数的时候，我们设置YUV或RGB数据回调：
设置YUV或RGB数据回调：
btnStartStopPlayback.setOnClickListener(new Button.OnClickListener() {
// @Override
public void onClick(View v) {
if (isPlaying) {
Log.i(TAG, "Stop playback stream++");
int iRet = libPlayer.SmartPlayerStopPlay(playerHandle);
if (iRet != 0) {
Log.e(TAG, "Call SmartPlayerStopPlay failed..");
return;
}
btnHardwareDecoder.setEnabled(true);
btnLowLatency.setEnabled(true);
if (!isRecording) {
btnPopInputUrl.setEnabled(true);
btnPopInputKey.setEnabled(true);
btnSetPlayBuffer.setEnabled(true);
btnFastStartup.setEnabled(true);
btnRecoderMgr.setEnabled(true);
libPlayer.SmartPlayerClose(playerHandle);
playerHandle = 0;
}
isPlaying = false;
btnStartStopPlayback.setText("开始播放 ");
if (is_enable_hardware_render_mode && sSurfaceView != null) {
sSurfaceView.setVisibility(View.GONE);
sSurfaceView.setVisibility(View.VISIBLE);
}
Log.i(TAG, "Stop playback stream--");
} else {
Log.i(TAG, "Start playback stream++");
if (!isRecording) {
InitAndSetConfig();
}
// 如果第二个参数设置为null，则播放纯音频
libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);
libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);
//int render_format = 1;
//libPlayer.SmartPlayerSetSurfaceRenderFormat(playerHandle, render_format);
//int is_enable_anti_alias = 1;
//libPlayer.SmartPlayerSetSurfaceAntiAlias(playerHandle, is_enable_anti_alias);
if (isHardwareDecoder && is_enable_hardware_render_mode) {
libPlayer.SmartPlayerSetHWRenderMode(playerHandle, 1);
}
// External Render test
//libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath));
libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath));
libPlayer.SmartPlayerSetUserDataCallback(playerHandle, new UserDataCallback());
//libPlayer.SmartPlayerSetSEIDataCallback(playerHandle, new SEIDataCallback());
libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);
if (isMute) {
libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
: 0);
}
if (isHardwareDecoder) {
int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);
int isSupportH264HwDecoder = libPlayer
.SetSmartPlayerVideoHWDecoder(playerHandle, 1);
Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
}
libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
: 0);
libPlayer.SmartPlayerSetFlipVertical(playerHandle, is_flip_vertical ? 1 : 0);
libPlayer.SmartPlayerSetFlipHorizontal(playerHandle, is_flip_horizontal ? 1 : 0);
libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);
libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume);
int iPlaybackRet = libPlayer
.SmartPlayerStartPlay(playerHandle);
if (iPlaybackRet != 0) {
Log.e(TAG, "Call SmartPlayerStartPlay failed..");
return;
}
btnStartStopPlayback.setText("停止播放 ");
btnPopInputUrl.setEnabled(false);
btnPopInputKey.setEnabled(false);
btnHardwareDecoder.setEnabled(false);
btnSetPlayBuffer.setEnabled(false);
btnLowLatency.setEnabled(false);
btnFastStartup.setEnabled(false);
btnRecoderMgr.setEnabled(false);
isPlaying = true;
Log.i(TAG, "Start playback stream--");
}
}
});对应的设置如下：
// External Render test
libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath));
libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath));如果是RGBA数据，处理如下：
/*
* RGBA数据回调处理
* Author: daniusdk.com
*/
private static class RGBAExternalRender implements NTExternalRender {
// public static final int NT_FRAME_FORMAT_RGBA = 1;
// public static final int NT_FRAME_FORMAT_ABGR = 2;
// public static final int NT_FRAME_FORMAT_I420 = 3;
private final String image_path_;
private long last_save_image_time_ms_;
private int width_;
private int height_;
private int row_bytes_;
private ByteBuffer rgba_buffer_;
public RGBAExternalRender(String image_path) {
this.image_path_ = image_path;
}
@Override
public int getNTFrameFormat() {
Log.i(TAG, "RGBAExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_RGBA);
return NT_FRAME_FORMAT_RGBA;
}
@Override
public void onNTFrameSizeChanged(int width, int height) {
width_ = width;
height_ = height;
row_bytes_ = width_ * 4;
rgba_buffer_ = ByteBuffer.allocateDirect(row_bytes_ * height_);
Log.i(TAG, "RGBAExternalRender::onNTFrameSizeChanged width_:" + width_ + " height_:" + height_);
}
@Override
public ByteBuffer getNTPlaneByteBuffer(int index) {
if (index == 0)
return rgba_buffer_;
Log.e(TAG, "RGBAExternalRender::getNTPlaneByteBuffer index error:" + index);
return null;
}
@Override
public int getNTPlanePerRowBytes(int index) {
if (index == 0)
return row_bytes_;
Log.e(TAG, "RGBAExternalRender::getNTPlanePerRowBytes index error:" + index);
return 0;
}
public void onNTRenderFrame(int width, int height, long timestamp) {
if (rgba_buffer_ == null)
return;
rgba_buffer_.rewind();
// copy buffer
// test
// byte[] test_buffer = new byte[16];
// rgba_buffer_.get(test_buffer);
Log.i(TAG, "RGBAExternalRender:onNTRenderFrame " + width + "*" + height + ", t:" + timestamp);
// Log.i(TAG, "RGBAExternalRender:onNTRenderFrame rgba:" +
// bytesToHexString(test_buffer));
}
}如果是I420数据：
/*
*YUV数据回调处理
* Author: daniusdk.com
*/
private static class I420ExternalRender implements NTExternalRender {
// public static final int NT_FRAME_FORMAT_RGBA = 1;
// public static final int NT_FRAME_FORMAT_ABGR = 2;
// public static final int NT_FRAME_FORMAT_I420 = 3;
private final String image_path_;
private long last_save_image_time_ms_;
private int width_;
private int height_;
private int y_row_bytes_;
private int u_row_bytes_;
private int v_row_bytes_;
private ByteBuffer y_buffer_;
private ByteBuffer u_buffer_;
private ByteBuffer v_buffer_;
public I420ExternalRender(String image_path) {
this.image_path_ = image_path;
}
@Override
public int getNTFrameFormat() {
Log.i(TAG, "I420ExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_I420);
return NT_FRAME_FORMAT_I420;
}
@Override
public void onNTFrameSizeChanged(int width, int height) {
width_ = width;
height_ = height;
y_row_bytes_ = width;
u_row_bytes_ = (width+1)/2;
v_row_bytes_ = (width+1)/2;
y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_*height_);
u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_*((height_ + 1) / 2));
v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_*((height_ + 1) / 2));
Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
+ width_ + " height_=" + height_ + " y_row_bytes_="
+ y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
+ " v_row_bytes_=" + v_row_bytes_);
}
@Override
public ByteBuffer getNTPlaneByteBuffer(int index) {
switch (index) {
case 0:
return y_buffer_;
case 1:
return u_buffer_;
case 2:
return v_buffer_;
default:
Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
return null;
}
}
@Override
public int getNTPlanePerRowBytes(int index) {
switch (index) {
case 0:
return y_row_bytes_;
case 1:
return u_row_bytes_;
case 2:
return v_row_bytes_;
default:
Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
return 0;
}
}
public void onNTRenderFrame(int width, int height, long timestamp) {
if (null == y_buffer_ || null == u_buffer_ || null == v_buffer_)
return;
y_buffer_.rewind();
u_buffer_.rewind();
v_buffer_.rewind();
Log.i(TAG, "I420ExternalRender::onNTRenderFrame " + width + "*" + height + ", t:" + timestamp);
// copy buffer
// test
// byte[] test_buffer = new byte[16];
// y_buffer_.get(test_buffer);
// Log.i(TAG, "I420ExternalRender::onNTRenderFrame y data:" + bytesToHexString(test_buffer));
// u_buffer_.get(test_buffer);
// Log.i(TAG, "I420ExternalRender::onNTRenderFrame u data:" + bytesToHexString(test_buffer));
// v_buffer_.get(test_buffer);
// Log.i(TAG, "I420ExternalRender::onNTRenderFrame v data:" + bytesToHexString(test_buffer));
}
}感兴趣的开发者，可以参考看看，如果需要测试，可以私信探讨。