diff options
Diffstat (limited to 'talk/app/webrtc/java/jni/androidmediaencoder_jni.cc')
| -rw-r--r-- | talk/app/webrtc/java/jni/androidmediaencoder_jni.cc | 473 |
1 files changed, 342 insertions, 131 deletions
diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc index ac349e7faf..64831c3174 100644 --- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc @@ -29,14 +29,16 @@ #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h" #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" +#include "talk/app/webrtc/java/jni/native_handle_impl.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/thread.h" +#include "webrtc/base/thread_checker.h" #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" -#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" -#include "webrtc/modules/video_coding/utility/include/quality_scaler.h" -#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/utility/quality_scaler.h" +#include "webrtc/modules/video_coding/utility/vp8_header_parser.h" #include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" #include "third_party/libyuv/include/libyuv/convert.h" @@ -56,6 +58,7 @@ using webrtc::VideoCodec; using webrtc::VideoCodecType; using webrtc::kVideoCodecH264; using webrtc::kVideoCodecVP8; +using webrtc::kVideoCodecVP9; namespace webrtc_jni { @@ -79,7 +82,9 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, public rtc::MessageHandler { public: virtual ~MediaCodecVideoEncoder(); - explicit MediaCodecVideoEncoder(JNIEnv* jni, VideoCodecType codecType); + MediaCodecVideoEncoder(JNIEnv* jni, + VideoCodecType codecType, + jobject egl_context); // webrtc::VideoEncoder implementation. Everything trampolines to // |codec_thread_| for execution. @@ -103,13 +108,18 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int GetTargetFramerate() override; + bool SupportsNativeHandle() const override { return true; } + const char* ImplementationName() const override; + private: // CHECK-fail if not running on |codec_thread_|. void CheckOnCodecThread(); - // Release() and InitEncode() in an attempt to restore the codec to an + private: + // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and + // InitEncodeOnCodecThread() in an attempt to restore the codec to an // operable state. Necessary after all manner of OMX-layer errors. - void ResetCodec(); + bool ResetCodecOnCodecThread(); // Implementation of webrtc::VideoEncoder methods above, all running on the // codec thread exclusively. @@ -117,10 +127,20 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // If width==0 then this is assumed to be a re-initialization and the // previously-current values are reused instead of the passed parameters // (makes it easier to reason about thread-safety). - int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps); + int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, + bool use_surface); + // Reconfigure to match |frame| in width, height. Also reconfigures the + // encoder if |frame| is a texture/byte buffer and the encoder is initialized + // for byte buffer/texture. Returns false if reconfiguring fails. + bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); int32_t EncodeOnCodecThread( const webrtc::VideoFrame& input_image, const std::vector<webrtc::FrameType>* frame_types); + bool EncodeByteBufferOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); + bool EncodeTextureOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame); + int32_t RegisterEncodeCompleteCallbackOnCodecThread( webrtc::EncodedImageCallback* callback); int32_t ReleaseOnCodecThread(); @@ -150,11 +170,14 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // State that is constant for the lifetime of this object once the ctor // returns. scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. + rtc::ThreadChecker codec_thread_checker_; ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; ScopedGlobalRef<jobject> j_media_codec_video_encoder_; jmethodID j_init_encode_method_; + jmethodID j_get_input_buffers_method_; jmethodID j_dequeue_input_buffer_method_; - jmethodID j_encode_method_; + jmethodID j_encode_buffer_method_; + jmethodID j_encode_texture_method_; jmethodID j_release_method_; jmethodID j_set_rates_method_; jmethodID j_dequeue_output_buffer_method_; @@ -170,6 +193,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int width_; // Frame width in pixels. int height_; // Frame height in pixels. bool inited_; + bool use_surface_; uint16_t picture_id_; enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. @@ -205,6 +229,16 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // H264 bitstream parser, used to extract QP from encoded bitstreams. webrtc::H264BitstreamParser h264_bitstream_parser_; + + // VP9 variables to populate codec specific structure. + webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for + // non-flexible VP9 mode. + uint8_t tl0_pic_idx_; + size_t gof_idx_; + + // EGL context - owned by factory, should not be allocated/destroyed + // by MediaCodecVideoEncoder. + jobject egl_context_; }; MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { @@ -213,11 +247,9 @@ MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { } MediaCodecVideoEncoder::MediaCodecVideoEncoder( - JNIEnv* jni, VideoCodecType codecType) : + JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : codecType_(codecType), callback_(NULL), - inited_(false), - picture_id_(0), codec_thread_(new Thread()), j_media_codec_video_encoder_class_( jni, @@ -228,7 +260,11 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( GetMethodID(jni, *j_media_codec_video_encoder_class_, "<init>", - "()V"))) { + "()V"))), + inited_(false), + use_surface_(false), + picture_id_(0), + egl_context_(egl_context) { ScopedLocalRefFrame local_ref_frame(jni); // It would be nice to avoid spinning up a new thread per MediaCodec, and // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug @@ -239,19 +275,27 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( // thread. codec_thread_->SetName("MediaCodecVideoEncoder", NULL); RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; - + codec_thread_checker_.DetachFromThread(); jclass j_output_buffer_info_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); j_init_encode_method_ = GetMethodID( jni, *j_media_codec_video_encoder_class_, "initEncode", - "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;IIII)" - "[Ljava/nio/ByteBuffer;"); + "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" + "IIIILorg/webrtc/EglBase14$Context;)Z"); + j_get_input_buffers_method_ = GetMethodID( + jni, + *j_media_codec_video_encoder_class_, + "getInputBuffers", + "()[Ljava/nio/ByteBuffer;"); j_dequeue_input_buffer_method_ = GetMethodID( jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); - j_encode_method_ = GetMethodID( - jni, *j_media_codec_video_encoder_class_, "encode", "(ZIIJ)Z"); + j_encode_buffer_method_ = GetMethodID( + jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); + j_encode_texture_method_ = GetMethodID( + jni, *j_media_codec_video_encoder_class_, "encodeTexture", + "(ZI[FJ)Z"); j_release_method_ = GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); j_set_rates_method_ = GetMethodID( @@ -275,6 +319,7 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( j_info_presentation_timestamp_us_field_ = GetFieldID( jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; + srand(time(NULL)); AllowBlockingCalls(); } @@ -295,8 +340,8 @@ int32_t MediaCodecVideoEncoder::InitEncode( << codecType_; ALOGD << "InitEncode request"; - scale_ = webrtc::field_trial::FindFullName( - "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"; + scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( + "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); if (scale_) { if (codecType_ == kVideoCodecVP8) { @@ -331,7 +376,8 @@ int32_t MediaCodecVideoEncoder::InitEncode( codec_settings->width, codec_settings->height, codec_settings->startBitrate, - codec_settings->maxFramerate)); + codec_settings->maxFramerate, + false /* use_surface */)); } int32_t MediaCodecVideoEncoder::Encode( @@ -374,6 +420,7 @@ int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, } void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); @@ -381,7 +428,6 @@ void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { // functor), so expect no ID/data. RTC_CHECK(!msg->message_id) << "Unexpected message!"; RTC_CHECK(!msg->pdata) << "Unexpected message!"; - CheckOnCodecThread(); if (!inited_) { return; } @@ -393,26 +439,24 @@ void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { codec_thread_->PostDelayed(kMediaCodecPollMs, this); } -void MediaCodecVideoEncoder::CheckOnCodecThread() { - RTC_CHECK(codec_thread_ == ThreadManager::Instance()->CurrentThread()) - << "Running on wrong thread!"; -} - -void MediaCodecVideoEncoder::ResetCodec() { - ALOGE << "ResetCodec"; - if (Release() != WEBRTC_VIDEO_CODEC_OK || - codec_thread_->Invoke<int32_t>(Bind( - &MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, - width_, height_, 0, 0)) != WEBRTC_VIDEO_CODEC_OK) { +bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + ALOGE << "ResetOnCodecThread"; + if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || + InitEncodeOnCodecThread(width_, height_, 0, 0, false) != + WEBRTC_VIDEO_CODEC_OK) { // TODO(fischman): wouldn't it be nice if there was a way to gracefully // degrade to a SW encoder at this point? There isn't one AFAICT :( // https://code.google.com/p/webrtc/issues/detail?id=2920 + return false; } + return true; } int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( - int width, int height, int kbps, int fps) { - CheckOnCodecThread(); + int width, int height, int kbps, int fps, bool use_surface) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); @@ -448,52 +492,63 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( render_times_ms_.clear(); frame_rtc_times_ms_.clear(); drop_next_input_frame_ = false; + use_surface_ = use_surface; picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; + gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); + tl0_pic_idx_ = static_cast<uint8_t>(rand()); + gof_idx_ = 0; + // We enforce no extra stride/padding in the format creation step. jobject j_video_codec_enum = JavaEnumFromIndex( jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); - jobjectArray input_buffers = reinterpret_cast<jobjectArray>( - jni->CallObjectMethod(*j_media_codec_video_encoder_, - j_init_encode_method_, - j_video_codec_enum, - width_, - height_, - kbps, - fps)); - CHECK_EXCEPTION(jni); - if (IsNull(jni, input_buffers)) { + const bool encode_status = jni->CallBooleanMethod( + *j_media_codec_video_encoder_, j_init_encode_method_, + j_video_codec_enum, width, height, kbps, fps, + (use_surface ? egl_context_ : nullptr)); + if (!encode_status) { + ALOGE << "Failed to configure encoder."; return WEBRTC_VIDEO_CODEC_ERROR; } + CHECK_EXCEPTION(jni); - inited_ = true; - switch (GetIntField(jni, *j_media_codec_video_encoder_, - j_color_format_field_)) { - case COLOR_FormatYUV420Planar: - encoder_fourcc_ = libyuv::FOURCC_YU12; - break; - case COLOR_FormatYUV420SemiPlanar: - case COLOR_QCOM_FormatYUV420SemiPlanar: - case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: - encoder_fourcc_ = libyuv::FOURCC_NV12; - break; - default: - LOG(LS_ERROR) << "Wrong color format."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - size_t num_input_buffers = jni->GetArrayLength(input_buffers); - RTC_CHECK(input_buffers_.empty()) - << "Unexpected double InitEncode without Release"; - input_buffers_.resize(num_input_buffers); - for (size_t i = 0; i < num_input_buffers; ++i) { - input_buffers_[i] = - jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); - int64_t yuv_buffer_capacity = - jni->GetDirectBufferCapacity(input_buffers_[i]); + if (!use_surface) { + jobjectArray input_buffers = reinterpret_cast<jobjectArray>( + jni->CallObjectMethod(*j_media_codec_video_encoder_, + j_get_input_buffers_method_)); CHECK_EXCEPTION(jni); - RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; + if (IsNull(jni, input_buffers)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + switch (GetIntField(jni, *j_media_codec_video_encoder_, + j_color_format_field_)) { + case COLOR_FormatYUV420Planar: + encoder_fourcc_ = libyuv::FOURCC_YU12; + break; + case COLOR_FormatYUV420SemiPlanar: + case COLOR_QCOM_FormatYUV420SemiPlanar: + case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: + encoder_fourcc_ = libyuv::FOURCC_NV12; + break; + default: + LOG(LS_ERROR) << "Wrong color format."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + size_t num_input_buffers = jni->GetArrayLength(input_buffers); + RTC_CHECK(input_buffers_.empty()) + << "Unexpected double InitEncode without Release"; + input_buffers_.resize(num_input_buffers); + for (size_t i = 0; i < num_input_buffers; ++i) { + input_buffers_[i] = + jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); + int64_t yuv_buffer_capacity = + jni->GetDirectBufferCapacity(input_buffers_[i]); + CHECK_EXCEPTION(jni); + RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; + } } - CHECK_EXCEPTION(jni); + inited_ = true; codec_thread_->PostDelayed(kMediaCodecPollMs, this); return WEBRTC_VIDEO_CODEC_OK; } @@ -501,40 +556,53 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( const webrtc::VideoFrame& frame, const std::vector<webrtc::FrameType>* frame_types) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } + frames_received_++; if (!DeliverPendingOutputs(jni)) { - ResetCodec(); - // Continue as if everything's fine. + if (!ResetCodecOnCodecThread()) + return WEBRTC_VIDEO_CODEC_ERROR; } if (drop_next_input_frame_) { - ALOGV("Encoder drop frame - failed callback."); + ALOGW << "Encoder drop frame - failed callback."; drop_next_input_frame_ = false; return WEBRTC_VIDEO_CODEC_OK; } RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; - // Check framerate before spatial resolution change. - if (scale_) - quality_scaler_.OnEncodeFrame(frame); - const VideoFrame& input_frame = - scale_ ? quality_scaler_.GetScaledFrame(frame) : frame; + VideoFrame input_frame = frame; + if (scale_) { + // Check framerate before spatial resolution change. + quality_scaler_.OnEncodeFrame(frame); + const webrtc::QualityScaler::Resolution scaled_resolution = + quality_scaler_.GetScaledResolution(); + if (scaled_resolution.width != frame.width() || + scaled_resolution.height != frame.height()) { + if (frame.native_handle() != nullptr) { + rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer( + static_cast<AndroidTextureBuffer*>( + frame.video_frame_buffer().get())->ScaleAndRotate( + scaled_resolution.width, + scaled_resolution.height, + webrtc::kVideoRotation_0)); + input_frame.set_video_frame_buffer(scaled_buffer); + } else { + input_frame = quality_scaler_.GetScaledFrame(frame); + } + } + } - if (input_frame.width() != width_ || input_frame.height() != height_) { - ALOGD << "Frame resolution change from " << width_ << " x " << height_ << - " to " << input_frame.width() << " x " << input_frame.height(); - width_ = input_frame.width(); - height_ = input_frame.height(); - ResetCodec(); - return WEBRTC_VIDEO_CODEC_OK; + if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { + ALOGE << "Failed to reconfigure encoder."; + return WEBRTC_VIDEO_CODEC_ERROR; } // Check if we accumulated too many frames in encoder input buffers @@ -552,65 +620,138 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( } } - int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, - j_dequeue_input_buffer_method_); - CHECK_EXCEPTION(jni); - if (j_input_buffer_index == -1) { - // Video codec falls behind - no input buffer available. - ALOGV("Encoder drop frame - no input buffers available"); - frames_dropped_++; - // Report dropped frame to quality_scaler_. - OnDroppedFrame(); - return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. - } - if (j_input_buffer_index == -2) { - ResetCodec(); + const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; + bool encode_status = true; + if (!input_frame.native_handle()) { + int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, + j_dequeue_input_buffer_method_); + CHECK_EXCEPTION(jni); + if (j_input_buffer_index == -1) { + // Video codec falls behind - no input buffer available. + ALOGW << "Encoder drop frame - no input buffers available"; + frames_dropped_++; + // Report dropped frame to quality_scaler_. + OnDroppedFrame(); + return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. + } + if (j_input_buffer_index == -2) { + ResetCodecOnCodecThread(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, + j_input_buffer_index); + } else { + encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); + } + + if (!encode_status) { + ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); + ResetCodecOnCodecThread(); return WEBRTC_VIDEO_CODEC_ERROR; } + last_input_timestamp_ms_ = + current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; + frames_in_queue_++; + + // Save input image timestamps for later output + timestamps_.push_back(input_frame.timestamp()); + render_times_ms_.push_back(input_frame.render_time_ms()); + frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); + current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; + + if (!DeliverPendingOutputs(jni)) { + ALOGE << "Failed deliver pending outputs."; + ResetCodecOnCodecThread(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( + const webrtc::VideoFrame& frame) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + + const bool is_texture_frame = frame.native_handle() != nullptr; + const bool reconfigure_due_to_format = is_texture_frame != use_surface_; + const bool reconfigure_due_to_size = + frame.width() != width_ || frame.height() != height_; + + if (reconfigure_due_to_format) { + ALOGD << "Reconfigure encoder due to format change. " + << (use_surface_ ? + "Reconfiguring to encode from byte buffer." : + "Reconfiguring to encode from texture."); + } + if (reconfigure_due_to_size) { + ALOGD << "Reconfigure encoder due to frame resolution change from " + << width_ << " x " << height_ << " to " << frame.width() << " x " + << frame.height(); + width_ = frame.width(); + height_ = frame.height(); + } + + if (!reconfigure_due_to_format && !reconfigure_due_to_size) + return true; + + ReleaseOnCodecThread(); + + return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == + WEBRTC_VIDEO_CODEC_OK; +} + +bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(!use_surface_); + ALOGV("Encoder frame in # %d. TS: %lld. Q: %d", frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_); - jobject j_input_buffer = input_buffers_[j_input_buffer_index]; + jobject j_input_buffer = input_buffers_[input_buffer_index]; uint8_t* yuv_buffer = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); CHECK_EXCEPTION(jni); RTC_CHECK(yuv_buffer) << "Indirect buffer??"; RTC_CHECK(!libyuv::ConvertFromI420( - input_frame.buffer(webrtc::kYPlane), input_frame.stride(webrtc::kYPlane), - input_frame.buffer(webrtc::kUPlane), input_frame.stride(webrtc::kUPlane), - input_frame.buffer(webrtc::kVPlane), input_frame.stride(webrtc::kVPlane), + frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), + frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), + frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), yuv_buffer, width_, width_, height_, encoder_fourcc_)) << "ConvertFromI420 failed"; - last_input_timestamp_ms_ = current_timestamp_us_ / 1000; - frames_in_queue_++; - // Save input image timestamps for later output - timestamps_.push_back(input_frame.timestamp()); - render_times_ms_.push_back(input_frame.render_time_ms()); - frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); - - bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, - j_encode_method_, + j_encode_buffer_method_, key_frame, - j_input_buffer_index, + input_buffer_index, yuv_size_, current_timestamp_us_); CHECK_EXCEPTION(jni); - current_timestamp_us_ += 1000000 / last_set_fps_; + return encode_status; +} - if (!encode_status || !DeliverPendingOutputs(jni)) { - ResetCodec(); - return WEBRTC_VIDEO_CODEC_ERROR; - } +bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(use_surface_); + NativeHandleImpl* handle = + static_cast<NativeHandleImpl*>(frame.native_handle()); + jfloatArray sampling_matrix = jni->NewFloatArray(16); + jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); - return WEBRTC_VIDEO_CODEC_OK; + bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, + j_encode_texture_method_, + key_frame, + handle->oes_texture_id, + sampling_matrix, + current_timestamp_us_); + CHECK_EXCEPTION(jni); + return encode_status; } int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( webrtc::EncodedImageCallback* callback) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); callback_ = callback; @@ -618,10 +759,10 @@ int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( } int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); if (!inited_) { return WEBRTC_VIDEO_CODEC_OK; } - CheckOnCodecThread(); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << frames_received_ << ". Encoded: " << frames_encoded_ << @@ -634,13 +775,14 @@ int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { CHECK_EXCEPTION(jni); rtc::MessageQueueManager::Clear(this); inited_ = false; + use_surface_ = false; ALOGD << "EncoderReleaseOnCodecThread done."; return WEBRTC_VIDEO_CODEC_OK; } int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); if (last_set_bitrate_kbps_ == new_bit_rate && last_set_fps_ == frame_rate) { return WEBRTC_VIDEO_CODEC_OK; @@ -659,7 +801,7 @@ int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, last_set_fps_); CHECK_EXCEPTION(jni); if (!ret) { - ResetCodec(); + ResetCodecOnCodecThread(); return WEBRTC_VIDEO_CODEC_ERROR; } return WEBRTC_VIDEO_CODEC_OK; @@ -691,6 +833,7 @@ jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( } bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); while (true) { jobject j_output_buffer_info = jni->CallObjectMethod( *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); @@ -702,7 +845,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { int output_buffer_index = GetOutputBufferInfoIndex(jni, j_output_buffer_info); if (output_buffer_index == -1) { - ResetCodec(); + ResetCodecOnCodecThread(); return false; } @@ -786,19 +929,42 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { info.codecSpecific.VP8.layerSync = false; info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; - picture_id_ = (picture_id_ + 1) & 0x7FFF; + } else if (codecType_ == kVideoCodecVP9) { + if (key_frame) { + gof_idx_ = 0; + } + info.codecSpecific.VP9.picture_id = picture_id_; + info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; + info.codecSpecific.VP9.flexible_mode = false; + info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; + info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; + info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; + info.codecSpecific.VP9.temporal_up_switch = true; + info.codecSpecific.VP9.inter_layer_predicted = false; + info.codecSpecific.VP9.gof_idx = + static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); + info.codecSpecific.VP9.num_spatial_layers = 1; + info.codecSpecific.VP9.spatial_layer_resolution_present = false; + if (info.codecSpecific.VP9.ss_data_available) { + info.codecSpecific.VP9.spatial_layer_resolution_present = true; + info.codecSpecific.VP9.width[0] = width_; + info.codecSpecific.VP9.height[0] = height_; + info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); + } } + picture_id_ = (picture_id_ + 1) & 0x7FFF; // Generate a header describing a single fragment. webrtc::RTPFragmentationHeader header; memset(&header, 0, sizeof(header)); - if (codecType_ == kVideoCodecVP8) { + if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { header.VerifyAndAllocateFragmentationHeader(1); header.fragmentationOffset[0] = 0; header.fragmentationLength[0] = image->_length; header.fragmentationPlType[0] = 0; header.fragmentationTimeDiff[0] = 0; - if (scale_) { + if (codecType_ == kVideoCodecVP8 && scale_) { int qp; if (webrtc::vp8::GetQp(payload, payload_size, &qp)) quality_scaler_.ReportQP(qp); @@ -829,7 +995,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] << " " << image->_buffer[2] << " " << image->_buffer[3] << " " << image->_buffer[4] << " " << image->_buffer[5]; - ResetCodec(); + ResetCodecOnCodecThread(); return false; } scPositions[scPositionsLength] = payload_size; @@ -852,7 +1018,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { output_buffer_index); CHECK_EXCEPTION(jni); if (!success) { - ResetCodec(); + ResetCodecOnCodecThread(); return false; } @@ -907,7 +1073,12 @@ int MediaCodecVideoEncoder::GetTargetFramerate() { return scale_ ? quality_scaler_.GetTargetFramerate() : -1; } -MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { +const char* MediaCodecVideoEncoder::ImplementationName() const { + return "MediaCodec"; +} + +MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() + : egl_context_(nullptr) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); @@ -923,6 +1094,16 @@ MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); } + bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( + j_encoder_class, + GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); + CHECK_EXCEPTION(jni); + if (is_vp9_hw_supported) { + ALOGD << "VP9 HW Encoder supported."; + supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", + MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); + } + bool is_h264_hw_supported = jni->CallStaticBooleanMethod( j_encoder_class, GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); @@ -936,9 +1117,37 @@ MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {} +void MediaCodecVideoEncoderFactory::SetEGLContext( + JNIEnv* jni, jobject render_egl_context) { + ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; + if (egl_context_) { + jni->DeleteGlobalRef(egl_context_); + egl_context_ = NULL; + } + if (!IsNull(jni, render_egl_context)) { + egl_context_ = jni->NewGlobalRef(render_egl_context); + if (CheckException(jni)) { + ALOGE << "error calling NewGlobalRef for EGL Context."; + egl_context_ = NULL; + } else { + jclass j_egl_context_class = + FindClass(jni, "org/webrtc/EglBase14$Context"); + if (!jni->IsInstanceOf(egl_context_, j_egl_context_class)) { + ALOGE << "Wrong EGL Context."; + jni->DeleteGlobalRef(egl_context_); + egl_context_ = NULL; + } + } + } + if (egl_context_ == NULL) { + ALOGW << "NULL VideoDecoder EGL context - HW surface encoding is disabled."; + } +} + webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( VideoCodecType type) { if (supported_codecs_.empty()) { + ALOGW << "No HW video encoder for type " << (int)type; return NULL; } for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); @@ -946,9 +1155,11 @@ webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( if (it->type == type) { ALOGD << "Create HW video encoder for type " << (int)type << " (" << it->name << ")."; - return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type); + return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, + egl_context_); } } + ALOGW << "Can not find HW video encoder for type " << (int)type; return NULL; } |