diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-04-21 19:56:55 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-04-21 19:56:55 +0000 |
commit | ddd0cff3eb5acdc581c8f6f5b36b1bf8b90c8bed (patch) | |
tree | 77daebc4c4c41dae651a3425504a52dd1ccc21b1 /service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video | |
parent | cad0938b23a933110df171fb7c9f96721f606cd2 (diff) | |
parent | 1879ee137ede0d62b88cd16da3d46138ca540dfa (diff) | |
download | ImsMedia-ddd0cff3eb5acdc581c8f6f5b36b1bf8b90c8bed.tar.gz |
Snap for 9989322 from 1879ee137ede0d62b88cd16da3d46138ca540dfa to mainline-rkpd-release
Change-Id: I35046c7ba53f650c5d42314f226f1df5b7cd8dc1
Diffstat (limited to 'service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video')
13 files changed, 467 insertions, 569 deletions
diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/ImsMediaVideoUtil.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/ImsMediaVideoUtil.cpp index 817a1153..4dea4783 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/ImsMediaVideoUtil.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/ImsMediaVideoUtil.cpp @@ -590,22 +590,20 @@ bool ImsMediaVideoUtil::ParseHevcSps(uint8_t* pbBuffer, uint32_t nBufferSize, tC ImsMediaBitReader objBitReader; objBitReader.SetBuffer(pbBuffer + nOffset, nBufferSize - nOffset); - objBitReader.Read(4); // sps_video_parameter_set_id; - uint32_t sps_max_sub_layers_minus1 = objBitReader.Read(3); - objBitReader.Read(1); // sps_temporal_id_nesting_flag; + objBitReader.Read(4); // sps_video_parameter_set_id; + uint32_t sps_max_sub_layers_minus1 = objBitReader.Read(3); // 0 + objBitReader.Read(1); // sps_temporal_id_nesting_flag; - /*-----------profile_tier_level start-----------------------*/ - objBitReader.Read(3); // general_profile_spac, general_tier_flag - objBitReader.Read(5); // general_profile_idc + objBitReader.Read(3); // general_profile_spac, general_tier_flag + pInfo->nProfile = objBitReader.Read(5); // general_profile_idc - // skip 13byte - flags, not handle - objBitReader.Read(24); + // skip 10byte - flags, not handle objBitReader.Read(24); objBitReader.Read(24); objBitReader.Read(24); objBitReader.Read(8); - objBitReader.Read(8); // general_level_idc + pInfo->nLevel = objBitReader.Read(8); // general_level_idc uint8_t sub_layer_profile_present_flag[sps_max_sub_layers_minus1]; uint8_t sub_layer_level_present_flag[sps_max_sub_layers_minus1]; @@ -645,19 +643,15 @@ bool ImsMediaVideoUtil::ParseHevcSps(uint8_t* pbBuffer, uint32_t nBufferSize, tC objBitReader.ReadByUEMode(); // sps_seq_parameter_set_id - uint32_t chroma_format_idc; - chroma_format_idc = objBitReader.ReadByUEMode(); + uint32_t chroma_format_idc = objBitReader.ReadByUEMode(); if (chroma_format_idc == 3) { objBitReader.Read(1); // separate_colour_plane_flag } - int32_t pic_width_in_luma_samples = objBitReader.ReadByUEMode(); - int32_t pic_height_in_luma_samples = objBitReader.ReadByUEMode(); - - pInfo->nWidth = pic_width_in_luma_samples; - pInfo->nHeight = pic_height_in_luma_samples; + pInfo->nWidth = objBitReader.ReadByUEMode(); + pInfo->nHeight = objBitReader.ReadByUEMode(); uint8_t conformance_window_flag = objBitReader.Read(1); diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoManager.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoManager.cpp index 9d98be37..4652511a 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoManager.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoManager.cpp @@ -21,9 +21,18 @@ using namespace android; VideoManager* VideoManager::manager; -VideoManager::VideoManager() {} +VideoManager::VideoManager() +{ + mRequestHandler.Init("VIDEO_REQUEST_EVENT"); + mResponseHandler.Init("VIDEO_RESPONSE_EVENT"); +} -VideoManager::~VideoManager() {} +VideoManager::~VideoManager() +{ + mRequestHandler.Deinit(); + mResponseHandler.Deinit(); + manager = nullptr; +} VideoManager* VideoManager::getInstance() { @@ -245,13 +254,6 @@ void VideoManager::SendInternalEvent( } } -VideoManager::RequestHandler::RequestHandler() : - ImsMediaEventHandler("VIDEO_REQUEST_EVENT") -{ -} - -VideoManager::RequestHandler::~RequestHandler() {} - void VideoManager::RequestHandler::processEvent( uint32_t event, uint64_t sessionId, uint64_t paramA, uint64_t paramB) { @@ -351,13 +353,6 @@ void VideoManager::RequestHandler::processEvent( } } -VideoManager::ResponseHandler::ResponseHandler() : - ImsMediaEventHandler("VIDEO_RESPONSE_EVENT") -{ -} - -VideoManager::ResponseHandler::~ResponseHandler() {} - void VideoManager::ResponseHandler::processEvent( uint32_t event, uint64_t sessionId, uint64_t paramA, uint64_t paramB) { @@ -407,12 +402,7 @@ void VideoManager::ResponseHandler::processEvent( // TODO : add implementation break; case kVideoMediaInactivityInd: - parcel.writeInt32(event); - parcel.writeInt32(static_cast<int>(paramA)); // type - parcel.writeInt32(static_cast<int>(paramB)); // duration - VideoManager::getInstance()->sendResponse(sessionId, parcel); - break; - case kVideoPacketLossInd: + case kVideoBitrateInd: parcel.writeInt32(event); parcel.writeInt32(static_cast<int>(paramA)); VideoManager::getInstance()->sendResponse(sessionId, parcel); diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoSession.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoSession.cpp index 074e37cf..ba43d0a0 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoSession.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoSession.cpp @@ -106,6 +106,7 @@ ImsMediaResult VideoSession::startGraph(RtpConfig* config) if (mGraphRtpTx != nullptr) { + mGraphRtpTx->setMediaQualityThreshold(&mThreshold); ret = mGraphRtpTx->update(config); if (ret != RESULT_SUCCESS) @@ -135,6 +136,7 @@ ImsMediaResult VideoSession::startGraph(RtpConfig* config) if (pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_SEND_ONLY || pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_SEND_RECEIVE) { + mGraphRtpTx->setMediaQualityThreshold(&mThreshold); ret = mGraphRtpTx->start(); } } @@ -265,6 +267,10 @@ void VideoSession::onEvent(int32_t type, uint64_t param1, uint64_t param2) ImsMediaEventHandler::SendEvent( "VIDEO_RESPONSE_EVENT", kVideoDataUsageInd, mSessionId, param1, param2); break; + case kImsMediaEventNotifyVideoLowestBitrate: + ImsMediaEventHandler::SendEvent( + "VIDEO_RESPONSE_EVENT", kVideoBitrateInd, mSessionId, param1, param2); + break; case kRequestVideoCvoUpdate: case kRequestVideoBitrateChange: case kRequestVideoIdrFrame: diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtcp.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtcp.cpp index 28ef246a..03a6f728 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtcp.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtcp.cpp @@ -94,7 +94,7 @@ ImsMediaResult VideoStreamGraphRtcp::update(RtpConfig* config) if (*reinterpret_cast<VideoConfig*>(mConfig) == *pConfig) { - IMLOGD0("[update] no update"); + IMLOGI0("[update] no update"); return RESULT_SUCCESS; } @@ -105,6 +105,12 @@ ImsMediaResult VideoStreamGraphRtcp::update(RtpConfig* config) mConfig = new VideoConfig(pConfig); + if (mConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_NO_FLOW) + { + IMLOGI0("[update] pause RTCP"); + return stop(); + } + ImsMediaResult ret = ImsMediaResult::RESULT_NOT_READY; // stop scheduler if (mGraphState == kStreamStateRunning) @@ -125,7 +131,8 @@ ImsMediaResult VideoStreamGraphRtcp::update(RtpConfig* config) } } - if (mGraphState == kStreamStateCreated && mConfig->getRtcpConfig().getIntervalSec() != 0) + if (mGraphState == kStreamStateCreated && + mConfig->getMediaDirection() != RtpConfig::MEDIA_DIRECTION_NO_FLOW) { IMLOGI0("[update] resume RTCP"); return start(); @@ -140,6 +147,24 @@ ImsMediaResult VideoStreamGraphRtcp::update(RtpConfig* config) return ret; } +ImsMediaResult VideoStreamGraphRtcp::start() +{ + IMLOGI1("[start] state[%d]", mGraphState); + + if (mConfig == nullptr) + { + return RESULT_INVALID_PARAM; + } + + if (mConfig->getMediaDirection() != RtpConfig::MEDIA_DIRECTION_NO_FLOW) + { + return BaseStreamGraph::start(); + } + + // not started + return RESULT_SUCCESS; +} + bool VideoStreamGraphRtcp::setMediaQualityThreshold(MediaQualityThreshold* threshold) { if (threshold != nullptr) diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpRx.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpRx.cpp index c93bf83a..30912f9e 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpRx.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpRx.cpp @@ -90,7 +90,7 @@ ImsMediaResult VideoStreamGraphRtpRx::update(RtpConfig* config) if (*reinterpret_cast<VideoConfig*>(mConfig) == *pConfig) { - IMLOGD0("[update] no update"); + IMLOGI0("[update] no update"); return RESULT_SUCCESS; } diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpTx.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpTx.cpp index ea3d4654..bc7c7dd0 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpTx.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/VideoStreamGraphRtpTx.cpp @@ -106,7 +106,7 @@ ImsMediaResult VideoStreamGraphRtpTx::update(RtpConfig* config) if (*reinterpret_cast<VideoConfig*>(mConfig) == *pConfig) { - IMLOGD0("[update] no update"); + IMLOGI0("[update] no update"); return RESULT_SUCCESS; } @@ -234,6 +234,23 @@ ImsMediaResult VideoStreamGraphRtpTx::start() return RESULT_SUCCESS; } +bool VideoStreamGraphRtpTx::setMediaQualityThreshold(MediaQualityThreshold* threshold) +{ + if (threshold != nullptr) + { + BaseNode* node = findNode(kNodeIdVideoSource); + + if (node != nullptr) + { + IVideoSourceNode* source = reinterpret_cast<IVideoSourceNode*>(node); + source->SetBitrateThreshold(threshold->getVideoBitrateBps()); + return true; + } + } + + return false; +} + void VideoStreamGraphRtpTx::setSurface(ANativeWindow* surface) { IMLOGI1("[setSurface] state[%d]", mGraphState); diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaCamera.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaCamera.cpp index 36a4f1eb..afca7820 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaCamera.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaCamera.cpp @@ -191,7 +191,6 @@ bool ImsMediaCamera::OpenCamera() return false; } - // Create back facing camera device camera_status_t status = ACameraManager_openCamera(mManager, mActiveCameraId.c_str(), GetDeviceListener(), &gCameraIds[mActiveCameraId].mDevice); @@ -220,6 +219,7 @@ bool ImsMediaCamera::OpenCamera() if (status == ACAMERA_OK) { mExposureRange.min = val.data.i64[0]; + if (mExposureRange.min < kMinExposureTime) { mExposureRange.min = kMinExposureTime; @@ -285,7 +285,7 @@ bool ImsMediaCamera::CreateSession(ANativeWindow* preview, ANativeWindow* record return false; } - if (MatchCaptureSizeRequest(preview) == false) + if (!MatchCaptureSizeRequest(preview)) { IMLOGE0("[CreateSession] resolution is not matched"); return false; @@ -339,7 +339,7 @@ bool ImsMediaCamera::CreateSession(ANativeWindow* preview, ANativeWindow* record } } - if (gCameraIds[mActiveCameraId].mAvailable == false) + if (!gCameraIds[mActiveCameraId].mAvailable) { gCondition.wait_timeout(MAX_WAIT_CAMERA); } @@ -395,39 +395,41 @@ bool ImsMediaCamera::DeleteSession() mCaptureSession = nullptr; } - for (int idxTarget = 0; idxTarget < mCaptureRequest.outputNativeWindows.size(); idxTarget++) + if (mCaptureRequest.request != nullptr) { - if (mCaptureRequest.outputNativeWindows[idxTarget] == nullptr) + for (int idxTarget = 0; idxTarget < mCaptureRequest.outputNativeWindows.size(); idxTarget++) { - continue; - } + if (mCaptureRequest.outputNativeWindows[idxTarget] == nullptr) + { + continue; + } - status = ACaptureRequest_removeTarget( - mCaptureRequest.request, mCaptureRequest.targets[idxTarget]); + status = ACaptureRequest_removeTarget( + mCaptureRequest.request, mCaptureRequest.targets[idxTarget]); - if (status != ACAMERA_OK) - { - IMLOGE1("[DeleteSession] error ACaptureRequest_removeTarget[%s]", GetErrorStr(status)); - } + if (status != ACAMERA_OK) + { + IMLOGE1("[DeleteSession] error ACaptureRequest_removeTarget[%s]", + GetErrorStr(status)); + } - ACameraOutputTarget_free(mCaptureRequest.targets[idxTarget]); - status = ACaptureSessionOutputContainer_remove( - mSessionOutputContainer, mCaptureRequest.sessionOutputs[idxTarget]); + ACameraOutputTarget_free(mCaptureRequest.targets[idxTarget]); + status = ACaptureSessionOutputContainer_remove( + mSessionOutputContainer, mCaptureRequest.sessionOutputs[idxTarget]); - if (status != ACAMERA_OK) - { - IMLOGE1("[DeleteSession] error ACaptureSessionOutputContainer_remove[%s]", - GetErrorStr(status)); - } + if (status != ACAMERA_OK) + { + IMLOGE1("[DeleteSession] error ACaptureSessionOutputContainer_remove[%s]", + GetErrorStr(status)); + } - ACaptureSessionOutput_free(mCaptureRequest.sessionOutputs[idxTarget]); - ANativeWindow_release(mCaptureRequest.outputNativeWindows[idxTarget]); - } + ACaptureSessionOutput_free(mCaptureRequest.sessionOutputs[idxTarget]); + ANativeWindow_release(mCaptureRequest.outputNativeWindows[idxTarget]); + } - if (mCaptureRequest.request != nullptr) - { IMLOGD0("[DeleteSession] free request"); ACaptureRequest_free(mCaptureRequest.request); + mCaptureRequest.request = nullptr; } mCaptureRequest.outputNativeWindows.resize(0); @@ -438,6 +440,7 @@ bool ImsMediaCamera::DeleteSession() { IMLOGD0("[DeleteSession] free container"); ACaptureSessionOutputContainer_free(mSessionOutputContainer); + mSessionOutputContainer = nullptr; } return true; @@ -575,7 +578,6 @@ void ImsMediaCamera::OnDeviceState(ACameraDevice* dev) IMLOGW1("[OnDeviceState] device %s is disconnected", id.c_str()); gCameraIds[id].mAvailable = false; ACameraDevice_close(gCameraIds[id].mDevice); - gCameraIds.erase(id); } /* diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaPauseImageSource.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaPauseImageSource.cpp index 29a472aa..58c2f9fa 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaPauseImageSource.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaPauseImageSource.cpp @@ -63,9 +63,10 @@ void ImsMediaPauseImageSource::Uninitialize() } } -bool ImsMediaPauseImageSource::Initialize(int width, int height) +bool ImsMediaPauseImageSource::Initialize(int width, int height, int stride) { - IMLOGD2("[ImsMediaPauseImageSource] Init(width:%d, height:%d)", width, height); + IMLOGD3("[ImsMediaPauseImageSource] Init(width:%d, height:%d, stride:%d)", width, height, + stride); mWidth = width; mHeight = height; @@ -105,12 +106,12 @@ bool ImsMediaPauseImageSource::Initialize(int width, int height) * (AndroidBitmapFormat)AImageDecoderHeaderInfo_getAndroidBitmapFormat(info); */ - size_t stride = AImageDecoder_getMinimumStride(decoder); // Image decoder does not + size_t decStride = AImageDecoder_getMinimumStride(decoder); // Image decoder does not // use padding by default - size_t size = height * stride; + size_t size = height * decStride; int8_t* pixels = reinterpret_cast<int8_t*>(malloc(size)); - result = AImageDecoder_decodeImage(decoder, pixels, stride, size); + result = AImageDecoder_decodeImage(decoder, pixels, decStride, size); if (result != ANDROID_IMAGE_DECODER_SUCCESS) { IMLOGE0("[ImsMediaPauseImageSource] error occurred, and the file could not be decoded."); @@ -119,7 +120,7 @@ bool ImsMediaPauseImageSource::Initialize(int width, int height) return false; } - mYuvImageBuffer = ConvertRgbaToYuv(pixels, width, height); + mYuvImageBuffer = ConvertRgbaToYuv(pixels, width, height, stride); AImageDecoder_delete(decoder); free(pixels); @@ -141,7 +142,8 @@ size_t ImsMediaPauseImageSource::GetYuvImage(uint8_t* buffer, size_t len) return mBufferSize; } - IMLOGE0("[ImsMediaPauseImageSource] buffer size is smaller. Cannot copy"); + IMLOGE2("[ImsMediaPauseImageSource] buffer size is smaller. Expected Bufsize[%d], passed[%d]", + mBufferSize, len); return 0; } @@ -200,31 +202,27 @@ const char* ImsMediaPauseImageSource::getImageFilePath() return nullptr; } -int8_t* ImsMediaPauseImageSource::ConvertRgbaToYuv(int8_t* pixels, int width, int height) +int8_t* ImsMediaPauseImageSource::ConvertRgbaToYuv( + int8_t* pixels, int width, int height, int stride) { // src array must be integer array, data have no padding alignment int32_t* pSrcArray = reinterpret_cast<int32_t*>(pixels); - mBufferSize = width * height * 1.5; + mBufferSize = stride * height * 1.5; int8_t* pDstArray = reinterpret_cast<int8_t*>(malloc(mBufferSize)); int32_t nYIndex = 0; - int32_t nUVIndex = width * height; - int32_t r, g, b; + int32_t nUVIndex = stride * height; + int32_t r, g, b, padLen = stride - width; double y, u, v; for (int32_t j = 0; j < height; j++) { + int32_t nIndex = width * j; for (int32_t i = 0; i < width; i++) { - int32_t nIndex = width * j + i; - - /* - * TODO: Decode alpha - * a = (pSrcArray[nIndex] & 0xff000000) >> 24; - */ - r = (pSrcArray[nIndex] & 0xff0000) >> 16; g = (pSrcArray[nIndex] & 0xff00) >> 8; b = (pSrcArray[nIndex] & 0xff) >> 0; + nIndex++; // rgb to yuv y = 0.257 * r + 0.504 * g + 0.098 * b + 16; @@ -240,7 +238,19 @@ int8_t* ImsMediaPauseImageSource::ConvertRgbaToYuv(int8_t* pixels, int width, in pDstArray[nUVIndex++] = (uint8_t)((u < 0) ? 0 : ((u > 255) ? 255 : u)); } } + + // Add padding if stride > width + if (padLen > 0) + { + nYIndex += padLen; + + if (j % 2 == 0) + { + nUVIndex += padLen; + } + } } + mBufferSize -= padLen; return pDstArray; } diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoRenderer.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoRenderer.cpp index a4fe774d..475425be 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoRenderer.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoRenderer.cpp @@ -210,10 +210,12 @@ void ImsMediaVideoRenderer::processBuffers() } auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO); + if (index >= 0) { size_t bufferSize = 0; uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize); + if (inputBuffer != nullptr) { FrameData* frame = mFrameDatas.front(); @@ -223,17 +225,8 @@ void ImsMediaVideoRenderer::processBuffers() "config[%d]", index, frame->size, frame->timestamp, frame->isConfig); - media_status_t err; - if (frame->isConfig) - { - err = AMediaCodec_queueInputBuffer(mCodec, index, 0, frame->size, - frame->timestamp * 1000, AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG); - } - else - { - err = AMediaCodec_queueInputBuffer( - mCodec, index, 0, frame->size, frame->timestamp * 1000, 0); - } + media_status_t err = AMediaCodec_queueInputBuffer( + mCodec, index, 0, frame->size, frame->timestamp * 1000, 0); if (err != AMEDIA_OK) { diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoSource.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoSource.cpp index c93d99a4..38b71771 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoSource.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/android/ImsMediaVideoSource.cpp @@ -30,7 +30,6 @@ ImsMediaVideoSource::ImsMediaVideoSource() mWindow = nullptr; mCodec = nullptr; mFormat = nullptr; - mRecordingSurface = nullptr; mImageReaderSurface = nullptr; mImageReader = nullptr; mCodecType = -1; @@ -164,7 +163,6 @@ void ImsMediaVideoSource::SetDeviceOrientation(const uint32_t degree) bool ImsMediaVideoSource::Start() { IMLOGD1("[Start], VideoMode[%d]", mVideoMode); - mRecordingSurface = nullptr; if (mVideoMode == kVideoModeRecording || mVideoMode == kVideoModePauseImage) { @@ -214,27 +212,30 @@ bool ImsMediaVideoSource::Start() return false; } - if (mWidth > mHeight) // Is Landscape Mode - { - err = AMediaCodec_createInputSurface(mCodec, &mRecordingSurface); + mImageReaderSurface = CreateImageReader(mWidth, mHeight); - if (err != AMEDIA_OK) - { - IMLOGE1("[Start] create input surface error[%d]", err); - AMediaCodec_delete(mCodec); - mCodec = nullptr; - AMediaFormat_delete(mFormat); - mFormat = nullptr; - return false; - } - } - else + if (mImageReaderSurface == nullptr) { - mImageReaderSurface = CreateImageReader(mWidth, mHeight); + IMLOGE0("[Start] create image reader failed"); + return false; + } - if (mImageReaderSurface == nullptr) + mCodecStride = mWidth; + AMediaFormat* encoderInputFormat = AMediaCodec_getInputFormat(mCodec); + + if (encoderInputFormat != nullptr) + { + // Check if encoder is initialized with the expected configurations. + int32_t width = 0, height = 0; + AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_WIDTH, &width); + AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_HEIGHT, &height); + AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_STRIDE, &mCodecStride); + AMediaFormat_delete(encoderInputFormat); + + // TODO: More configuration checks should be added + if (mWidth != width || mHeight != height || width > mCodecStride) { - IMLOGE0("[Start] create image reader failed"); + IMLOGE0("Encoder doesn't support requested configuration."); return false; } } @@ -268,9 +269,7 @@ bool ImsMediaVideoSource::Start() return false; } - ANativeWindow* recording = mRecordingSurface ? mRecordingSurface : mImageReaderSurface; - - if (mCamera->CreateSession(mWindow, recording) == false) + if (mCamera->CreateSession(mWindow, mImageReaderSurface) == false) { IMLOGE0("[Start] error create camera session"); AMediaCodec_delete(mCodec); @@ -294,15 +293,14 @@ bool ImsMediaVideoSource::Start() } else if (mVideoMode == kVideoModePauseImage) { - mPauseImageSource.Initialize(mWidth, mHeight); - } - - // start encoder output thread - if (mCodec != nullptr) - { - mStopped = false; - std::thread t1(&ImsMediaVideoSource::processOutputBuffer, this); - t1.detach(); + mPauseImageSource.Initialize(mWidth, mHeight, mCodecStride); + // start encoder output thread + if (mCodec != nullptr) + { + mStopped = false; + std::thread t1(&ImsMediaVideoSource::EncodePauseImage, this); + t1.detach(); + } } mDeviceOrientation = -1; @@ -314,19 +312,19 @@ void ImsMediaVideoSource::Stop() { IMLOGD0("[Stop]"); - mMutex.lock(); + std::lock_guard<std::mutex> guard(mMutex); mStopped = true; - mMutex.unlock(); - if (mCamera != nullptr) + if (mImageReader != nullptr) { - mCamera->StopSession(); + AImageReader_delete(mImageReader); + mImageReader = nullptr; + mImageReaderSurface = nullptr; } - IMLOGD0("[Stop] deinitialize camera"); - if (mCamera != nullptr) { + mCamera->StopSession(); mCamera->DeleteSession(); mCamera->DeInitialize(); mCamera = nullptr; @@ -334,19 +332,12 @@ void ImsMediaVideoSource::Stop() if (mCodec != nullptr) { - if (mRecordingSurface != nullptr) + if (mVideoMode == kVideoModePauseImage) { - AMediaCodec_signalEndOfInputStream(mCodec); + mConditionExit.wait_timeout(mFramerate != 0 ? 1000 / mFramerate : 66); } AMediaCodec_stop(mCodec); - - if (mRecordingSurface != nullptr) - { - ANativeWindow_release(mRecordingSurface); - mRecordingSurface = nullptr; - } - AMediaCodec_delete(mCodec); mCodec = nullptr; } @@ -357,15 +348,10 @@ void ImsMediaVideoSource::Stop() mFormat = nullptr; } - if (mImageReader != nullptr) + if (mVideoMode == kVideoModePauseImage) { - std::lock_guard<std::mutex> guard(mImageReaderMutex); - AImageReader_delete(mImageReader); - mImageReader = nullptr; - mImageReaderSurface = nullptr; + mPauseImageSource.Uninitialize(); } - - mPauseImageSource.Uninitialize(); } bool ImsMediaVideoSource::IsStopped() @@ -376,7 +362,7 @@ bool ImsMediaVideoSource::IsStopped() void ImsMediaVideoSource::onCameraFrame(AImage* pImage) { - std::lock_guard<std::mutex> guard(mImageReaderMutex); + std::lock_guard<std::mutex> guard(mMutex); if (mImageReader == nullptr || pImage == nullptr) { @@ -389,7 +375,6 @@ void ImsMediaVideoSource::onCameraFrame(AImage* pImage) { size_t buffCapacity = 0; uint8_t* encoderBuf = AMediaCodec_getInputBuffer(mCodec, index, &buffCapacity); - if (!encoderBuf || !buffCapacity) { IMLOGE1("[onCameraFrame] returned null buffer pointer or buffCapacity[%d]", @@ -397,69 +382,93 @@ void ImsMediaVideoSource::onCameraFrame(AImage* pImage) return; } - int32_t width, height, ylen, uvlen; + int32_t width, height, ylen, uvlen, result = 0; uint8_t *yPlane, *uvPlane; AImage_getWidth(pImage, &width); AImage_getHeight(pImage, &height); AImage_getPlaneData(pImage, 0, &yPlane, &ylen); AImage_getPlaneData(pImage, 1, &uvPlane, &uvlen); - int32_t facing, sensorOrientation; - mCamera->GetSensorOrientation(mCameraId, &facing, &sensorOrientation); - - switch (facing) + if (mWidth > mHeight) // landscape mode, copy without rotate { - case ACAMERA_LENS_FACING_FRONT: - { - ImsMediaImageRotate::YUV420_SP_Rotate270( - encoderBuf, yPlane, uvPlane, width, height); - } - break; + memcpy(encoderBuf, yPlane, ylen); + memcpy(encoderBuf + ylen, uvPlane, uvlen); + } + else + { + int32_t facing, sensorOrientation; + mCamera->GetSensorOrientation(mCameraId, &facing, &sensorOrientation); - case ACAMERA_LENS_FACING_BACK: + switch (facing) { - ImsMediaImageRotate::YUV420_SP_Rotate90(encoderBuf, yPlane, uvPlane, width, height); - } - break; + case ACAMERA_LENS_FACING_FRONT: + { + result = ImsMediaImageRotate::YUV420_SP_Rotate270( + encoderBuf, buffCapacity, mCodecStride, yPlane, uvPlane, width, height); + } + break; - case ACAMERA_LENS_FACING_EXTERNAL: - { - uint32_t size = width * height; - memcpy(encoderBuf, yPlane, size); - memcpy(encoderBuf + size, uvPlane, size / 2); + case ACAMERA_LENS_FACING_BACK: + { + result = ImsMediaImageRotate::YUV420_SP_Rotate90( + encoderBuf, buffCapacity, mCodecStride, yPlane, uvPlane, width, height); + } + break; + + case ACAMERA_LENS_FACING_EXTERNAL: + { + uint32_t size = width * height; + memcpy(encoderBuf, yPlane, size); + memcpy(encoderBuf + size, uvPlane, size / 2); + } + break; } - break; } IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[onCameraFrame] queue buffer size[%d]", ylen + uvlen); - AMediaCodec_queueInputBuffer( - mCodec, index, 0, ylen + uvlen, ImsMediaTimer::GetTimeInMicroSeconds(), 0); + if (result == 0) + { + AMediaCodec_queueInputBuffer( + mCodec, index, 0, ylen + uvlen, ImsMediaTimer::GetTimeInMicroSeconds(), 0); + } + else + { + IMLOGE5("Camera image resolution[%dx%d]. Encoder resolution[%dx%d] buffer size[%d]", + width, height, mWidth, mHeight, buffCapacity); + AMediaCodec_queueInputBuffer(mCodec, index, 0, 0, 0, 0); + return; + } } else { IMLOGE1("[onCameraFrame] dequeueInputBuffer returned index[%d]", index); } + + processOutputBuffer(); } -void ImsMediaVideoSource::changeBitrate(const uint32_t bitrate) +bool ImsMediaVideoSource::changeBitrate(const uint32_t bitrate) { IMLOGD1("[changeBitrate] bitrate[%d]", bitrate); std::lock_guard<std::mutex> guard(mMutex); if (mStopped) { - return; + return false; } AMediaFormat* params = AMediaFormat_new(); - AMediaFormat_setInt32(params, AMEDIAFORMAT_KEY_BIT_RATE, bitrate); + AMediaFormat_setInt32(params, AMEDIACODEC_KEY_VIDEO_BITRATE, bitrate); media_status_t status = AMediaCodec_setParameters(mCodec, params); if (status != AMEDIA_OK) { IMLOGE1("[changeBitrate] error[%d]", status); + return false; } + + return true; } void ImsMediaVideoSource::requestIdrFrame() @@ -484,29 +493,8 @@ void ImsMediaVideoSource::requestIdrFrame() void ImsMediaVideoSource::EncodePauseImage() { - auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO); - if (index >= 0) - { - size_t buffCapacity = 0; - uint8_t* encoderBuf = AMediaCodec_getInputBuffer(mCodec, index, &buffCapacity); - if (!encoderBuf || !buffCapacity) - { - IMLOGE1("[EncodePauseImage] returned null buffer pointer or buffCapacity[%d]", - buffCapacity); - return; - } - size_t len = mPauseImageSource.GetYuvImage(encoderBuf, buffCapacity); - AMediaCodec_queueInputBuffer( - mCodec, index, 0, len, ImsMediaTimer::GetTimeInMicroSeconds(), 0); - } - else - { - IMLOGE1("[EncodePauseImage] dequeueInputBuffer returned index[%d]", index); - } -} + IMLOGD0("[EncodePauseImage] start"); -void ImsMediaVideoSource::processOutputBuffer() -{ uint32_t nextTime = ImsMediaTimer::GetTimeInMilliSeconds(); uint32_t timeInterval = 66; @@ -515,74 +503,37 @@ void ImsMediaVideoSource::processOutputBuffer() timeInterval = 1000 / mFramerate; } - IMLOGD2("[processOutputBuffer] interval[%d] CameraId[%d]", timeInterval, mCameraId); - - for (;;) + while (!IsStopped()) { - if (IsStopped()) - { - IMLOGD0("[processOutputBuffer] terminated"); - break; - } - - if (mVideoMode == kVideoModePauseImage) - { - EncodePauseImage(); - } - - AMediaCodecBufferInfo info; - auto index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO); + mMutex.lock(); + auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO); if (index >= 0) { - IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO, - "[processOutputBuffer] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", - index, info.size, info.offset, info.presentationTimeUs, info.flags); - - if (info.size > 0) - { - size_t buffCapacity; - uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity); - - if (IsStopped()) - { - break; - } - - if (buf != nullptr && buffCapacity > 0) - { - if (mListener != nullptr) - { - mListener->OnUplinkEvent( - buf + info.offset, info.size, info.presentationTimeUs, info.flags); - } - } - - AMediaCodec_releaseOutputBuffer(mCodec, index, false); - } - } - else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) - { - IMLOGI0("[processOutputBuffer] Encoder output buffer changed"); - } - else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) - { - if (mFormat != nullptr) + size_t buffCapacity = 0; + uint8_t* encoderBuf = AMediaCodec_getInputBuffer(mCodec, index, &buffCapacity); + if (!encoderBuf || !buffCapacity) { - AMediaFormat_delete(mFormat); + IMLOGE1("[EncodePauseImage] returned null buffer pointer or buffCapacity[%d]", + buffCapacity); + return; } - mFormat = AMediaCodec_getOutputFormat(mCodec); - IMLOGI1("[processOutputBuffer] Encoder format changed, format[%s]", - AMediaFormat_toString(mFormat)); + size_t len = mPauseImageSource.GetYuvImage(encoderBuf, buffCapacity); + AMediaCodec_queueInputBuffer( + mCodec, index, 0, len, ImsMediaTimer::GetTimeInMicroSeconds(), 0); } - else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER) + else { - IMLOGD_PACKET0(IM_PACKET_LOG_VIDEO, "[processOutputBuffer] no output buffer"); + IMLOGE1("[EncodePauseImage] dequeueInputBuffer returned index[%d]", index); } - else + + processOutputBuffer(); + mMutex.unlock(); + + if (IsStopped()) { - IMLOGI1("[processOutputBuffer] unexpected index[%d]", index); + break; } nextTime += timeInterval; @@ -591,28 +542,77 @@ void ImsMediaVideoSource::processOutputBuffer() if (nextTime > nCurrTime) { uint32_t timeDiff = nextTime - nCurrTime; - IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[processOutputBuffer] timeDiff[%u]", timeDiff); + IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[EncodePauseImage] timeDiff[%u]", timeDiff); ImsMediaTimer::Sleep(timeDiff); } } - IMLOGD0("[processOutputBuffer] exit"); + IMLOGD0("[EncodePauseImage] end"); + mConditionExit.signal(); } -static void ImageCallback(void* context, AImageReader* reader) +void ImsMediaVideoSource::processOutputBuffer() { - if (context == nullptr) + AMediaCodecBufferInfo info; + auto index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO); + + if (index >= 0) { - return; + IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO, + "[processOutputBuffer] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", + index, info.size, info.offset, info.presentationTimeUs, info.flags); + + if (info.size > 0) + { + size_t buffCapacity; + uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity); + + if (buf != nullptr && buffCapacity > 0) + { + if (mListener != nullptr) + { + mListener->OnUplinkEvent( + buf + info.offset, info.size, info.presentationTimeUs, info.flags); + } + } + + AMediaCodec_releaseOutputBuffer(mCodec, index, false); + } + } + else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) + { + IMLOGI0("[processOutputBuffer] Encoder output buffer changed"); } + else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) + { + if (mFormat != nullptr) + { + AMediaFormat_delete(mFormat); + } - ImsMediaVideoSource* pVideoSource = static_cast<ImsMediaVideoSource*>(context); + mFormat = AMediaCodec_getOutputFormat(mCodec); + IMLOGI1("[processOutputBuffer] Encoder format changed, format[%s]", + AMediaFormat_toString(mFormat)); + } + else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER) + { + IMLOGD_PACKET0(IM_PACKET_LOG_VIDEO, "[processOutputBuffer] no output buffer"); + } + else + { + IMLOGI1("[processOutputBuffer] unexpected index[%d]", index); + } +} - if (pVideoSource->IsStopped()) +static void ImageCallback(void* context, AImageReader* reader) +{ + if (context == nullptr) { return; } + ImsMediaVideoSource* pVideoSource = static_cast<ImsMediaVideoSource*>(context); + AImage* image = nullptr; auto status = AImageReader_acquireNextImage(reader, &image); diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoRendererNode.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoRendererNode.cpp index 88713fd1..9cff5794 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoRendererNode.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoRendererNode.cpp @@ -53,7 +53,6 @@ IVideoRendererNode::IVideoRendererNode(BaseSessionCallback* callback) : mFirstFrame = false; mSubtype = MEDIASUBTYPE_UNDEFINED; mFramerate = 0; - mWaitIntraFrame = 0; mLossDuration = 0; mLossRateThreshold = 0; } @@ -87,7 +86,7 @@ ImsMediaResult IVideoRendererNode::Start() mVideoRenderer->SetDeviceOrientation(mDeviceOrientation); mVideoRenderer->SetSurface(mWindow); - if (mVideoRenderer->Start() == false) + if (!mVideoRenderer->Start()) { return RESULT_NOT_READY; } @@ -162,134 +161,105 @@ bool IVideoRendererNode::IsSameConfig(void* config) void IVideoRendererNode::ProcessData() { std::lock_guard<std::mutex> guard(mMutex); - uint8_t* pData = nullptr; - uint32_t nDataSize = 0; - uint32_t nTimeStamp = 0; - bool bMark = false; - uint32_t nSeqNum = 0; + uint8_t* data = nullptr; + uint32_t dataSize = 0; + uint32_t prevTimestamp = 0; + bool mark = false; + uint32_t seq = 0; uint32_t timestamp = 0; - uint32_t nBitstreamSize = 0; + uint32_t frameSize = 0; ImsMediaSubType subtype = MEDIASUBTYPE_UNDEFINED; - uint32_t nInitialSeqNum = 0; - uint32_t nBufferOffset = 0; + uint32_t initialSeq = 0; ImsMediaSubType dataType; - while (GetData(&subtype, &pData, &nDataSize, &nTimeStamp, &bMark, &nSeqNum, &dataType)) + while (GetData(&subtype, &data, &dataSize, ×tamp, &mark, &seq, &dataType)) { IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO, - "[ProcessData] subtype[%d], Size[%d], TimeStamp[%d] nBitstreamSize[%d]", subtype, - nDataSize, nTimeStamp, nBitstreamSize); + "[ProcessData] subtype[%d], Size[%d], TS[%d] frameSize[%d]", subtype, dataSize, + timestamp, frameSize); - if (timestamp == 0) + if (prevTimestamp == 0) { - timestamp = nTimeStamp; + prevTimestamp = timestamp; } - else if (timestamp != nTimeStamp) + else if (timestamp != prevTimestamp || (frameSize != 0 && hasStartingCode(data, dataSize))) { + // break when the timestamp is changed or next data has another starting code break; } - if (nDataSize >= MAX_RTP_PAYLOAD_BUFFER_SIZE) + if (dataSize >= MAX_RTP_PAYLOAD_BUFFER_SIZE) { - IMLOGE1("[ProcessData] exceed buffer size[%d]", nDataSize); + IMLOGE1("[ProcessData] exceed buffer size[%d]", dataSize); return; } - memcpy(mBuffer + nBitstreamSize, pData, nDataSize); - nBitstreamSize += nDataSize; + memcpy(mBuffer + frameSize, data, dataSize); + frameSize += dataSize; - if (nInitialSeqNum == 0) + if (initialSeq == 0) { - nInitialSeqNum = nSeqNum; + initialSeq = seq; } DeleteData(); - if (bMark) + if (mark) { break; } } - if (nBitstreamSize == 0) + if (frameSize == 0) { return; } // remove AUD nal unit - uint32_t nDatabufferSize = nBitstreamSize; - uint8_t* pDataBuff = mBuffer; - RemoveAUDNalUnit(mBuffer, nBitstreamSize, &pDataBuff, &nDatabufferSize); - - // check Config String for updating config frame - nBufferOffset = 0; - if ((mCodecType == kVideoCodecHevc || mCodecType == kVideoCodecAvc) && - IsConfigFrame(pDataBuff, nDatabufferSize, &nBufferOffset) == true) + uint32_t size = frameSize; + uint8_t* buffer = mBuffer; + RemoveAUDNalUnit(mBuffer, frameSize, &buffer, &size); + + FrameType frameType = GetFrameType(buffer, size); + + if (frameType == SPS) { - SaveConfigFrame(pDataBuff + nBufferOffset, nDatabufferSize - nBufferOffset, kConfigSps); - SaveConfigFrame(pDataBuff + nBufferOffset, nDatabufferSize - nBufferOffset, kConfigPps); + SaveConfigFrame(buffer, size, kConfigSps); + tCodecConfig codecConfig; - if (mCodecType == kVideoCodecHevc) + if (mCodecType == kVideoCodecAvc) { - SaveConfigFrame(pDataBuff + nBufferOffset, nDatabufferSize - nBufferOffset, kConfigVps); + if (ImsMediaVideoUtil::ParseAvcSps(buffer, size, &codecConfig)) + { + CheckResolution(codecConfig.nWidth, codecConfig.nHeight); + } } - - if (IsSps(pDataBuff, nDatabufferSize, &nBufferOffset) == true) + else if (mCodecType == kVideoCodecHevc) { - IMLOGD_PACKET1( - IM_PACKET_LOG_VIDEO, "[ProcessData] parse SPS - nOffset[%d]", nBufferOffset); - tCodecConfig codecConfig; - - if (mCodecType == kVideoCodecAvc) + if (ImsMediaVideoUtil::ParseHevcSps(buffer, size, &codecConfig)) { - if (ImsMediaVideoUtil::ParseAvcSps(pDataBuff + nBufferOffset, - nDatabufferSize - nBufferOffset, &codecConfig) == true) - { - CheckResolution(codecConfig.nWidth, codecConfig.nHeight); - } - } - else if (mCodecType == kVideoCodecHevc) - { - if (ImsMediaVideoUtil::ParseHevcSps(pDataBuff + nBufferOffset, - nDatabufferSize - nBufferOffset, &codecConfig) == true) - { - CheckResolution(codecConfig.nWidth, codecConfig.nHeight); - } + CheckResolution(codecConfig.nWidth, codecConfig.nHeight); } } return; } - - IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[ProcessData] nBitstreamSize[%d] nDatabufferSize[%d]", - nBitstreamSize, nDatabufferSize); - - bool isIntraFrame = IsIntraFrame(pDataBuff, nDatabufferSize); - - // drop non-idr frame when idr frame is not received - if (mWaitIntraFrame > 0 && nDatabufferSize > 0) + else if (frameType == PPS) { - if (isIntraFrame) - { - mWaitIntraFrame = 0; - } - else - { - // Send FIR when I-frame wasn't received - if ((mWaitIntraFrame % mFramerate) == 0) // every 1 sec - { - // TODO: send PLI event - IMLOGD0("[ProcessData] request Send PLI"); - } + SaveConfigFrame(buffer, size, kConfigPps); + return; + } + else if (frameType == VPS) + { + SaveConfigFrame(buffer, size, kConfigVps); + return; + } - mWaitIntraFrame--; - nDatabufferSize = 0; // drop non-DIR frame + IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[ProcessData] frame type[%d] size[%d]", frameType, size); - IMLOGD1("[ProcessData] wait intra frame[%d]", mWaitIntraFrame); - } - } + // TODO: Send PLI or FIR when I-frame wasn't received since beginning. - if (mFirstFrame == false) + if (!mFirstFrame) { IMLOGD0("[ProcessData] notify first frame"); mFirstFrame = true; @@ -297,6 +267,11 @@ void IVideoRendererNode::ProcessData() if (mCallback != nullptr) { mCallback->SendEvent(kImsMediaEventFirstPacketReceived); + + if (mCvoValue <= 0) + { + mCallback->SendEvent(kImsMediaEventResolutionChanged, mWidth, mHeight); + } } } @@ -308,7 +283,8 @@ void IVideoRendererNode::ProcessData() subtype = MEDIASUBTYPE_ROT0; } - if (mSubtype != subtype) + // rotation changed + if (mSubtype != subtype && (subtype >= MEDIASUBTYPE_ROT0 && subtype <= MEDIASUBTYPE_ROT270)) { mSubtype = subtype; int degree = 0; @@ -335,13 +311,13 @@ void IVideoRendererNode::ProcessData() } } - // send sps/pps before send I frame - if (isIntraFrame) + // send config frames before send I frame + if (frameType == IDR) { QueueConfigFrame(timestamp); } - mVideoRenderer->OnDataFrame(pDataBuff, nDatabufferSize, timestamp, false); + mVideoRenderer->OnDataFrame(buffer, size, timestamp, false); } void IVideoRendererNode::UpdateSurface(ANativeWindow* window) @@ -371,214 +347,84 @@ void IVideoRendererNode::SetPacketLossParam(uint32_t time, uint32_t rate) mLossRateThreshold = rate; } -bool IVideoRendererNode::IsIntraFrame(uint8_t* pbBuffer, uint32_t nBufferSize) +bool IVideoRendererNode::hasStartingCode(uint8_t* buffer, uint32_t bufferSize) { - bool bIntraFrame = false; - - if (nBufferSize <= 4) + if (bufferSize <= 4) { return false; } - IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[IsIntraFrame] size[%d], data[%s]", nBufferSize, - ImsMediaTrace::IMTrace_Bin2String( - reinterpret_cast<const char*>(pbBuffer), nBufferSize > 16 ? 16 : nBufferSize)); - - switch (mCodecType) + // Check for NAL unit delimiter 0x00000001 + if (buffer[0] == 0x00 && buffer[1] == 0x00 && buffer[2] == 0x00 && buffer[3] == 0x01) { - case kVideoCodecAvc: - { - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - - while (nCurrSize >= 5) - { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && (nCurrBuff[4] & 0x1F) == 5) - { - bIntraFrame = true; - break; - } - nCurrBuff++; - nCurrSize--; - } - - break; - } - case kVideoCodecHevc: - { - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - while (nCurrSize >= 5) - { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && - (((nCurrBuff[4] >> 1) & 0x3F) == 19 || ((nCurrBuff[4] >> 1) & 0x3F) == 20)) - { - bIntraFrame = true; - break; - } - nCurrBuff++; - nCurrSize--; - } - break; - } - default: - IMLOGE1("[IsIntraFrame] Invalid video codec type %d", mCodecType); - return true; + return true; } - return bIntraFrame; + return false; } -bool IVideoRendererNode::IsConfigFrame( - uint8_t* pbBuffer, uint32_t nBufferSize, uint32_t* nBufferOffset) +FrameType IVideoRendererNode::GetFrameType(uint8_t* buffer, uint32_t bufferSize) { - bool bConfigFrame = false; - - if (nBufferSize <= 4) - return false; + if (!hasStartingCode(buffer, bufferSize)) + { + return UNKNOWN; + } - IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[IsConfigFrame] size[%d], data[%s]", nBufferSize, - ImsMediaTrace::IMTrace_Bin2String( - reinterpret_cast<const char*>(pbBuffer), nBufferSize > 16 ? 16 : nBufferSize)); + uint8_t nalType = buffer[4]; switch (mCodecType) { case kVideoCodecAvc: { - uint32_t nOffset = 0; - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - - while (nCurrSize >= 5) + if ((nalType & 0x1F) == 5) { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && - ((nCurrBuff[4] & 0x1F) == 7 || ((nCurrBuff[4] & 0x1F) == 8))) - { - bConfigFrame = true; - - if (nBufferOffset) - { - *nBufferOffset = nOffset; - } - break; - } - - nOffset++; - nCurrBuff++; - nCurrSize--; + return IDR; } - break; - } - case kVideoCodecHevc: - { - uint32_t nOffset = 0; - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - - while (nCurrSize >= 5) + else if ((nalType & 0x1F) == 7) { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && - (((nCurrBuff[4] >> 1) & 0x3F) == 32 || ((nCurrBuff[4] >> 1) & 0x3F) == 33 || - ((nCurrBuff[4] >> 1) & 0x3F) == 34)) - { - bConfigFrame = true; - if (nBufferOffset) - { - *nBufferOffset = nOffset; - } - break; - } - nOffset++; - nCurrBuff++; - nCurrSize--; + return SPS; } - break; - } - default: - return false; - } - - return bConfigFrame; -} - -bool IVideoRendererNode::IsSps(uint8_t* pbBuffer, uint32_t nBufferSize, uint32_t* nBufferOffset) -{ - bool bSPS = false; - if (nBufferSize <= 4) - { - return false; - } - - IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[IsSps] size[%d], data[%s]", nBufferSize, - ImsMediaTrace::IMTrace_Bin2String( - reinterpret_cast<const char*>(pbBuffer), nBufferSize > 16 ? 16 : nBufferSize)); - - switch (mCodecType) - { - case kVideoCodecAvc: - { - uint32_t nOffset = 0; - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - - while (nCurrSize >= 5) + else if ((nalType & 0x1F) == 8) { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && (nCurrBuff[4] & 0x1F) == 7) - { - bSPS = true; - - if (nBufferOffset) - { - *nBufferOffset = nOffset; - } - - break; - } - - nOffset++; - nCurrBuff++; - nCurrSize--; + return PPS; + } + else + { + return NonIDR; } break; } case kVideoCodecHevc: { - uint32_t nOffset = 0; - uint32_t nCurrSize = nBufferSize; - uint8_t* nCurrBuff = pbBuffer; - - while (nCurrSize >= 5) + if (((nalType >> 1) & 0x3F) == 19 || ((nalType >> 1) & 0x3F) == 20) { - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && ((nCurrBuff[4] >> 1) & 0x3F) == 33) - { - bSPS = true; - - if (nBufferOffset) - { - *nBufferOffset = nOffset; - } - - break; - } - - nOffset++; - nCurrBuff++; - nCurrSize--; + return IDR; + } + else if (((nalType >> 1) & 0x3F) == 32) + { + return VPS; + } + else if (((nalType >> 1) & 0x3F) == 33) + { + return SPS; + } + else if (((nalType >> 1) & 0x3F) == 34) + { + return PPS; + } + else + { + return NonIDR; } + break; } default: - return false; + IMLOGE1("[GetFrameType] Invalid video codec type %d", mCodecType); } - return bSPS; + return UNKNOWN; } void IVideoRendererNode::SaveConfigFrame(uint8_t* pbBuffer, uint32_t nBufferSize, uint32_t eMode) @@ -610,18 +456,17 @@ void IVideoRendererNode::SaveConfigFrame(uint8_t* pbBuffer, uint32_t nBufferSize if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && nCurrBuff[3] == 0x01) { - if (eMode == kConfigSps && bSPSString == false && ((nCurrBuff[4] & 0x1F) == 7)) + if (eMode == kConfigSps && !bSPSString && ((nCurrBuff[4] & 0x1F) == 7)) { nOffset = nCurrSize; bSPSString = true; } - else if (eMode == kConfigPps && bPPSString == false && - ((nCurrBuff[4] & 0x1F) == 8)) + else if (eMode == kConfigPps && !bPPSString && ((nCurrBuff[4] & 0x1F) == 8)) { nOffset = nCurrSize; bPPSString = true; } - else if (bSPSString == true || bPPSString == true) + else if (bSPSString || bPPSString) { nConfigSize = nCurrSize - nOffset; break; @@ -684,21 +529,20 @@ void IVideoRendererNode::SaveConfigFrame(uint8_t* pbBuffer, uint32_t nBufferSize if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && nCurrBuff[3] == 0x01) { - if (eMode == kConfigVps && bVPSString == false && - (((nCurrBuff[4] >> 1) & 0x3F) == 32)) + if (eMode == kConfigVps && !bVPSString && (((nCurrBuff[4] >> 1) & 0x3F) == 32)) { nOffset = nCurrSize; bVPSString = true; break; } - else if (eMode == kConfigSps && bSPSString == false && + else if (eMode == kConfigSps && !bSPSString && (((nCurrBuff[4] >> 1) & 0x3F) == 33)) { nOffset = nCurrSize; bSPSString = true; break; } - else if (eMode == kConfigPps && bPPSString == false && + else if (eMode == kConfigPps && !bPPSString && (((nCurrBuff[4] >> 1) & 0x3F) == 34)) { nOffset = nCurrSize; @@ -711,7 +555,7 @@ void IVideoRendererNode::SaveConfigFrame(uint8_t* pbBuffer, uint32_t nBufferSize nCurrSize++; } - if (bVPSString == true || bSPSString == true || bPPSString == true) + if (bVPSString || bSPSString || bPPSString) { if ((nBufferSize - nOffset) > 0) { @@ -762,13 +606,13 @@ void IVideoRendererNode::SaveConfigFrame(uint8_t* pbBuffer, uint32_t nBufferSize } bool IVideoRendererNode::RemoveAUDNalUnit( - uint8_t* pInBuffer, uint32_t nInBufferSize, uint8_t** ppOutBuffer, uint32_t* pOutBufferSize) + uint8_t* inBuffer, uint32_t inBufferSize, uint8_t** outBuffer, uint32_t* outBufferSize) { - bool bAUDUnit = false; - *ppOutBuffer = pInBuffer; - *pOutBufferSize = nInBufferSize; + bool IsAudUnit = false; + *outBuffer = inBuffer; + *outBufferSize = inBufferSize; - if (nInBufferSize <= 4) + if (inBufferSize <= 4) { return false; } @@ -777,29 +621,29 @@ bool IVideoRendererNode::RemoveAUDNalUnit( { case kVideoCodecAvc: { - uint32_t nCurrSize = nInBufferSize; - uint8_t* nCurrBuff = pInBuffer; - uint32_t nCnt = 0; + uint32_t currSize = inBufferSize; + uint8_t* currBuffer = inBuffer; + uint32_t count = 0; - while (nCurrSize >= 5 && nCnt <= 12) + while (currSize >= 5 && count <= 12) { - if (bAUDUnit == true && - (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01)) + if (IsAudUnit && + (currBuffer[0] == 0x00 && currBuffer[1] == 0x00 && currBuffer[2] == 0x00 && + currBuffer[3] == 0x01)) { - *ppOutBuffer = nCurrBuff; - *pOutBufferSize = nCurrSize; + *outBuffer = currBuffer; + *outBufferSize = currSize; break; } - if (nCurrBuff[0] == 0x00 && nCurrBuff[1] == 0x00 && nCurrBuff[2] == 0x00 && - nCurrBuff[3] == 0x01 && nCurrBuff[4] == 0x09) + if (currBuffer[0] == 0x00 && currBuffer[1] == 0x00 && currBuffer[2] == 0x00 && + currBuffer[3] == 0x01 && currBuffer[4] == 0x09) { - bAUDUnit = true; + IsAudUnit = true; } - nCurrBuff++; - nCurrSize--; - nCnt++; + currBuffer++; + currSize--; + count++; } } break; @@ -808,7 +652,7 @@ bool IVideoRendererNode::RemoveAUDNalUnit( return false; } - return bAUDUnit; + return IsAudUnit; } void IVideoRendererNode::CheckResolution(uint32_t nWidth, uint32_t nHeight) @@ -838,16 +682,16 @@ void IVideoRendererNode::QueueConfigFrame(uint32_t timestamp) for (int32_t i = 0; i < nNumOfConfigString; i++) { - uint8_t* pConfigData = nullptr; - uint32_t nConfigLen = mConfigLen[i]; - pConfigData = mConfigBuffer[i]; + uint8_t* configFrame = nullptr; + uint32_t configLen = mConfigLen[i]; + configFrame = mConfigBuffer[i]; - if (nConfigLen == 0 || mVideoRenderer == nullptr) + if (configLen == 0 || mVideoRenderer == nullptr) { continue; } - mVideoRenderer->OnDataFrame(pConfigData, nConfigLen, timestamp, true); + mVideoRenderer->OnDataFrame(configFrame, configLen, timestamp, true); } } @@ -858,6 +702,8 @@ void IVideoRendererNode::NotifyPeerDimensionChanged() return; } + IMLOGD1("[NotifyPeerDimensionChanged] subtype[%d]", mSubtype); + // assume the device is portrait if (mWidth > mHeight) // landscape { @@ -869,7 +715,7 @@ void IVideoRendererNode::NotifyPeerDimensionChanged() { mCallback->SendEvent(kImsMediaEventResolutionChanged, mWidth, mHeight); } - else + else if (mSubtype == MEDIASUBTYPE_ROT90 || mSubtype == MEDIASUBTYPE_ROT270) { mCallback->SendEvent(kImsMediaEventResolutionChanged, mHeight, mWidth); } @@ -881,7 +727,7 @@ void IVideoRendererNode::NotifyPeerDimensionChanged() { mCallback->SendEvent(kImsMediaEventResolutionChanged, mHeight, mWidth); } - else + else if (mSubtype == MEDIASUBTYPE_ROT90 || mSubtype == MEDIASUBTYPE_ROT270) { mCallback->SendEvent(kImsMediaEventResolutionChanged, mWidth, mHeight); } @@ -894,7 +740,7 @@ void IVideoRendererNode::NotifyPeerDimensionChanged() { mCallback->SendEvent(kImsMediaEventResolutionChanged, mWidth, mHeight); } - else + else if (mSubtype == MEDIASUBTYPE_ROT90 || mSubtype == MEDIASUBTYPE_ROT270) { mCallback->SendEvent(kImsMediaEventResolutionChanged, mHeight, mWidth); } diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoSourceNode.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoSourceNode.cpp index 65b429e2..c95f6240 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoSourceNode.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/IVideoSourceNode.cpp @@ -45,6 +45,8 @@ IVideoSourceNode::IVideoSourceNode(BaseSessionCallback* callback) : mImagePath = ""; mDeviceOrientation = 0; mWindow = nullptr; + mMinBitrateThreshold = 0; + mBitrateNotified = false; } IVideoSourceNode::~IVideoSourceNode() {} @@ -57,7 +59,6 @@ kBaseNodeId IVideoSourceNode::GetNodeId() ImsMediaResult IVideoSourceNode::Start() { IMLOGD3("[Start] codec[%d], mode[%d], cameraId[%d]", mCodecType, mVideoMode, mCameraId); - std::lock_guard<std::mutex> guard(mMutex); if (mVideoSource) { @@ -84,12 +85,13 @@ ImsMediaResult IVideoSourceNode::Start() mVideoSource->SetSurface(mWindow); - if (mVideoSource->Start() == false) + if (!mVideoSource->Start()) { return RESULT_NOT_READY; } mVideoSource->SetDeviceOrientation(mDeviceOrientation); + mBitrateNotified = false; } mNodeState = kNodeStateRunning; @@ -99,7 +101,6 @@ ImsMediaResult IVideoSourceNode::Start() void IVideoSourceNode::Stop() { IMLOGD0("[Stop]"); - std::lock_guard<std::mutex> guard(mMutex); if (mVideoSource) { @@ -263,9 +264,9 @@ void IVideoSourceNode::UpdateSurface(ANativeWindow* window) mWindow = window; } -void IVideoSourceNode::OnUplinkEvent(uint8_t* data, uint32_t size, int64_t timestamp, uint32_t flag) +void IVideoSourceNode::OnUplinkEvent( + uint8_t* data, uint32_t size, int64_t timestamp, uint32_t /*flag*/) { - (void)flag; IMLOGD_PACKET2( IM_PACKET_LOG_VIDEO, "[OnUplinkEvent] size[%zu], timestamp[%ld]", size, timestamp); std::lock_guard<std::mutex> guard(mMutex); @@ -277,6 +278,12 @@ void IVideoSourceNode::OnUplinkEvent(uint8_t* data, uint32_t size, int64_t times } } +void IVideoSourceNode::SetBitrateThreshold(int32_t bitrate) +{ + IMLOGD1("[SetBitrateThreshold] bitrate[%d]", bitrate); + mMinBitrateThreshold = bitrate; +} + void IVideoSourceNode::OnEvent(int32_t type, int32_t param1, int32_t param2) { IMLOGD3("[OnEvent] type[%d], param1[%d], param2[%d]", type, param1, param2); @@ -298,7 +305,15 @@ void IVideoSourceNode::OnEvent(int32_t type, int32_t param1, int32_t param2) case kRequestVideoBitrateChange: if (mVideoSource != nullptr) { - mVideoSource->changeBitrate(param1); + if (mVideoSource->changeBitrate(param1)) + { + if (mMinBitrateThreshold != 0 && param1 <= mMinBitrateThreshold && + mCallback != nullptr && !mBitrateNotified) + { + mCallback->SendEvent(kImsMediaEventNotifyVideoLowestBitrate, param1); + mBitrateNotified = true; + } + } } break; case kRequestVideoIdrFrame: diff --git a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/VideoRtpPayloadEncoderNode.cpp b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/VideoRtpPayloadEncoderNode.cpp index 6ec6a3cd..e601c16c 100644 --- a/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/VideoRtpPayloadEncoderNode.cpp +++ b/service/src/com/android/telephony/imsmedia/lib/libimsmedia/core/video/nodes/VideoRtpPayloadEncoderNode.cpp @@ -502,7 +502,7 @@ void VideoRtpPayloadEncoderNode::EncodeHevc( memset(mPPS, 0, MAX_CONFIG_LEN); memcpy(mPPS, pCurDataPos, nCurDataSize); mPpsSize = nCurDataSize; - IMLOGD1("[EncodeHevc] PPS Size [%d]", mSpsSize); + IMLOGD1("[EncodeHevc] PPS Size [%d]", mPpsSize); } if (nDataSize < nCurDataSize + 4) |