From 28032b2737bd058b3894dcccbf19859a766886eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1n=20Sebechlebsk=C3=BD?= Date: Thu, 9 Nov 2023 10:48:51 +0000 Subject: Revert^2 "Move rendering to dedicated thread." This reverts commit cfc4361ac90b4cf08216d2726f7f8f52c4e44528. Reason for revert: Test failure fixed by ag/25312696. Change-Id: Ib9a7ca2e064bf4d2af5fb6e91e9439cfa815fe48 --- devices/VirtualCamera/Android.bp | 2 + devices/VirtualCamera/VirtualCameraRenderThread.cc | 386 +++++++++++++++++++++ devices/VirtualCamera/VirtualCameraRenderThread.h | 165 +++++++++ devices/VirtualCamera/VirtualCameraSession.cc | 354 +++---------------- devices/VirtualCamera/VirtualCameraSession.h | 41 +-- .../VirtualCamera/VirtualCameraSessionContext.cc | 171 +++++++++ .../VirtualCamera/VirtualCameraSessionContext.h | 98 ++++++ devices/VirtualCamera/VirtualCameraStream.cc | 57 ++- devices/VirtualCamera/VirtualCameraStream.h | 14 +- 9 files changed, 894 insertions(+), 394 deletions(-) create mode 100644 devices/VirtualCamera/VirtualCameraRenderThread.cc create mode 100644 devices/VirtualCamera/VirtualCameraRenderThread.h create mode 100644 devices/VirtualCamera/VirtualCameraSessionContext.cc create mode 100644 devices/VirtualCamera/VirtualCameraSessionContext.h diff --git a/devices/VirtualCamera/Android.bp b/devices/VirtualCamera/Android.bp index 5996ede..b4e9133 100644 --- a/devices/VirtualCamera/Android.bp +++ b/devices/VirtualCamera/Android.bp @@ -69,6 +69,8 @@ cc_library_static { "VirtualCameraSession.cc", "VirtualCameraStream.cc", "VirtualCameraService.cc", + "VirtualCameraSessionContext.cc", + "VirtualCameraRenderThread.cc", ], defaults: [ "libvirtualcamera_defaults", diff --git a/devices/VirtualCamera/VirtualCameraRenderThread.cc b/devices/VirtualCamera/VirtualCameraRenderThread.cc new file mode 100644 index 0000000..49e210d --- /dev/null +++ b/devices/VirtualCamera/VirtualCameraRenderThread.cc @@ -0,0 +1,386 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "VirtualCameraRenderThread" +#include "VirtualCameraRenderThread.h" + +#include +#include +#include +#include +#include +#include + +#include "VirtualCameraSessionContext.h" +#include "aidl/android/hardware/camera/common/Status.h" +#include "aidl/android/hardware/camera/device/BufferStatus.h" +#include "aidl/android/hardware/camera/device/CameraMetadata.h" +#include "aidl/android/hardware/camera/device/CaptureResult.h" +#include "aidl/android/hardware/camera/device/ErrorCode.h" +#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h" +#include "aidl/android/hardware/camera/device/ShutterMsg.h" +#include "aidl/android/hardware/camera/device/StreamBuffer.h" +#include "android-base/thread_annotations.h" +#include "android/binder_auto_utils.h" +#include "android/hardware_buffer.h" +#include "util/EglFramebuffer.h" +#include "util/JpegUtil.h" +#include "util/MetadataBuilder.h" +#include "util/TestPatternHelper.h" +#include "util/Util.h" +#include "utils/Errors.h" + +namespace android { +namespace companion { +namespace virtualcamera { + +using ::aidl::android::hardware::camera::common::Status; +using ::aidl::android::hardware::camera::device::BufferStatus; +using ::aidl::android::hardware::camera::device::CameraMetadata; +using ::aidl::android::hardware::camera::device::CaptureResult; +using ::aidl::android::hardware::camera::device::ErrorCode; +using ::aidl::android::hardware::camera::device::ErrorMsg; +using ::aidl::android::hardware::camera::device::ICameraDeviceCallback; +using ::aidl::android::hardware::camera::device::NotifyMsg; +using ::aidl::android::hardware::camera::device::ShutterMsg; +using ::aidl::android::hardware::camera::device::Stream; +using ::aidl::android::hardware::camera::device::StreamBuffer; +using ::aidl::android::hardware::graphics::common::PixelFormat; +using ::android::base::ScopedLockAssertion; + +namespace { + +using namespace std::chrono_literals; + +static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms; + +CameraMetadata createCaptureResultMetadata( + const std::chrono::nanoseconds timestamp) { + std::unique_ptr metadata = + MetadataBuilder().setSensorTimestamp(timestamp).build(); + if (metadata == nullptr) { + ALOGE("%s: Failed to build capture result metadata", __func__); + return CameraMetadata(); + } + return std::move(*metadata); +} + +NotifyMsg createShutterNotifyMsg(int frameNumber, + std::chrono::nanoseconds timestamp) { + NotifyMsg msg; + msg.set(ShutterMsg{ + .frameNumber = frameNumber, + .timestamp = timestamp.count(), + }); + return msg; +} + +NotifyMsg createErrorNotifyMsg(int frameNumber, int streamId, ErrorCode error) { + NotifyMsg msg; + msg.set(ErrorMsg{.frameNumber = frameNumber, + .errorStreamId = streamId, + .errorCode = error}); + return msg; +} + +} // namespace + +CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId, + sp fence) + : mStreamId(streamId), mBufferId(bufferId), mFence(fence) { +} + +int CaptureRequestBuffer::getStreamId() const { + return mStreamId; +} + +int CaptureRequestBuffer::getBufferId() const { + return mBufferId; +} + +sp CaptureRequestBuffer::getFence() const { + return mFence; +} + +VirtualCameraRenderThread::VirtualCameraRenderThread( + VirtualCameraSessionContext& sessionContext, const int mWidth, + const int mHeight, + std::shared_ptr cameraDeviceCallback, bool testMode) + : mCameraDeviceCallback(cameraDeviceCallback), + mInputSurfaceWidth(mWidth), + mInputSurfaceHeight(mHeight), + mTestMode(testMode), + mSessionContext(sessionContext) { +} + +VirtualCameraRenderThread::~VirtualCameraRenderThread() { + stop(); + if (mThread.joinable()) { + mThread.join(); + } +} + +ProcessCaptureRequestTask::ProcessCaptureRequestTask( + int frameNumber, const std::vector& requestBuffers) + : mFrameNumber(frameNumber), mBuffers(requestBuffers) { +} + +int ProcessCaptureRequestTask::getFrameNumber() const { + return mFrameNumber; +} + +const std::vector& ProcessCaptureRequestTask::getBuffers() + const { + return mBuffers; +} + +void VirtualCameraRenderThread::enqueueTask( + std::unique_ptr task) { + std::lock_guard lock(mLock); + mQueue.emplace_back(std::move(task)); + mCondVar.notify_one(); +} + +void VirtualCameraRenderThread::start() { + mThread = std::thread(&VirtualCameraRenderThread::threadLoop, this); +} + +void VirtualCameraRenderThread::stop() { + { + std::lock_guard lock(mLock); + mPendingExit = true; + mCondVar.notify_one(); + } +} + +sp VirtualCameraRenderThread::getInputSurface() { + return mInputSurfacePromise.get_future().get(); +} + +std::unique_ptr +VirtualCameraRenderThread::dequeueTask() { + std::unique_lock lock(mLock); + // Clang's thread safety analysis doesn't perform alias analysis, + // so it doesn't support moveable std::unique_lock. + // + // Lock assertion below is basically explicit declaration that + // the lock is held in this scope, which is true, since it's only + // released during waiting inside mCondVar.wait calls. + ScopedLockAssertion lockAssertion(mLock); + + mCondVar.wait(lock, [this]() REQUIRES(mLock) { + return mPendingExit || !mQueue.empty(); + }); + if (mPendingExit) { + return nullptr; + } + std::unique_ptr task = std::move(mQueue.front()); + mQueue.pop_front(); + return task; +} + +void VirtualCameraRenderThread::threadLoop() { + ALOGV("Render thread starting"); + + mEglDisplayContext = std::make_unique(); + mEglTextureProgram = std::make_unique(); + mEglSurfaceTexture = std::make_unique(mInputSurfaceWidth, + mInputSurfaceHeight); + mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface()); + + while (std::unique_ptr task = dequeueTask()) { + processCaptureRequest(*task); + } + + ALOGV("Render thread exiting"); +} + +void VirtualCameraRenderThread::processCaptureRequest( + const ProcessCaptureRequestTask& request) { + const std::chrono::nanoseconds timestamp = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()); + + CaptureResult captureResult; + captureResult.fmqResultSize = 0; + captureResult.frameNumber = request.getFrameNumber(); + captureResult.partialResult = 1; + captureResult.inputBuffer.streamId = -1; + captureResult.physicalCameraMetadata.resize(0); + captureResult.result = createCaptureResultMetadata(timestamp); + + const std::vector& buffers = request.getBuffers(); + captureResult.outputBuffers.resize(buffers.size()); + + if (mTestMode) { + // In test mode let's just render something to the Surface ourselves. + renderTestPatternYCbCr420(mEglSurfaceTexture->getSurface(), + request.getFrameNumber()); + } + + mEglSurfaceTexture->updateTexture(); + + for (int i = 0; i < buffers.size(); ++i) { + const CaptureRequestBuffer& reqBuffer = buffers[i]; + StreamBuffer& resBuffer = captureResult.outputBuffers[i]; + resBuffer.streamId = reqBuffer.getStreamId(); + resBuffer.bufferId = reqBuffer.getBufferId(); + resBuffer.status = BufferStatus::OK; + + const std::optional streamConfig = + mSessionContext.getStreamConfig(reqBuffer.getStreamId()); + + if (!streamConfig.has_value()) { + resBuffer.status = BufferStatus::ERROR; + continue; + } + + auto status = streamConfig->format == PixelFormat::BLOB + ? renderIntoBlobStreamBuffer( + reqBuffer.getStreamId(), reqBuffer.getBufferId(), + streamConfig->bufferSize, reqBuffer.getFence()) + : renderIntoImageStreamBuffer(reqBuffer.getStreamId(), + reqBuffer.getBufferId(), + reqBuffer.getFence()); + if (!status.isOk()) { + resBuffer.status = BufferStatus::ERROR; + } + } + + std::vector notifyMsg{ + createShutterNotifyMsg(request.getFrameNumber(), timestamp)}; + for (const StreamBuffer& resBuffer : captureResult.outputBuffers) { + if (resBuffer.status != BufferStatus::OK) { + notifyMsg.push_back(createErrorNotifyMsg(request.getFrameNumber(), + resBuffer.streamId, + ErrorCode::ERROR_BUFFER)); + } + } + + auto status = mCameraDeviceCallback->notify(notifyMsg); + if (!status.isOk()) { + ALOGE("%s: notify call failed: %s", __func__, + status.getDescription().c_str()); + return; + } + + std::vector<::aidl::android::hardware::camera::device::CaptureResult> + captureResults(1); + captureResults[0] = std::move(captureResult); + + status = mCameraDeviceCallback->processCaptureResult(captureResults); + if (!status.isOk()) { + ALOGE("%s: processCaptureResult call failed: %s", __func__, + status.getDescription().c_str()); + return; + } + + ALOGD("%s: Successfully called processCaptureResult", __func__); +} + +ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( + const int streamId, const int bufferId, const size_t bufferSize, + sp fence) { + ALOGV("%s", __func__); + sp gBuffer = mEglSurfaceTexture->getCurrentBuffer(); + std::shared_ptr hwBuffer = + mSessionContext.fetchHardwareBuffer(streamId, bufferId); + + AHardwareBuffer_Planes planes_info; + + int32_t rawFence = fence != nullptr ? fence->get() : -1; + int result = AHardwareBuffer_lockPlanes(hwBuffer.get(), + AHARDWAREBUFFER_USAGE_CPU_READ_RARELY, + rawFence, nullptr, &planes_info); + if (result != OK) { + ALOGE("%s: Failed to lock planes for BLOB buffer: %d", __func__, result); + return cameraStatus(Status::INTERNAL_ERROR); + } + + android_ycbcr ycbcr; + status_t status = + gBuffer->lockYCbCr(AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, &ycbcr); + ALOGV("Locked buffers"); + if (status != NO_ERROR) { + AHardwareBuffer_unlock(hwBuffer.get(), nullptr); + ALOGE("%s: Failed to lock graphic buffer: %d", __func__, status); + return cameraStatus(Status::INTERNAL_ERROR); + } + + bool success = compressJpeg(gBuffer->getWidth(), gBuffer->getHeight(), ycbcr, + bufferSize, planes_info.planes[0].data); + + status_t res = gBuffer->unlock(); + if (res != NO_ERROR) { + ALOGE("Failed to unlock graphic buffer: %d", res); + } + AHardwareBuffer_unlock(hwBuffer.get(), nullptr); + ALOGV("Unlocked buffers"); + return success ? ndk::ScopedAStatus::ok() + : cameraStatus(Status::INTERNAL_ERROR); +} + +ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoImageStreamBuffer( + int streamId, int bufferId, sp fence) { + ALOGV("%s", __func__); + + const std::chrono::nanoseconds before = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()); + + // Render test pattern using EGL. + std::shared_ptr framebuffer = + mSessionContext.fetchOrCreateEglFramebuffer( + mEglDisplayContext->getEglDisplay(), streamId, bufferId); + if (framebuffer == nullptr) { + ALOGE( + "%s: Failed to get EGL framebuffer corresponding to buffer id " + "%d for streamId %d", + __func__, bufferId, streamId); + return cameraStatus(Status::ILLEGAL_ARGUMENT); + } + + // Wait for fence to clear. + if (fence != nullptr && fence->isValid()) { + status_t ret = fence->wait(kAcquireFenceTimeout.count()); + if (ret != 0) { + ALOGE( + "Timeout while waiting for the acquire fence for buffer %d" + " for streamId %d", + bufferId, streamId); + return cameraStatus(Status::INTERNAL_ERROR); + } + } + + mEglDisplayContext->makeCurrent(); + framebuffer->beforeDraw(); + + mEglTextureProgram->draw(mEglSurfaceTexture->updateTexture()); + framebuffer->afterDraw(); + + const std::chrono::nanoseconds after = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()); + + ALOGV("Rendering to buffer %d, stream %d took %lld ns", bufferId, streamId, + after.count() - before.count()); + + return ndk::ScopedAStatus::ok(); +} + +} // namespace virtualcamera +} // namespace companion +} // namespace android diff --git a/devices/VirtualCamera/VirtualCameraRenderThread.h b/devices/VirtualCamera/VirtualCameraRenderThread.h new file mode 100644 index 0000000..647b8e2 --- /dev/null +++ b/devices/VirtualCamera/VirtualCameraRenderThread.h @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H +#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H + +#include +#include +#include +#include + +#include "VirtualCameraSessionContext.h" +#include "aidl/android/hardware/camera/device/CaptureRequest.h" +#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h" +#include "android/hardware_buffer.h" +#include "util/EglDisplayContext.h" +#include "util/EglProgram.h" +#include "util/EglSurfaceTexture.h" + +namespace android { +namespace companion { +namespace virtualcamera { + +// Represents single output buffer of capture request. +class CaptureRequestBuffer { + public: + CaptureRequestBuffer(int streamId, int bufferId, sp fence = nullptr); + + int getStreamId() const; + int getBufferId() const; + sp getFence() const; + + private: + const int mStreamId; + const int mBufferId; + const sp mFence; +}; + +// Represents single capture request to fill set of buffers. +class ProcessCaptureRequestTask { + public: + ProcessCaptureRequestTask( + int frameNumber, const std::vector& requestBuffers); + + // Returns frame number corresponding to the request. + int getFrameNumber() const; + + // Get reference to vector describing output buffers corresponding + // to this request. + // + // Note that the vector is owned by the ProcessCaptureRequestTask instance, + // so it cannot be access outside of its lifetime. + const std::vector& getBuffers() const; + + private: + const int mFrameNumber; + const std::vector mBuffers; +}; + +// Wraps dedicated rendering thread and rendering business with corresponding +// input surface. +class VirtualCameraRenderThread { + public: + // Create VirtualCameraRenderThread instance: + // * sessionContext - VirtualCameraSessionContext reference for shared access + // to mapped buffers. + // * inputWidth - requested width of input surface ("virtual camera sensor") + // * inputHeight - requested height of input surface ("virtual camera sensor") + // * cameraDeviceCallback - callback for corresponding camera instance + // * testMode - when set to true, test pattern is rendered to input surface + // before each capture request is processed to simulate client input. + VirtualCameraRenderThread( + VirtualCameraSessionContext& sessionContext, int inputWidth, + int inputHeight, + std::shared_ptr< + ::aidl::android::hardware::camera::device::ICameraDeviceCallback> + cameraDeviceCallback, + bool testMode = false); + + ~VirtualCameraRenderThread(); + + // Start rendering thread. + void start(); + // Stop rendering thread. + void stop(); + + // Equeue capture task for processing on render thread. + void enqueueTask(std::unique_ptr task); + + // Returns input surface corresponding to "virtual camera sensor". + sp getInputSurface(); + + private: + std::unique_ptr dequeueTask(); + + // Rendering thread entry point. + void threadLoop(); + + // Process single capture request task (always called on render thread). + void processCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask); + + // TODO(b/301023410) - Refactor the actual rendering logic off this class for + // easier testability. + + // Render current image to the BLOB buffer. + // If fence is specified, this function will block until the fence is cleared + // before writing to the buffer. + // Always called on render thread. + ndk::ScopedAStatus renderIntoBlobStreamBuffer(const int streamId, + const int bufferId, + const size_t bufferSize, + sp fence = nullptr); + + // Render current image to the YCbCr buffer. + // If fence is specified, this function will block until the fence is cleared + // before writing to the buffer. + // Always called on render thread. + ndk::ScopedAStatus renderIntoImageStreamBuffer(int streamId, int bufferId, + sp fence = nullptr); + + // Camera callback + const std::shared_ptr< + ::aidl::android::hardware::camera::device::ICameraDeviceCallback> + mCameraDeviceCallback; + + const int mInputSurfaceWidth; + const int mInputSurfaceHeight; + const int mTestMode; + + VirtualCameraSessionContext& mSessionContext; + + std::thread mThread; + + // Blocking queue implementation. + std::mutex mLock; + std::deque> mQueue GUARDED_BY(mLock); + std::condition_variable mCondVar; + volatile bool mPendingExit GUARDED_BY(mLock); + + // EGL helpers - constructed and accessed only from rendering thread. + std::unique_ptr mEglDisplayContext; + std::unique_ptr mEglTextureProgram; + std::unique_ptr mEglSurfaceTexture; + + std::promise> mInputSurfacePromise; +}; + +} // namespace virtualcamera +} // namespace companion +} // namespace android + +#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H diff --git a/devices/VirtualCamera/VirtualCameraSession.cc b/devices/VirtualCamera/VirtualCameraSession.cc index 96a765e..eb9bac1 100644 --- a/devices/VirtualCamera/VirtualCameraSession.cc +++ b/devices/VirtualCamera/VirtualCameraSession.cc @@ -34,6 +34,7 @@ #include "CameraMetadata.h" #include "EGL/egl.h" +#include "VirtualCameraRenderThread.h" #include "VirtualCameraStream.h" #include "aidl/android/hardware/camera/common/Status.h" #include "aidl/android/hardware/camera/device/BufferCache.h" @@ -42,6 +43,7 @@ #include "aidl/android/hardware/camera/device/HalStream.h" #include "aidl/android/hardware/camera/device/NotifyMsg.h" #include "aidl/android/hardware/camera/device/ShutterMsg.h" +#include "aidl/android/hardware/camera/device/StreamBuffer.h" #include "aidl/android/hardware/camera/device/StreamConfiguration.h" #include "aidl/android/hardware/camera/device/StreamRotation.h" #include "aidl/android/hardware/graphics/common/BufferUsage.h" @@ -66,18 +68,13 @@ namespace virtualcamera { using ::aidl::android::companion::virtualcamera::IVirtualCameraCallback; using ::aidl::android::hardware::camera::common::Status; using ::aidl::android::hardware::camera::device::BufferCache; -using ::aidl::android::hardware::camera::device::BufferStatus; using ::aidl::android::hardware::camera::device::CameraMetadata; using ::aidl::android::hardware::camera::device::CameraOfflineSessionInfo; using ::aidl::android::hardware::camera::device::CaptureRequest; -using ::aidl::android::hardware::camera::device::ErrorCode; -using ::aidl::android::hardware::camera::device::ErrorMsg; using ::aidl::android::hardware::camera::device::HalStream; using ::aidl::android::hardware::camera::device::ICameraDeviceCallback; using ::aidl::android::hardware::camera::device::ICameraOfflineSession; -using ::aidl::android::hardware::camera::device::NotifyMsg; using ::aidl::android::hardware::camera::device::RequestTemplate; -using ::aidl::android::hardware::camera::device::ShutterMsg; using ::aidl::android::hardware::camera::device::Stream; using ::aidl::android::hardware::camera::device::StreamBuffer; using ::aidl::android::hardware::camera::device::StreamConfiguration; @@ -95,8 +92,6 @@ using metadata_ptr = using namespace std::chrono_literals; -static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms; - // Size of request/result metadata fast message queue. // Setting to 0 to always disables FMQ. static constexpr size_t kMetadataMsgQueueSize = 0; @@ -104,12 +99,6 @@ static constexpr size_t kMetadataMsgQueueSize = 0; // Maximum number of buffers to use per single stream. static constexpr size_t kMaxStreamBuffers = 2; -// Whether to use EGL for rendering - will be removed soon. -static constexpr bool kUseEGL = true; -// Whether to render through Surface -> EGL Texture -> Camera buffer. -// // TODO(b/301023410) Will be removed once we expose surface to the client API. -static constexpr bool kRenderThroughSurfaceTexture = true; - CameraMetadata createDefaultRequestSettings(RequestTemplate type) { hardware::camera::common::V1_0::helper::CameraMetadata metadataHelper; @@ -137,35 +126,6 @@ CameraMetadata createDefaultRequestSettings(RequestTemplate type) { return (metadata != nullptr) ? std::move(*metadata) : CameraMetadata(); } -CameraMetadata createCaptureResultMetadata( - const std::chrono::nanoseconds timestamp) { - std::unique_ptr metadata = - MetadataBuilder().setSensorTimestamp(timestamp).build(); - if (metadata == nullptr) { - ALOGE("%s: Failed to build capture result metadata", __func__); - return CameraMetadata(); - } - return std::move(*metadata); -} - -NotifyMsg createShutterNotifyMsg(int frameNumber, - std::chrono::nanoseconds timestamp) { - NotifyMsg msg; - msg.set(ShutterMsg{ - .frameNumber = frameNumber, - .timestamp = timestamp.count(), - }); - return msg; -} - -NotifyMsg createErrorNotifyMsg(int frameNumber, int streamId, ErrorCode error) { - NotifyMsg msg; - msg.set(ErrorMsg{.frameNumber = frameNumber, - .errorStreamId = streamId, - .errorCode = error}); - return msg; -} - HalStream getHalStream(const Stream& stream) { HalStream halStream; halStream.id = stream.id; @@ -183,8 +143,7 @@ HalStream getHalStream(const Stream& stream) { } halStream.overrideDataSpace = stream.dataSpace; - halStream.producerUsage = - kUseEGL ? BufferUsage::GPU_RENDER_TARGET : BufferUsage::CPU_WRITE_OFTEN; + halStream.producerUsage = BufferUsage::GPU_RENDER_TARGET; halStream.supportOffline = false; return halStream; } @@ -209,18 +168,6 @@ VirtualCameraSession::VirtualCameraSession( if (!mResultMetadataQueue->isValid()) { ALOGE("%s: invalid result fmq", __func__); } - - if (kUseEGL) { - mEglDisplayContext = std::make_unique(); - if (kRenderThroughSurfaceTexture) { - mEglTextureProgram = std::make_unique(); - // TODO(b/301023410) Initialize the texture based on supported formats and - // formats requested by client in configure call. - mEglSurfaceTexture = std::make_unique(640, 480); - } else { - mEglTestPatternProgram = std::make_unique(); - } - } } ndk::ScopedAStatus VirtualCameraSession::close() { @@ -230,8 +177,7 @@ ndk::ScopedAStatus VirtualCameraSession::close() { mVirtualCameraClientCallback->onStreamClosed(/*streamId=*/0); } - std::lock_guard lock(mLock); - mStreams.clear(); + mSessionContext.closeAllStreams(); return ndk::ScopedAStatus::ok(); } @@ -245,13 +191,18 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams( return cameraStatus(Status::ILLEGAL_ARGUMENT); } - removeStreamsNotInStreamConfiguration(in_requestedConfiguration); + mSessionContext.removeStreamsNotInStreamConfiguration( + in_requestedConfiguration); auto& streams = in_requestedConfiguration.streams; auto& halStreams = *_aidl_return; halStreams.clear(); halStreams.resize(in_requestedConfiguration.streams.size()); + sp inputSurface = nullptr; + int inputWidth; + int inputHeight; + { std::lock_guard lock(mLock); for (int i = 0; i < in_requestedConfiguration.streams.size(); ++i) { @@ -265,24 +216,31 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams( return cameraStatus(Status::ILLEGAL_ARGUMENT); } halStreams[i] = getHalStream(streams[i]); - const auto& [_, newlyInserted] = mStreams.emplace( - std::piecewise_construct, std::forward_as_tuple(streams[i].id), - std::forward_as_tuple(streams[i])); - if (newlyInserted) { + if (mSessionContext.initializeStream(streams[i])) { ALOGV("Configured new stream: %s", streams[i].toString().c_str()); } } + + inputWidth = streams[0].width; + inputHeight = streams[0].height; + if (mRenderThread == nullptr) { + // If there's no client callback, start camera in test mode. + const bool testMode = mVirtualCameraClientCallback == nullptr; + mRenderThread = std::make_unique( + mSessionContext, inputWidth, inputHeight, mCameraDeviceCallback, + testMode); + mRenderThread->start(); + inputSurface = mRenderThread->getInputSurface(); + } } - if (mVirtualCameraClientCallback != nullptr && kUseEGL) { + if (mVirtualCameraClientCallback != nullptr && inputSurface != nullptr) { // TODO(b/301023410) Pass streamId based on client input stream id once // support for multiple input streams is implemented. For now we always // create single texture. mVirtualCameraClientCallback->onStreamConfigured( - /*streamId=*/0, - aidl::android::view::Surface(mEglSurfaceTexture->getSurface().get()), - mEglSurfaceTexture->getWidth(), mEglSurfaceTexture->getHeight(), - PixelFormat::YCBCR_420_888); + /*streamId=*/0, aidl::android::view::Surface(inputSurface.get()), + inputWidth, inputHeight, PixelFormat::YCBCR_420_888); } mFirstRequest.store(true); @@ -357,7 +315,7 @@ ndk::ScopedAStatus VirtualCameraSession::processCaptureRequest( ALOGV("%s", __func__); if (!in_cachesToRemove.empty()) { - removeBufferCaches(in_cachesToRemove); + mSessionContext.removeBufferCaches(in_cachesToRemove); } for (const auto& captureRequest : in_requests) { @@ -406,92 +364,7 @@ ndk::ScopedAStatus VirtualCameraSession::repeatingRequestEnd( } std::set VirtualCameraSession::getStreamIds() const { - std::set result; - - std::lock_guard lock(mLock); - for (const auto& [streamId, _] : mStreams) { - result.insert(streamId); - } - return result; -} - -void VirtualCameraSession::removeBufferCaches( - const std::vector& cachesToRemove) { - std::lock_guard lock(mLock); - for (const auto& bufferCache : cachesToRemove) { - auto it = mStreams.find(bufferCache.streamId); - if (it == mStreams.end()) { - ALOGE("%s: Ask to remove buffer %" PRId64 " from unknown stream %d", - __func__, bufferCache.bufferId, bufferCache.streamId); - continue; - } - if (it->second.removeBuffer(bufferCache.bufferId)) { - ALOGD("%s: Successfully removed buffer %" PRId64 - " from cache of stream %d", - __func__, bufferCache.bufferId, bufferCache.streamId); - } else { - ALOGE("%s: Failed to buffer %" PRId64 " from cache of stream %d", - __func__, bufferCache.bufferId, bufferCache.streamId); - } - } -} - -void VirtualCameraSession::removeStreamsNotInStreamConfiguration( - const StreamConfiguration& streamConfiguration) { - std::unordered_set newConfigurationStreamIds; - for (const Stream& stream : streamConfiguration.streams) { - newConfigurationStreamIds.insert(stream.id); - } - - std::lock_guard lock(mLock); - for (auto it = mStreams.begin(); it != mStreams.end();) { - if (newConfigurationStreamIds.find(it->first) == - newConfigurationStreamIds.end()) { - ALOGV( - "Disposing of stream %d, since it is not referenced by new " - "configuration.", - it->first); - it = mStreams.erase(it); - } else { - ++it; - } - } -} - -std::optional VirtualCameraSession::getStreamConfig( - const StreamBuffer& streamBuffer) const { - std::lock_guard lock(mLock); - auto it = mStreams.find(streamBuffer.streamId); - if (it == mStreams.end()) { - ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, - streamBuffer.streamId); - return std::optional(); - } - return {it->second.getStreamConfig()}; -} - -std::shared_ptr VirtualCameraSession::fetchHardwareBuffer( - const StreamBuffer& streamBuffer) { - std::lock_guard lock(mLock); - auto it = mStreams.find(streamBuffer.streamId); - if (it == mStreams.end()) { - ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, - streamBuffer.streamId); - return nullptr; - } - return it->second.getHardwareBuffer(streamBuffer); -} - -std::shared_ptr VirtualCameraSession::fetchEglFramebuffer( - const EGLDisplay eglDisplay, const StreamBuffer& streamBuffer) { - std::lock_guard lock(mLock); - auto it = mStreams.find(streamBuffer.streamId); - if (it == mStreams.end()) { - ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, - streamBuffer.streamId); - return nullptr; - } - return it->second.getEglFrameBuffer(eglDisplay, streamBuffer); + return mSessionContext.getStreamIds(); } ndk::ScopedAStatus VirtualCameraSession::processCaptureRequest( @@ -516,172 +389,23 @@ ndk::ScopedAStatus VirtualCameraSession::processCaptureRequest( return cameraStatus(Status::INTERNAL_ERROR); } - const std::chrono::nanoseconds timestamp = - std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()); - - ::aidl::android::hardware::camera::device::CaptureResult captureResult; - captureResult.fmqResultSize = 0; - captureResult.frameNumber = request.frameNumber; - captureResult.result = request.settings; - captureResult.partialResult = 1; - captureResult.inputBuffer.streamId = -1; - captureResult.outputBuffers.resize(request.outputBuffers.size()); - captureResult.physicalCameraMetadata.resize(0); - captureResult.result = createCaptureResultMetadata(timestamp); - - for (int i = 0; i < request.outputBuffers.size(); ++i) { - const StreamBuffer& reqBuffer = request.outputBuffers[i]; - StreamBuffer& resBuffer = captureResult.outputBuffers[i]; - resBuffer.streamId = reqBuffer.streamId; - resBuffer.bufferId = reqBuffer.bufferId; - resBuffer.status = BufferStatus::OK; - - const std::optional streamConfig = getStreamConfig(reqBuffer); - - if (!streamConfig.has_value()) { - resBuffer.status = BufferStatus::ERROR; - continue; - } - - auto status = streamConfig->format == PixelFormat::BLOB - ? renderIntoBlobStreamBuffer(request, reqBuffer, - streamConfig->bufferSize) - : renderIntoImageStreamBuffer(request, reqBuffer); - if (!status.isOk()) { - resBuffer.status = BufferStatus::ERROR; - } - } - - std::vector notifyMsg{ - createShutterNotifyMsg(request.frameNumber, timestamp)}; - for (const StreamBuffer& resBuffer : captureResult.outputBuffers) { - if (resBuffer.status != BufferStatus::OK) { - notifyMsg.push_back(createErrorNotifyMsg( - request.frameNumber, resBuffer.streamId, ErrorCode::ERROR_BUFFER)); - } - } - - auto status = cameraCallback->notify(notifyMsg); - if (!status.isOk()) { - ALOGE("%s: notify call failed: %s", __func__, - status.getDescription().c_str()); - return cameraStatus(Status::INTERNAL_ERROR); - } - - std::vector<::aidl::android::hardware::camera::device::CaptureResult> - captureResults(1); - captureResults[0] = std::move(captureResult); - - status = cameraCallback->processCaptureResult(captureResults); - if (!status.isOk()) { - ALOGE("%s: processCaptureResult call failed: %s", __func__, - status.getDescription().c_str()); - return cameraStatus(Status::INTERNAL_ERROR); - } - - ALOGD("%s: Successfully called processCaptureResult", __func__); - - return ndk::ScopedAStatus::ok(); -} - -ndk::ScopedAStatus VirtualCameraSession::renderIntoBlobStreamBuffer( - const ::aidl::android::hardware::camera::device::CaptureRequest& request, - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer, - const size_t bufferSize) { - ALOGV("%s", __func__); - (void)request; - sp fence = importFence(streamBuffer.acquireFence); - - sp gBuffer = mEglSurfaceTexture->getCurrentBuffer(); - std::shared_ptr hwBuffer = fetchHardwareBuffer(streamBuffer); - - AHardwareBuffer_Planes planes_info; - int result = AHardwareBuffer_lockPlanes(hwBuffer.get(), - AHARDWAREBUFFER_USAGE_CPU_READ_RARELY, - fence->get(), nullptr, &planes_info); - if (result != OK) { - ALOGE("%s: Failed to lock planes for BLOB buffer: %d", __func__, result); + if (!mSessionContext.importBuffersFromCaptureRequest(request)) { + ALOGE("Failed to import buffers from capture request."); return cameraStatus(Status::INTERNAL_ERROR); } - android_ycbcr ycbcr; - status_t status = - gBuffer->lockYCbCr(AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, &ycbcr); - if (status != NO_ERROR) { - AHardwareBuffer_unlock(hwBuffer.get(), nullptr); - ALOGE("%s: Failed to lock graphic buffer: %d", __func__, status); - return cameraStatus(Status::INTERNAL_ERROR); + std::vector taskBuffers; + taskBuffers.reserve(request.outputBuffers.size()); + for (const StreamBuffer& streamBuffer : request.outputBuffers) { + taskBuffers.emplace_back(streamBuffer.streamId, streamBuffer.bufferId, + importFence(streamBuffer.acquireFence)); } - bool success = compressJpeg(gBuffer->getWidth(), gBuffer->getHeight(), ycbcr, - bufferSize, planes_info.planes[0].data); - - gBuffer->unlock(); - AHardwareBuffer_unlock(hwBuffer.get(), nullptr); - return success ? ndk::ScopedAStatus::ok() - : cameraStatus(Status::INTERNAL_ERROR); -} - -ndk::ScopedAStatus VirtualCameraSession::renderIntoImageStreamBuffer( - const ::aidl::android::hardware::camera::device::CaptureRequest& request, - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer) { - ALOGV("%s", __func__); - sp fence = importFence(streamBuffer.acquireFence); - if (kUseEGL && mEglDisplayContext->isInitialized()) { - // Wait for fence to clear. - if (fence->isValid()) { - status_t ret = fence->wait(kAcquireFenceTimeout.count()); - if (ret != 0) { - ALOGE("Timeout while waiting for the acquire fence for buffer%" PRId64 - " for streamId %d", - streamBuffer.bufferId, streamBuffer.streamId); - return cameraStatus(Status::INTERNAL_ERROR); - } - } - - // Render test pattern using EGL. - std::shared_ptr framebuffer = - fetchEglFramebuffer(mEglDisplayContext->getEglDisplay(), streamBuffer); - if (framebuffer == nullptr) { - ALOGE( - "%s: Failed to get EGL framebuffer corresponding to buffer id " - "%" PRId64 " for streamId %d", - __func__, streamBuffer.bufferId, streamBuffer.streamId); - return cameraStatus(Status::ILLEGAL_ARGUMENT); - } - - if (kRenderThroughSurfaceTexture && - mVirtualCameraClientCallback == nullptr) { - // Since we don't have client API yet to pass Surface to, let's just - // render something to the Surface ourselves. - renderTestPatternYCbCr420(mEglSurfaceTexture->getSurface(), - request.frameNumber); - } - - mEglDisplayContext->makeCurrent(); - framebuffer->beforeDraw(); - - if (kRenderThroughSurfaceTexture) { - mEglTextureProgram->draw(mEglSurfaceTexture->updateTexture()); - } else { - mEglTestPatternProgram->draw(framebuffer->getWidth(), - framebuffer->getHeight(), - request.frameNumber); - } - framebuffer->afterDraw(); - } else { - // Render test pattern on CPU. - std::shared_ptr hwBuffer = - fetchHardwareBuffer(streamBuffer); - if (hwBuffer == nullptr) { - ALOGE("%s: Failed to get hardware buffer id %" PRId64 " for streamId %d", - __func__, streamBuffer.bufferId, streamBuffer.streamId); - return cameraStatus(Status::ILLEGAL_ARGUMENT); - } - renderTestPatternYCbCr420(hwBuffer, request.frameNumber, fence->get()); + { + std::lock_guard lock(mLock); + mRenderThread->enqueueTask(std::make_unique( + request.frameNumber, taskBuffers)); } - return ndk::ScopedAStatus::ok(); } diff --git a/devices/VirtualCamera/VirtualCameraSession.h b/devices/VirtualCamera/VirtualCameraSession.h index 09a1661..61b36d5 100644 --- a/devices/VirtualCamera/VirtualCameraSession.h +++ b/devices/VirtualCamera/VirtualCameraSession.h @@ -21,6 +21,8 @@ #include #include +#include "VirtualCameraRenderThread.h" +#include "VirtualCameraSessionContext.h" #include "VirtualCameraStream.h" #include "aidl/android/companion/virtualcamera/IVirtualCameraCallback.h" #include "aidl/android/hardware/camera/common/Status.h" @@ -119,39 +121,9 @@ class VirtualCameraSession std::set getStreamIds() const EXCLUDES(mLock); private: - void removeBufferCaches( - const std::vector<::aidl::android::hardware::camera::device::BufferCache>& - cachesToRemove) EXCLUDES(mLock); - - void removeStreamsNotInStreamConfiguration( - const ::aidl::android::hardware::camera::device::StreamConfiguration& - streamConfiguration) EXCLUDES(mLock); - - std::optional<::aidl::android::hardware::camera::device::Stream> - getStreamConfig(const ::aidl::android::hardware::camera::device::StreamBuffer& - streamBuffer) const EXCLUDES(mLock); - - std::shared_ptr fetchHardwareBuffer( - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer) - EXCLUDES(mLock); - - std::shared_ptr fetchEglFramebuffer( - const EGLDisplay eglDisplay, - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer) - EXCLUDES(mLock); - ndk::ScopedAStatus processCaptureRequest( const ::aidl::android::hardware::camera::device::CaptureRequest& request); - ndk::ScopedAStatus renderIntoBlobStreamBuffer( - const ::aidl::android::hardware::camera::device::CaptureRequest& request, - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer, - size_t bufferSize); - - ndk::ScopedAStatus renderIntoImageStreamBuffer( - const ::aidl::android::hardware::camera::device::CaptureRequest& request, - const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer); - const std::string mCameraId; mutable std::mutex mLock; @@ -163,8 +135,7 @@ class VirtualCameraSession ::aidl::android::companion::virtualcamera::IVirtualCameraCallback> mVirtualCameraClientCallback; - // Mapping stream id -> VirtualCameraStream. - std::map mStreams GUARDED_BY(mLock); + VirtualCameraSessionContext mSessionContext; using RequestMetadataQueue = AidlMessageQueue< int8_t, ::aidl::android::hardware::common::fmq::SynchronizedReadWrite>; @@ -176,11 +147,7 @@ class VirtualCameraSession std::atomic_bool mFirstRequest{true}; - std::unique_ptr mEglDisplayContext; - std::unique_ptr mEglTestPatternProgram; - // Shader program to render external texture pattern. - std::unique_ptr mEglTextureProgram; - std::unique_ptr mEglSurfaceTexture; + std::unique_ptr mRenderThread GUARDED_BY(mLock); }; } // namespace virtualcamera diff --git a/devices/VirtualCamera/VirtualCameraSessionContext.cc b/devices/VirtualCamera/VirtualCameraSessionContext.cc new file mode 100644 index 0000000..284ad05 --- /dev/null +++ b/devices/VirtualCamera/VirtualCameraSessionContext.cc @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VirtualCameraSessionContext.h" + +#include +#include +#include + +#include "VirtualCameraStream.h" +#include "aidl/android/hardware/camera/device/StreamConfiguration.h" + +namespace android { +namespace companion { +namespace virtualcamera { + +using ::aidl::android::hardware::camera::device::BufferCache; +using ::aidl::android::hardware::camera::device::Stream; +using ::aidl::android::hardware::camera::device::StreamBuffer; +using ::aidl::android::hardware::camera::device::StreamConfiguration; + +bool VirtualCameraSessionContext::initializeStream( + const ::aidl::android::hardware::camera::device::Stream& stream) { + std::lock_guard lock(mLock); + + auto s = std::make_unique(stream); + + const auto& [_, newlyInserted] = mStreams.emplace( + std::piecewise_construct, std::forward_as_tuple(stream.id), + std::forward_as_tuple(std::move(s))); + return newlyInserted; +} + +void VirtualCameraSessionContext::closeAllStreams() { + std::lock_guard lock(mLock); + mStreams.clear(); +} + +bool VirtualCameraSessionContext::importBuffersFromCaptureRequest( + const ::aidl::android::hardware::camera::device::CaptureRequest& + captureRequest) { + std::lock_guard lock(mLock); + + for (const StreamBuffer& buffer : captureRequest.outputBuffers) { + auto it = mStreams.find(buffer.streamId); + if (it == mStreams.end()) { + ALOGE("%s: Cannot import buffer for unknown stream with id %d", __func__, + buffer.streamId); + return false; + } + VirtualCameraStream& stream = *it->second; + if (stream.getHardwareBuffer(buffer.bufferId) != nullptr) { + // This buffer is already imported. + continue; + } + + if (stream.importBuffer(buffer) == nullptr) { + ALOGE("%s: Failed to import buffer %" PRId64 " for streamId %d", __func__, + buffer.bufferId, buffer.streamId); + return false; + } + } + + return true; +} + +void VirtualCameraSessionContext::removeBufferCaches( + const std::vector& cachesToRemove) { + std::lock_guard lock(mLock); + for (const auto& bufferCache : cachesToRemove) { + auto it = mStreams.find(bufferCache.streamId); + if (it == mStreams.end()) { + ALOGE("%s: Ask to remove buffer %" PRId64 " from unknown stream %d", + __func__, bufferCache.bufferId, bufferCache.streamId); + continue; + } + if (it->second->removeBuffer(bufferCache.bufferId)) { + ALOGD("%s: Successfully removed buffer %" PRId64 + " from cache of stream %d", + __func__, bufferCache.bufferId, bufferCache.streamId); + } else { + ALOGE("%s: Failed to remove buffer %" PRId64 " from cache of stream %d", + __func__, bufferCache.bufferId, bufferCache.streamId); + } + } +} + +void VirtualCameraSessionContext::removeStreamsNotInStreamConfiguration( + const StreamConfiguration& streamConfiguration) { + std::unordered_set newConfigurationStreamIds; + for (const Stream& stream : streamConfiguration.streams) { + newConfigurationStreamIds.insert(stream.id); + } + + std::lock_guard lock(mLock); + for (auto it = mStreams.begin(); it != mStreams.end();) { + if (newConfigurationStreamIds.find(it->first) == + newConfigurationStreamIds.end()) { + ALOGV( + "Disposing of stream %d, since it is not referenced by new " + "configuration.", + it->first); + it = mStreams.erase(it); + } else { + ++it; + } + } +} + +std::optional VirtualCameraSessionContext::getStreamConfig( + int streamId) const { + std::lock_guard lock(mLock); + auto it = mStreams.find(streamId); + if (it == mStreams.end()) { + ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, + streamId); + return std::optional(); + } + return {it->second->getStreamConfig()}; +} + +std::shared_ptr VirtualCameraSessionContext::fetchHardwareBuffer( + const int streamId, const int bufferId) const { + std::lock_guard lock(mLock); + auto it = mStreams.find(streamId); + if (it == mStreams.end()) { + ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, + streamId); + return nullptr; + } + return it->second->getHardwareBuffer(bufferId); +} + +std::shared_ptr +VirtualCameraSessionContext::fetchOrCreateEglFramebuffer( + const EGLDisplay eglDisplay, const int streamId, const int bufferId) { + std::lock_guard lock(mLock); + auto it = mStreams.find(streamId); + if (it == mStreams.end()) { + ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__, + streamId); + return nullptr; + } + return it->second->getEglFrameBuffer(eglDisplay, bufferId); +} + +std::set VirtualCameraSessionContext::getStreamIds() const { + std::set result; + std::lock_guard lock(mLock); + for (const auto& [streamId, _] : mStreams) { + result.insert(streamId); + } + return result; +} + +} // namespace virtualcamera +} // namespace companion +} // namespace android diff --git a/devices/VirtualCamera/VirtualCameraSessionContext.h b/devices/VirtualCamera/VirtualCameraSessionContext.h new file mode 100644 index 0000000..e5cf5f4 --- /dev/null +++ b/devices/VirtualCamera/VirtualCameraSessionContext.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H +#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H + +#include +#include +#include +#include + +#include "VirtualCameraStream.h" +#include "aidl/android/hardware/camera/device/BufferCache.h" +#include "aidl/android/hardware/camera/device/CaptureRequest.h" +#include "aidl/android/hardware/camera/device/Stream.h" +#include "aidl/android/hardware/camera/device/StreamConfiguration.h" + +namespace android { +namespace companion { +namespace virtualcamera { + +// Encapsulates set of streams belonging to the same camera session. +class VirtualCameraSessionContext { + public: + // (Re)initialize the stream. + // + // Returns true if the stream is initialized for the first time. + bool initializeStream( + const ::aidl::android::hardware::camera::device::Stream& stream) + EXCLUDES(mLock); + + // Close all streams and free all asociated buffers. + void closeAllStreams() EXCLUDES(mLock); + + // Remove no longer needed buffers. + void removeBufferCaches( + const std::vector<::aidl::android::hardware::camera::device::BufferCache>& + cachesToRemove) EXCLUDES(mLock); + + // Remove all streams not referenced by provided configuration. + void removeStreamsNotInStreamConfiguration( + const ::aidl::android::hardware::camera::device::StreamConfiguration& + streamConfiguration) EXCLUDES(mLock); + + // Importored all not-yet imported buffers referenced by the capture request. + bool importBuffersFromCaptureRequest( + const ::aidl::android::hardware::camera::device::CaptureRequest& + captureRequest) EXCLUDES(mLock); + + // Get stream configuration for provided stream id. + // Returns nullopt in case there's no stream with provided stream id. + std::optional<::aidl::android::hardware::camera::device::Stream> + getStreamConfig(int streamId) const EXCLUDES(mLock); + + // Get hardware buffer for provided streamId & bufferId. + // Returns nullptr in case there's no such buffer. + std::shared_ptr fetchHardwareBuffer(int streamId, + int bufferId) const + EXCLUDES(mLock); + + // Get EGL framebuffer for provided EGL display, streamId & buffer id. + // + // This will also lazily create EglFrameBuffer for the provided EGLDisplay + // connection and will cache it (subsequent calls for same EGLDisplay and + // buffer will return same instance of EglFrameBuffer). + // + // Returns nullptr in case there's no such buffer or it was not possible + // to create EglFrameBuffer. + std::shared_ptr fetchOrCreateEglFramebuffer( + const EGLDisplay eglDisplay, int streamId, int bufferId) EXCLUDES(mLock); + + // Returns set of all stream ids managed by this instance. + std::set getStreamIds() const EXCLUDES(mLock); + + private: + mutable std::mutex mLock; + // streamId -> VirtualCameraStream mapping. + std::map> mStreams GUARDED_BY(mLock); +}; + +} // namespace virtualcamera +} // namespace companion +} // namespace android + +#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H diff --git a/devices/VirtualCamera/VirtualCameraStream.cc b/devices/VirtualCamera/VirtualCameraStream.cc index 1537479..03da171 100644 --- a/devices/VirtualCamera/VirtualCameraStream.cc +++ b/devices/VirtualCamera/VirtualCameraStream.cc @@ -83,8 +83,8 @@ sp createYCbCr420GraphicBuffer(GraphicBufferMapper& mapper, width); } -std::shared_ptr importBuffer(const NativeHandle& aidlHandle, - const Stream& streamConfig) { +std::shared_ptr importBufferInternal( + const NativeHandle& aidlHandle, const Stream& streamConfig) { if (aidlHandle.fds.empty()) { ALOGE("Empty handle - nothing to import"); return nullptr; @@ -126,28 +126,27 @@ VirtualCameraStream::VirtualCameraStream(const Stream& stream) : mStreamConfig(stream) { } -std::shared_ptr VirtualCameraStream::getHardwareBuffer( - const StreamBuffer& buffer) { - if (buffer.streamId != mStreamConfig.id) { - ALOGE("%s: Caller requesting buffer belonging to stream %d from stream %d", - __func__, buffer.streamId, mStreamConfig.id); - return nullptr; +std::shared_ptr VirtualCameraStream::importBuffer( + const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) { + auto hwBufferPtr = importBufferInternal(buffer.buffer, mStreamConfig); + if (hwBufferPtr != nullptr) { + std::lock_guard lock(mLock); + mBuffers.emplace(std::piecewise_construct, + std::forward_as_tuple(buffer.bufferId), + std::forward_as_tuple(hwBufferPtr)); } + return hwBufferPtr; +} +std::shared_ptr VirtualCameraStream::getHardwareBuffer( + const int bufferId) { std::lock_guard lock(mLock); - return getHardwareBufferLocked(buffer); + return getHardwareBufferLocked(bufferId); } std::shared_ptr VirtualCameraStream::getEglFrameBuffer( - const EGLDisplay eglDisplay, - const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) { - if (buffer.streamId != mStreamConfig.id) { - ALOGE("%s: Caller requesting buffer belonging to stream %d from stream %d", - __func__, buffer.streamId, mStreamConfig.id); - return nullptr; - } - - const FramebufferMapKey key(buffer.bufferId, eglDisplay); + const EGLDisplay eglDisplay, const int bufferId) { + const FramebufferMapKey key(bufferId, eglDisplay); std::lock_guard lock(mLock); @@ -156,7 +155,8 @@ std::shared_ptr VirtualCameraStream::getEglFrameBuffer( return it->second; } - std::shared_ptr hwBufferPtr = getHardwareBufferLocked(buffer); + std::shared_ptr hwBufferPtr = + getHardwareBufferLocked(bufferId); if (hwBufferPtr == nullptr) { return nullptr; } @@ -169,22 +169,9 @@ std::shared_ptr VirtualCameraStream::getEglFrameBuffer( } std::shared_ptr VirtualCameraStream::getHardwareBufferLocked( - const StreamBuffer& buffer) { - auto it = mBuffers.find(buffer.bufferId); - if (it != mBuffers.end()) { - return it->second; - } - - ALOGV("%s: Importing buffer %" PRId64 " for stream %d", __func__, - buffer.bufferId, buffer.streamId); - - auto hwBufferPtr = importBuffer(buffer.buffer, mStreamConfig); - if (hwBufferPtr != nullptr) { - mBuffers.emplace(std::piecewise_construct, - std::forward_as_tuple(buffer.bufferId), - std::forward_as_tuple(hwBufferPtr)); - } - return hwBufferPtr; + const int bufferId) { + auto it = mBuffers.find(bufferId); + return it != mBuffers.end() ? it->second : nullptr; } bool VirtualCameraStream::removeBuffer(int bufferId) { diff --git a/devices/VirtualCamera/VirtualCameraStream.h b/devices/VirtualCamera/VirtualCameraStream.h index 2543f4d..d76d05c 100644 --- a/devices/VirtualCamera/VirtualCameraStream.h +++ b/devices/VirtualCamera/VirtualCameraStream.h @@ -41,18 +41,19 @@ class VirtualCameraStream { VirtualCameraStream( const ::aidl::android::hardware::camera::device::Stream& stream); + std::shared_ptr importBuffer( + const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer); + // Get AHardwareBuffer instance corresponding to StreamBuffer from camera AIDL. // In case this is the first occurrence of the buffer, this will perform mapping // and stores hardware buffer in cache for further use. // // Returns nullptr in case buffer cannot be mapped or retrieved from the cache. - std::shared_ptr getHardwareBuffer( - const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) + std::shared_ptr getHardwareBuffer(int bufferId) EXCLUDES(mLock); - std::shared_ptr getEglFrameBuffer( - const EGLDisplay eglDisplay, - const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) + std::shared_ptr getEglFrameBuffer(const EGLDisplay eglDisplay, + int bufferId) EXCLUDES(mLock); // Un-maps the previously mapped buffer and removes it from the stream cache. @@ -63,8 +64,7 @@ class VirtualCameraStream { ::aidl::android::hardware::camera::device::Stream getStreamConfig() const; private: - std::shared_ptr getHardwareBufferLocked( - const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) + std::shared_ptr getHardwareBufferLocked(int bufferId) REQUIRES(mLock); const ::aidl::android::hardware::camera::device::Stream mStreamConfig; -- cgit v1.2.3