diff options
| author | Chih-Yu Huang <akahuang@google.com> | 2021-10-20 07:05:20 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-10-20 07:05:20 +0000 |
| commit | 1e53b54acd0e2ca0e7310b694e14f78d3d193474 (patch) | |
| tree | 4f2de25bd0e0390a69b84380b44212787708a678 | |
| parent | ccfbdec4ee8cdc36a712fd24fd76a03f1c2911f0 (diff) | |
| parent | 783186680c63be066f1822982c77a732b4514333 (diff) | |
| download | v4l2_codec2-1e53b54acd0e2ca0e7310b694e14f78d3d193474.tar.gz | |
Merge latest version to Android master. am: 783186680c
Original change: https://android-review.googlesource.com/c/platform/external/v4l2_codec2/+/1864513
Change-Id: Icd23c5667e1e9a104c19dcca2d2720a47127bcd3
25 files changed, 1004 insertions, 304 deletions
@@ -1,17 +1,366 @@ -# V4L2-based Codec2 Component Implementation +## General Information -## Description of Sub-folders +### Scope of this document -* accel/ -Core V4L2 API and codec utilities, ported from Chromium project. +This document is aimed to provide information about the v4l2\_codec2 project. +The target readers of this document are the developers and following maintainers +of this project, and the partners who are willing to use the V4L2 components. -* common/ -Common helper classes for components. +### Introduction -* components/ -The C2Component implementations based on V4L2 API, and the implementation of -C2ComponentStore for creating all the C2Components. +v4l2\_codec2 project provides a component implementation of Codec2 framework, +the next-generation codec framework. The component implementation delegates the +request to the driver via the V4L2 API. -* service/ -The Codec2's V4L2 IComponentStore service. The service initiates the component -store implemented at store/ folder, and registers it as the default service. +## Quick Start Guide + +### Prerequisites + +* Gralloc support for graphic buffer allocation +* ION or Gralloc support for linear buffer allocation +* DRM for buffer identification +* [V4L2 stateful decoding API](https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-decoder.html) +* [V4L2 encoding API](https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-encoder.html) +* Widevine DRM for secure playback + +### Enable V4L2 Components + +Install the build package and files, and set the parameters in device.mk + +```makefile +# Add the folder to the namespace. +PRODUCT_SOONG_NAMESPACES += external/v4l2_codec2 + +# Add the build target. +PRODUCT_PACKAGES += \ + android.hardware.media.c2@1.0-service-v4l2 \ + libc2plugin_store + +# If a customized allocator is needed, then add this package. +# See more detail at "Customized allocator" section. +PRODUCT_PACKAGES += \ + libv4l2_codec2_vendor_allocator + +# Install media_codecs_c2.xml. +# The destination is: /vendor/etc/media_codecs_c2.xml +PRODUCT_COPY_FILES += \ + <path_to_file>:$(TARGET_COPY_OUT_VENDOR)/etc/media_codecs_c2.xml + +# Set the customized property of v4l2_codec2, including: +# - The DRM device name and path pattern. +# - The maximum concurrent instances for decoder/encoder. +# It should be the same as "concurrent-instances" at media_codec_c2.xml. +PRODUCT_PROPERTY_OVERRIDES += \ + ro.vendor.v4l2_codec2.drm_device_name=virtio_gpu \ + ro.vendor.v4l2_codec2.drm_device_path=/dev/dri/renderD* \ + ro.vendor.v4l2_codec2.decode_concurrent_instances=8 \ + ro.vendor.v4l2_codec2.encode_concurrent_instances=8 + +# Codec2.0 poolMask: +# ION(16) +# BUFFERQUEUE(18) +# BLOB(19) +# V4L2_BUFFERQUEUE(20) +# V4L2_BUFFERPOOL(21) +# SECURE_LINEAR(22) +# SECURE_GRAPHIC(23) +# +# For linear buffer allocation: +# If ION is chosen, then the mask should be 0xf50000 +# If BLOB is chosen, then the mask should be 0xfc0000 +PRODUCT_PROPERTY_OVERRIDES += \ + debug.stagefright.c2-poolmask=0xf50000 + +# Install extended policy for codec2. +# The destination is: /vendor/etc/seccomp_policy/codec2.vendor.ext.policy +PRODUCT_COPY_FILES += \ + <path_to_policy>:$(TARGET_COPY_OUT_VENDOR)/etc/seccomp_policy/codec2.vendor.ext.policy +``` + +Enable codec2 hidl in manifest.xml + +```xml +<manifest version="1.0" type="device"> + <hal format="hidl"> + <name>android.hardware.media.c2</name> + <transport>hwbinder</transport> + <version>1.0</version> + <interface> + <name>IComponentStore</name> + <instance>default</instance> + </interface> + <interface> + <name>IConfigurable</name> + <instance>default</instance> + </interface> + </hal> +</manifest> +``` + +Add decode and encode components in media\_codecs\_c2.xml + +```xml +<?xml version="1.0" encoding="utf-8" ?> +<MediaCodecs> + <Encoders> + <MediaCodec name="c2.v4l2.avc.encoder" type="video/avc"> + <Limit name="size" min="32x32" max="1920x1088" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" range="1-244800" /> + <Limit name="bitrate" range="1-12000000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-1280x720" range="30-30" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp8.encoder" type="video/x-vnd.on2.vp8"> + <Limit name="size" min="32x32" max="1920x1088" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" range="1-244800" /> + <Limit name="bitrate" range="1-12000000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-1280x720" range="30-30" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp9.encoder" type="video/x-vnd.on2.vp9"> + <Limit name="size" min="32x32" max="1920x1088" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" range="1-244800" /> + <Limit name="bitrate" range="1-12000000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-1280x720" range="30-30" /> + </MediaCodec> + </Encoders> + + <Decoders> + <MediaCodec name="c2.v4l2.avc.decoder" type="video/avc" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="1879200" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp8.decoder" type="video/x-vnd.on2.vp8" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="1984500" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp9.decoder" type="video/x-vnd.on2.vp9" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="2073600" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.avc.decoder.secure" type="video/avc" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="1879200" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + <Feature name="secure-playback" required="true" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp8.decoder.secure" type="video/x-vnd.on2.vp8" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="1984500" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + <Feature name="secure-playback" required="true" /> + </MediaCodec> + + <MediaCodec name="c2.v4l2.vp9.decoder.secure" type="video/x-vnd.on2.vp9" > + <Limit name="size" min="16x16" max="4096x4096" /> + <Limit name="alignment" value="2x2" /> + <Limit name="block-size" value="16x16" /> + <Limit name="blocks-per-second" min="1" max="2073600" /> + <Limit name="bitrate" range="1-62500000" /> + <Limit name="concurrent-instances" max="8" /> + <Limit name="performance-point-3840x2160" range="30-30" /> + <Feature name="adaptive-playback" /> + <Feature name="secure-playback" required="true" /> + </MediaCodec> + </Decoders> +</MediaCodecs> +``` + +Set SELinux file policy in sepolicy/file\_contexts + +``` +/vendor/bin/hw/android\.hardware\.media\.c2@1\.0-service-v4l2(.*)? u:object_r:mediacodec_exec:s0 +``` + +Add additional permission in codec2.vendor.ext.policy + +``` +_llseek: 1 +epoll_create1: 1 +epoll_ctl: 1 +epoll_pwait: 1 +eventfd2: 1 +fstat64: 1 +fstatat64: 1 +fstatfs64: 1 +getcwd: 1 +getdents64: 1 +getuid32: 1 +mmap2: 1 +pselect6: 1 +statfs64: 1 +sysinfo: 1 +ugetrlimit: 1 +``` + +Set file permission in ueventd.rc + +``` +/dev/video* 0600 media media +``` + +### Customized Allocator + +The default allocator of the decoder's output buffer is C2AllocatorGralloc. +However, C2AllocatorGralloc might not fit the requirement for secure playback. +In this case, we can implement a customized C2Allocator, and load it at +run-time. Please create a library `libv4l2_codec2_vendor_allocator`, and export +a function `CreateVendorAllocator() `for creating the customized C2Allocator. +Here is an example below. + +#### Example of Android.bp + +```json +cc_library_shared { + name: "libv4l2_codec2_vendor_allocator", + vendor: true, + + defaults: [ + "libcodec2-impl-defaults", + ], + + srcs: [ + "C2VendorAllocatorFactory.cpp", + ], + + shared_libs: [ + "libc2plugin_store", + ], +} +``` + +#### Example of C2VendorAllocatorFactory.cpp + +```cpp +//#define LOG_NDEBUG 0 +#define LOG_TAG "C2VendorAllocatorFactory" + +#include <C2AllocatorGralloc.h> +#include <utils/Log.h> +#include <v4l2_codec2/plugin_store/V4L2AllocatorId.h> + +namespace android { + +::C2Allocator* CreateVendorAllocator(::C2Allocator::id_t allocatorId) { + ALOGV("%s(%d)", __func__, allocatorId); + + // Change to create vendor-customized implementation. + switch (allocatorId) { + case V4L2AllocatorId::V4L2_BUFFERQUEUE: + return new C2AllocatorGralloc(V4L2AllocatorId::V4L2_BUFFERQUEUE, true); + case V4L2AllocatorId::V4L2_BUFFERPOOL: + return new C2AllocatorGralloc(V4L2AllocatorId::V4L2_BUFFERPOOL, true); + case V4L2AllocatorId::SECURE_LINEAR: + return new C2AllocatorGralloc(V4L2AllocatorId::SECURE_LINEAR, true); + case V4L2AllocatorId::SECURE_GRAPHIC: + return new C2AllocatorGralloc(V4L2AllocatorId::SECURE_GRAPHIC, true); + default: + ALOGE("%s(): Unknown allocator ID: %d", __func__, allocatorId); + } + return nullptr; +} + +} // namespace android + +extern "C" ::C2Allocator* CreateAllocator(::C2Allocator::id_t allocatorId) { + return ::android::CreateVendorAllocator(allocatorId); +} +``` + +## V4L2 Encoder + +### Supported Codecs + +Currently the V4L2 encoder has support for the H.264, VP8 and VP9 codecs. Codec +selection can be done by selecting the encoder with the appropriate name. + +- H26: *c2.v4l2.avc.encoder* +- VP8: *c2.v4l2.vp8.encoder* +- VP9: *c2.v4l2.vp9.encoder* + +### Supported Parameters: + +The following parameters are static and can not be changed at run-time: + +- *C2_PARAMKEY_PICTURE_SIZE*: This parameter can be used to configure the +resolution of the encoded video stream. +- *C2_PARAMKEY_PROFILE_LEVEL*: This parameter can be used to adjust the desired +profile level. When using the H.264 codec the profile level might be adjusted to +conform to the minimum requirements for the specified bitrate and framerate. +- *C2_PARAMKEY_SYNC_FRAME_INTERVAL*: This parameter can be used to configure the +desired time between subsequent key frames in microseconds. +- *C2_PARAMKEY_BITRATE_MODE*: This parameter can be used to switch between +constant (*C2Config::BITRATE_CONST*) and variable (*C2Config::BITRATE_VARIABLE*) +bitrate modes. When using CBR the encoder will try to maintain a constant +bitrate. This mode is preferable for video conferencing where maintaining a +stable bitrate is more important than quality. When using VBR the encoder will +be allowed to dynamically adjust the bitrate to maintain a constant quality. As +the mediacodec framework does not provide facilities to configure the peak +bitrate when using VBR, it is currently always set to 2x the target bitrate. + +The following parameters are dynamic, and can be freely adjusted at run-time: + +- *C2_PARAMKEY_BITRATE*: Use this parameter to specify the desired bitrate. +- *C2_PARAMKEY_FRAME_RATE*: This parameter can be used to configure the desired +framerate. Note that the encoder will automatically try to adjust the framerate +if the timestamps on the input video frames don't match the configured +framerate. +- *C2_PARAMKEY_REQUEST_SYNC_FRAME*: This parameter can be used to request +additional key frames in addition to the periodic ones requested through the +*C2_PARAMKEY_SYNC_FRAME_INTERVAL* parameter. + +### Supported Input Pixel Formats: + +The V4L2 encoder supports various input pixel formats, however frames are +currently always passed to the V4L2 encoder in the NV12 format. If a video frame +using a different pixel format is passed to the encoder, format conversion will +be performed to convert the frame to the NV12 format. + +### Additional Features: + +To improve the resilience of H.264 video streams when data is missing, SPS and +PPS NAL units are prepended to IDR frames by enabling the +*V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR* control. If the V4L2 driver does not +support this control the encoder will manually cache and prepend SPS and PPS NAL +units. diff --git a/common/EncodeHelpers.cpp b/common/EncodeHelpers.cpp index 4575197..c07281f 100644 --- a/common/EncodeHelpers.cpp +++ b/common/EncodeHelpers.cpp @@ -18,6 +18,32 @@ namespace android { +namespace { + +// Android frameworks needs 4 bytes start code. +constexpr uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01}; +constexpr size_t kH264StartCodeSize = 4; + +// Copy an H.264 NAL unit with size |srcSize| (without a start code) into a buffer with size +// |dstSize|. An H.264 start code is prepended to the NAL unit. After copying |dst| is adjusted to +// point to the address immediately following the copied data, and the |dstSize| is updated to +// reflect the remaining destination buffer size. +bool copyNALUPrependingStartCode(const uint8_t* src, size_t srcSize, uint8_t** dst, + size_t* dstSize) { + size_t naluSize = srcSize + kH264StartCodeSize; + if (naluSize > *dstSize) { + ALOGE("Couldn't copy NAL unit, not enough space in destination buffer"); + return false; + } + memcpy(*dst, kH264StartCode, kH264StartCodeSize); + memcpy(*dst + kH264StartCodeSize, src, srcSize); + *dst += naluSize; + *dstSize -= naluSize; + return true; +} + +} // namespace + uint8_t c2LevelToV4L2Level(C2Config::level_t level) { switch (level) { case C2Config::LEVEL_AVC_1: @@ -84,41 +110,101 @@ android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block) { return ycbcr; } -void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, - size_t length) { - // Android frameworks needs 4 bytes start code. - constexpr uint8_t kStartCode[] = {0x00, 0x00, 0x00, 0x01}; - constexpr int kStartCodeLength = 4; +bool extractSPSPPS(const uint8_t* data, size_t length, std::vector<uint8_t>* sps, + std::vector<uint8_t>* pps) { + bool foundSPS = false; + bool foundPPS = false; + NalParser parser(data, length); + while (!(foundSPS && foundPPS) && parser.locateNextNal()) { + switch (parser.type()) { + case NalParser::kSPSType: + sps->resize(parser.length()); + memcpy(sps->data(), parser.data(), parser.length()); + foundSPS = true; + break; + case NalParser::kPPSType: + pps->resize(parser.length()); + memcpy(pps->data(), parser.data(), parser.length()); + foundPPS = true; + break; + } + } + return foundSPS && foundPPS; +} +bool extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, + size_t length) { csd->reset(); - // Temporarily allocate a byte array to copy codec config data. This should be freed after - // codec config data extraction is done. - auto tmpConfigData = std::make_unique<uint8_t[]>(length); - uint8_t* tmpOutput = tmpConfigData.get(); - uint8_t* tmpConfigDataEnd = tmpOutput + length; + std::vector<uint8_t> sps; + std::vector<uint8_t> pps; + if (!extractSPSPPS(data, length, &sps, &pps)) { + return false; + } - NalParser parser(data, length); + size_t configDataLength = sps.size() + pps.size() + (2u * kH264StartCodeSize); + ALOGV("Extracted codec config data: length=%zu", configDataLength); + + *csd = C2StreamInitDataInfo::output::AllocUnique(configDataLength, 0u); + uint8_t* csdBuffer = (*csd)->m.value; + return copyNALUPrependingStartCode(sps.data(), sps.size(), &csdBuffer, &configDataLength) && + copyNALUPrependingStartCode(pps.data(), pps.size(), &csdBuffer, &configDataLength); +} + +size_t prependSPSPPSToIDR(const uint8_t* src, size_t srcSize, uint8_t* dst, size_t dstSize, + std::vector<uint8_t>* sps, std::vector<uint8_t>* pps) { + bool foundStreamParams = false; + size_t remainingDstSize = dstSize; + NalParser parser(src, srcSize); while (parser.locateNextNal()) { - if (parser.length() == 0) continue; - uint8_t nalType = parser.type(); - ALOGV("find next NAL: type=%d, length=%zu", nalType, parser.length()); - if (nalType != NalParser::kSPSType && nalType != NalParser::kPPSType) continue; - - if (tmpOutput + kStartCodeLength + parser.length() > tmpConfigDataEnd) { - ALOGE("Buffer overflow on extracting codec config data (length=%zu)", length); - return; + switch (parser.type()) { + case NalParser::kSPSType: + // SPS found, copy to cache. + ALOGV("Found SPS (length %zu)", parser.length()); + sps->resize(parser.length()); + memcpy(sps->data(), parser.data(), parser.length()); + foundStreamParams = true; + break; + case NalParser::kPPSType: + // PPS found, copy to cache. + ALOGV("Found PPS (length %zu)", parser.length()); + pps->resize(parser.length()); + memcpy(pps->data(), parser.data(), parser.length()); + foundStreamParams = true; + break; + case NalParser::kIDRType: + ALOGV("Found IDR (length %zu)", parser.length()); + if (foundStreamParams) { + ALOGV("Not injecting SPS and PPS before IDR, already present"); + break; + } + + // Prepend the cached SPS and PPS to the IDR NAL unit. + if (sps->empty() || pps->empty()) { + ALOGE("No cached SPS or PPS NAL unit available to inject before IDR"); + return 0; + } + if (!copyNALUPrependingStartCode(sps->data(), sps->size(), &dst, &remainingDstSize)) { + ALOGE("Not enough space to inject SPS NAL unit before IDR"); + return 0; + } + if (!copyNALUPrependingStartCode(pps->data(), pps->size(), &dst, &remainingDstSize)) { + ALOGE("Not enough space to inject PPS NAL unit before IDR"); + return 0; + } + + ALOGV("Stream header injected before IDR"); + break; + } + + // Copy the NAL unit to the new output buffer. + if (!copyNALUPrependingStartCode(parser.data(), parser.length(), &dst, &remainingDstSize)) { + ALOGE("NAL unit does not fit in the provided output buffer"); + return 0; } - std::memcpy(tmpOutput, kStartCode, kStartCodeLength); - tmpOutput += kStartCodeLength; - std::memcpy(tmpOutput, parser.data(), parser.length()); - tmpOutput += parser.length(); } - size_t configDataLength = tmpOutput - tmpConfigData.get(); - ALOGV("Extracted codec config data: length=%zu", configDataLength); - *csd = C2StreamInitDataInfo::output::AllocUnique(configDataLength, 0u); - std::memcpy((*csd)->m.value, tmpConfigData.get(), configDataLength); + return dstSize - remainingDstSize; } } // namespace android diff --git a/common/Fourcc.cpp b/common/Fourcc.cpp index f7d3efd..11ea31e 100644 --- a/common/Fourcc.cpp +++ b/common/Fourcc.cpp @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +//#define LOG_NDEBUG 0 +#define LOG_TAG "Fourcc" + #include <v4l2_codec2/common/Fourcc.h> #include <linux/videodev2.h> @@ -36,7 +39,7 @@ std::optional<Fourcc> Fourcc::fromUint32(uint32_t fourcc) { case MM21: return Fourcc(static_cast<Value>(fourcc)); } - ALOGE("Unmapped fourcc: %s", fourccToString(fourcc).c_str()); + ALOGV("Unmapped fourcc: %s", fourccToString(fourcc).c_str()); return std::nullopt; } diff --git a/common/NalParser.cpp b/common/NalParser.cpp index 3216574..1df95d4 100644 --- a/common/NalParser.cpp +++ b/common/NalParser.cpp @@ -31,28 +31,35 @@ enum H264ProfileIDC { constexpr uint32_t kYUV444Idc = 3; // Read unsigned int encoded with exponential-golomb. -uint32_t parseUE(ABitReader* br) { +bool parseUE(ABitReader* br, uint32_t* val) { uint32_t numZeroes = 0; - while (br->getBits(1) == 0) { + uint32_t bit; + if (!br->getBitsGraceful(1, &bit)) return false; + while (bit == 0) { ++numZeroes; + if (!br->getBitsGraceful(1, &bit)) return false; } - uint32_t val = br->getBits(numZeroes); - return val + (1u << numZeroes) - 1; + if (!br->getBitsGraceful(numZeroes, val)) return false; + *val += (1u << numZeroes) - 1; + return true; } // Read signed int encoded with exponential-golomb. -int32_t parseSE(ABitReader* br) { - uint32_t codeNum = parseUE(br); - return (codeNum & 1) ? (codeNum + 1) >> 1 : -static_cast<int32_t>(codeNum >> 1); +bool parseSE(ABitReader* br, int32_t* val) { + uint32_t codeNum; + if (!parseUE(br, &codeNum)) return false; + *val = (codeNum & 1) ? (codeNum + 1) >> 1 : -static_cast<int32_t>(codeNum >> 1); + return true; } // Skip a H.264 sequence scaling list in the specified bitstream. -void skipScalingList(ABitReader* br, size_t scalingListSize) { +bool skipScalingList(ABitReader* br, size_t scalingListSize) { size_t nextScale = 8; size_t lastScale = 8; for (size_t j = 0; j < scalingListSize; ++j) { if (nextScale != 0) { - int32_t deltaScale = parseSE(br); // delta_sl + int32_t deltaScale; + if (!parseSE(br, &deltaScale)) return false; // delta_sl if (deltaScale < -128) { ALOGW("delta scale (%d) is below range, capping to -128", deltaScale); deltaScale = -128; @@ -64,19 +71,24 @@ void skipScalingList(ABitReader* br, size_t scalingListSize) { } lastScale = (nextScale == 0) ? lastScale : nextScale; } + return true; } // Skip the H.264 sequence scaling matrix in the specified bitstream. -void skipScalingMatrix(ABitReader* br, size_t numScalingLists) { +bool skipScalingMatrix(ABitReader* br, size_t numScalingLists) { for (size_t i = 0; i < numScalingLists; ++i) { - if (br->getBits(1)) { // seq_scaling_list_present_flag + uint32_t seq_scaling_list_present_flag; + if (!br->getBitsGraceful(1, &seq_scaling_list_present_flag)) + return false; // seq_scaling_list_present_flag + if (seq_scaling_list_present_flag) { if (i < 6) { - skipScalingList(br, 16); + if (!skipScalingList(br, 16)) return false; } else { - skipScalingList(br, 64); + if (!skipScalingList(br, 64)) return false; } } } + return true; } } // namespace @@ -136,82 +148,120 @@ bool NalParser::findCodedColorAspects(ColorAspects* colorAspects) { // Skip first byte containing type. ABitReader br(mCurrNalDataPos + 1, length() - 1); - uint32_t profileIDC = br.getBits(8); // profile_idc - br.skipBits(16); // constraint flags + reserved bits + level_idc - parseUE(&br); // seq_parameter_set_id + uint32_t unused; + uint32_t profileIDC; + if (!br.getBitsGraceful(8, &profileIDC)) return false; // profile_idc + br.skipBits(16); // constraint flags + reserved bits + level_idc + parseUE(&br, &unused); // seq_parameter_set_id if (profileIDC == kProfileIDCHigh || profileIDC == kProfileIDHigh10 || profileIDC == kProfileIDHigh422 || profileIDC == kProfileIDHigh444Predictive || profileIDC == kProfileIDCAVLC444 || profileIDC == kProfileIDScalableBaseline || profileIDC == kProfileIDScalableHigh || profileIDC == kProfileIDSMultiviewHigh || profileIDC == kProfileIDStereoHigh) { - uint32_t chromaFormatIDC = parseUE(&br); + uint32_t chromaFormatIDC; + if (!parseUE(&br, &chromaFormatIDC)) return false; if (chromaFormatIDC == kYUV444Idc) { // chroma_format_idc br.skipBits(1); // separate_colour_plane_flag } - parseUE(&br); // bit_depth_luma_minus8 - parseUE(&br); // bit_depth_chroma_minus8 - br.skipBits(1); // lossless_qpprime_y_zero_flag - if (br.getBits(1)) { // seq_scaling_matrix_present_flag + parseUE(&br, &unused); // bit_depth_luma_minus8 + parseUE(&br, &unused); // bit_depth_chroma_minus8 + br.skipBits(1); // lossless_qpprime_y_zero_flag + + uint32_t seqScalingMatrixPresentFlag; + if (!br.getBitsGraceful(1, &seqScalingMatrixPresentFlag)) + return false; // seq_scaling_matrix_present_flag + if (seqScalingMatrixPresentFlag) { const size_t numScalingLists = (chromaFormatIDC != kYUV444Idc) ? 8 : 12; - skipScalingMatrix(&br, numScalingLists); + if (!skipScalingMatrix(&br, numScalingLists)) return false; } } - parseUE(&br); // log2_max_frame_num_minus4 - uint32_t pictureOrderCountType = parseUE(&br); // pic_order_cnt_type + parseUE(&br, &unused); // log2_max_frame_num_minus4 + uint32_t pictureOrderCountType; + if (!parseUE(&br, &pictureOrderCountType)) return false; // pic_order_cnt_type if (pictureOrderCountType == 0) { - parseUE(&br); // log2_max_pic_order_cnt_lsb_minus4 + parseUE(&br, &unused); // log2_max_pic_order_cnt_lsb_minus4 } else if (pictureOrderCountType == 1) { - br.skipBits(1); // delta_pic_order_always_zero_flag - parseSE(&br); // offset_for_non_ref_pic - parseSE(&br); // offset_for_top_to_bottom_field - uint32_t numReferenceFrames = parseUE(&br); // num_ref_frames_in_pic_order_cnt_cycle + br.skipBits(1); // delta_pic_order_always_zero_flag + int32_t unused_i; + parseSE(&br, &unused_i); // offset_for_non_ref_pic + parseSE(&br, &unused_i); // offset_for_top_to_bottom_field + uint32_t numReferenceFrames; + if (!parseUE(&br, &numReferenceFrames)) + return false; // num_ref_frames_in_pic_order_cnt_cycle for (uint32_t i = 0; i < numReferenceFrames; ++i) { - parseUE(&br); // offset_for_ref_frame + parseUE(&br, &unused); // offset_for_ref_frame } } - parseUE(&br); // num_ref_frames - br.skipBits(1); // gaps_in_frame_num_value_allowed_flag - parseUE(&br); // pic_width_in_mbs_minus1 - parseUE(&br); // pic_height_in_map_units_minus1 - if (!br.getBits(1)) { // frame_mbs_only_flag - br.skipBits(1); // mb_adaptive_frame_field_flag + parseUE(&br, &unused); // num_ref_frames + br.skipBits(1); // gaps_in_frame_num_value_allowed_flag + parseUE(&br, &unused); // pic_width_in_mbs_minus1 + parseUE(&br, &unused); // pic_height_in_map_units_minus1 + uint32_t frameMbsOnlyFlag; + if (!br.getBitsGraceful(1, &frameMbsOnlyFlag)) return false; // frame_mbs_only_flag + if (!frameMbsOnlyFlag) { + br.skipBits(1); // mb_adaptive_frame_field_flag } br.skipBits(1); // direct_8x8_inference_flag - if (br.getBits(1)) { // frame_cropping_flag - parseUE(&br); // frame_cropping_rect_left_offset - parseUE(&br); // frame_cropping_rect_right_offset - parseUE(&br); // frame_cropping_rect_top_offset - parseUE(&br); // frame_cropping_rect_bottom_offset + uint32_t frameCroppingFlag; + if (!br.getBitsGraceful(1, &frameCroppingFlag)) return false; // frame_cropping_flag + if (frameCroppingFlag) { + parseUE(&br, &unused); // frame_cropping_rect_left_offset + parseUE(&br, &unused); // frame_cropping_rect_right_offset + parseUE(&br, &unused); // frame_cropping_rect_top_offset + parseUE(&br, &unused); // frame_cropping_rect_bottom_offset } - if (br.getBits(1)) { // vui_parameters_present_flag - if (br.getBits(1)) { // VUI aspect_ratio_info_present_flag - if (br.getBits(8) == 255) { // VUI aspect_ratio_idc == extended sample aspect ratio - br.skipBits(32); // VUI sar_width + sar_height + uint32_t vuiParametersPresentFlag; + if (!br.getBitsGraceful(1, &vuiParametersPresentFlag)) + return false; // vui_parameters_present_flag + if (vuiParametersPresentFlag) { + uint32_t aspectRatioInfoPresentFlag; + if (!br.getBitsGraceful(1, &aspectRatioInfoPresentFlag)) + return false; // VUI aspect_ratio_info_present_flag + if (aspectRatioInfoPresentFlag) { + uint32_t aspectRatioIdc; + if (!br.getBitsGraceful(8, &aspectRatioIdc)) return false; // VUI aspect_ratio_idc + if (aspectRatioIdc == 255) { // VUI aspect_ratio_idc == extended sample aspect ratio + br.skipBits(32); // VUI sar_width + sar_height } } - if (br.getBits(1)) { // VUI overscan_info_present_flag - br.skipBits(1); // VUI overscan_appropriate_flag + uint32_t overscanInfoPresentFlag; + if (!br.getBitsGraceful(1, &overscanInfoPresentFlag)) + return false; // VUI overscan_info_present_flag + if (overscanInfoPresentFlag) { + br.skipBits(1); // VUI overscan_appropriate_flag } - if (br.getBits(1)) { // VUI video_signal_type_present_flag - br.skipBits(3); // VUI video_format - colorAspects->fullRange = br.getBits(1); // VUI video_full_range_flag - if (br.getBits(1)) { // VUI color_description_present_flag - colorAspects->primaries = br.getBits(8); // VUI colour_primaries - colorAspects->transfer = br.getBits(8); // VUI transfer_characteristics - colorAspects->coeffs = br.getBits(8); // VUI matrix_coefficients - return !br.overRead(); + uint32_t videoSignalTypePresentFlag; + if (!br.getBitsGraceful(1, &videoSignalTypePresentFlag)) + return false; // VUI video_signal_type_present_flag + if (videoSignalTypePresentFlag) { + br.skipBits(3); // VUI video_format + uint32_t videoFullRangeFlag; + if (!br.getBitsGraceful(1, &videoFullRangeFlag)) + return false; // VUI videoFullRangeFlag + colorAspects->fullRange = videoFullRangeFlag; + uint32_t color_description_present_flag; + if (!br.getBitsGraceful(1, &color_description_present_flag)) + return false; // VUI color_description_present_flag + if (color_description_present_flag) { + if (!br.getBitsGraceful(8, &colorAspects->primaries)) + return false; // VUI colour_primaries + if (!br.getBitsGraceful(8, &colorAspects->transfer)) + return false; // VUI transfer_characteristics + if (!br.getBitsGraceful(8, &colorAspects->coeffs)) + return false; // VUI matrix_coefficients + return true; } } } - return false; + return false; // The NAL unit doesn't contain color aspects info. } } // namespace android diff --git a/common/V4L2Device.cpp b/common/V4L2Device.cpp index a31d82b..1efb4e3 100644 --- a/common/V4L2Device.cpp +++ b/common/V4L2Device.cpp @@ -313,7 +313,7 @@ private: V4L2BufferRefBase::V4L2BufferRefBase(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue) : mQueue(std::move(queue)), mReturnTo(mQueue->mFreeBuffers) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(V4L2_TYPE_IS_MULTIPLANAR(v4l2Buffer.type)); ALOG_ASSERT(v4l2Buffer.length <= base::size(mV4l2Planes)); ALOG_ASSERT(mReturnTo); @@ -331,7 +331,7 @@ V4L2BufferRefBase::~V4L2BufferRefBase() { } bool V4L2BufferRefBase::queueBuffer() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (!mQueue) return false; @@ -341,7 +341,7 @@ bool V4L2BufferRefBase::queueBuffer() { } void* V4L2BufferRefBase::getPlaneMapping(const size_t plane) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (!mQueue) return nullptr; @@ -349,7 +349,7 @@ void* V4L2BufferRefBase::getPlaneMapping(const size_t plane) { } bool V4L2BufferRefBase::checkNumFDsForFormat(const size_t numFds) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (!mQueue) return false; @@ -383,24 +383,24 @@ bool V4L2BufferRefBase::checkNumFDsForFormat(const size_t numFds) const { V4L2WritableBufferRef::V4L2WritableBufferRef(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue) : mBufferData(std::make_unique<V4L2BufferRefBase>(v4l2Buffer, std::move(queue))) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); } V4L2WritableBufferRef::V4L2WritableBufferRef(V4L2WritableBufferRef&& other) : mBufferData(std::move(other.mBufferData)) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); DCHECK_CALLED_ON_VALID_SEQUENCE(other.mSequenceChecker); } V4L2WritableBufferRef::~V4L2WritableBufferRef() { // Only valid references should be sequence-checked if (mBufferData) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); } } V4L2WritableBufferRef& V4L2WritableBufferRef::operator=(V4L2WritableBufferRef&& other) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); DCHECK_CALLED_ON_VALID_SEQUENCE(other.mSequenceChecker); if (this == &other) return *this; @@ -411,14 +411,14 @@ V4L2WritableBufferRef& V4L2WritableBufferRef::operator=(V4L2WritableBufferRef&& } enum v4l2_memory V4L2WritableBufferRef::memory() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return static_cast<enum v4l2_memory>(mBufferData->mV4l2Buffer.memory); } bool V4L2WritableBufferRef::doQueue() && { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); bool queued = mBufferData->queueBuffer(); @@ -430,7 +430,7 @@ bool V4L2WritableBufferRef::doQueue() && { } bool V4L2WritableBufferRef::queueMMap() && { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); // Move ourselves so our data gets freed no matter when we return @@ -445,7 +445,7 @@ bool V4L2WritableBufferRef::queueMMap() && { } bool V4L2WritableBufferRef::queueUserPtr(const std::vector<void*>& ptrs) && { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); // Move ourselves so our data gets freed no matter when we return @@ -471,7 +471,7 @@ bool V4L2WritableBufferRef::queueUserPtr(const std::vector<void*>& ptrs) && { } bool V4L2WritableBufferRef::queueDMABuf(const std::vector<int>& fds) && { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); // Move ourselves so our data gets freed no matter when we return @@ -491,14 +491,14 @@ bool V4L2WritableBufferRef::queueDMABuf(const std::vector<int>& fds) && { } size_t V4L2WritableBufferRef::planesCount() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.length; } size_t V4L2WritableBufferRef::getPlaneSize(const size_t plane) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -510,7 +510,7 @@ size_t V4L2WritableBufferRef::getPlaneSize(const size_t plane) const { } void V4L2WritableBufferRef::setPlaneSize(const size_t plane, const size_t size) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); enum v4l2_memory mem = memory(); @@ -529,28 +529,28 @@ void V4L2WritableBufferRef::setPlaneSize(const size_t plane, const size_t size) } void* V4L2WritableBufferRef::getPlaneMapping(const size_t plane) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->getPlaneMapping(plane); } void V4L2WritableBufferRef::setTimeStamp(const struct timeval& timestamp) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); mBufferData->mV4l2Buffer.timestamp = timestamp; } const struct timeval& V4L2WritableBufferRef::getTimeStamp() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.timestamp; } void V4L2WritableBufferRef::setPlaneBytesUsed(const size_t plane, const size_t bytesUsed) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -567,7 +567,7 @@ void V4L2WritableBufferRef::setPlaneBytesUsed(const size_t plane, const size_t b } size_t V4L2WritableBufferRef::getPlaneBytesUsed(const size_t plane) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -579,7 +579,7 @@ size_t V4L2WritableBufferRef::getPlaneBytesUsed(const size_t plane) const { } void V4L2WritableBufferRef::setPlaneDataOffset(const size_t plane, const size_t dataOffset) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -591,7 +591,7 @@ void V4L2WritableBufferRef::setPlaneDataOffset(const size_t plane, const size_t } size_t V4L2WritableBufferRef::bufferId() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.index; @@ -600,7 +600,7 @@ size_t V4L2WritableBufferRef::bufferId() const { V4L2ReadableBuffer::V4L2ReadableBuffer(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue) : mBufferData(std::make_unique<V4L2BufferRefBase>(v4l2Buffer, std::move(queue))) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); } V4L2ReadableBuffer::~V4L2ReadableBuffer() { @@ -611,42 +611,42 @@ V4L2ReadableBuffer::~V4L2ReadableBuffer() { } bool V4L2ReadableBuffer::isLast() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.flags & V4L2_BUF_FLAG_LAST; } bool V4L2ReadableBuffer::isKeyframe() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.flags & V4L2_BUF_FLAG_KEYFRAME; } struct timeval V4L2ReadableBuffer::getTimeStamp() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.timestamp; } size_t V4L2ReadableBuffer::planesCount() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.length; } const void* V4L2ReadableBuffer::getPlaneMapping(const size_t plane) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); DCHECK(mBufferData); return mBufferData->getPlaneMapping(plane); } size_t V4L2ReadableBuffer::getPlaneBytesUsed(const size_t plane) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -658,7 +658,7 @@ size_t V4L2ReadableBuffer::getPlaneBytesUsed(const size_t plane) const { } size_t V4L2ReadableBuffer::getPlaneDataOffset(const size_t plane) const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); if (plane >= planesCount()) { @@ -670,7 +670,7 @@ size_t V4L2ReadableBuffer::getPlaneDataOffset(const size_t plane) const { } size_t V4L2ReadableBuffer::bufferId() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(mBufferData); return mBufferData->mV4l2Buffer.index; @@ -698,11 +698,11 @@ public: V4L2Queue::V4L2Queue(scoped_refptr<V4L2Device> dev, enum v4l2_buf_type type, base::OnceClosure destroyCb) : mType(type), mDevice(dev), mDestroyCb(std::move(destroyCb)) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); } V4L2Queue::~V4L2Queue() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (mIsStreaming) { ALOGEQ("Queue is still streaming, trying to stop it..."); @@ -756,7 +756,7 @@ std::pair<std::optional<struct v4l2_format>, int> V4L2Queue::getFormat() { } size_t V4L2Queue::allocateBuffers(size_t count, enum v4l2_memory memory) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); ALOG_ASSERT(!mFreeBuffers); ALOG_ASSERT(mQueuedBuffers.size() == 0u); @@ -827,7 +827,7 @@ size_t V4L2Queue::allocateBuffers(size_t count, enum v4l2_memory memory) { } bool V4L2Queue::deallocateBuffers() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (isStreaming()) { ALOGEQ("Cannot deallocate buffers while streaming."); @@ -860,7 +860,7 @@ bool V4L2Queue::deallocateBuffers() { } size_t V4L2Queue::getMemoryUsage() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); size_t usage = 0; for (const auto& buf : mBuffers) { usage += buf->getMemoryUsage(); @@ -873,7 +873,7 @@ v4l2_memory V4L2Queue::getMemoryType() const { } std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); // No buffers allocated at the moment? if (!mFreeBuffers) return std::nullopt; @@ -886,7 +886,7 @@ std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer() { } std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer(size_t requestedBufferIid) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); // No buffers allocated at the moment? if (!mFreeBuffers) return std::nullopt; @@ -899,7 +899,7 @@ std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer(size_t requestedBu } bool V4L2Queue::queueBuffer(struct v4l2_buffer* v4l2Buffer) { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); int ret = mDevice->ioctl(VIDIOC_QBUF, v4l2Buffer); if (ret) { @@ -919,7 +919,7 @@ bool V4L2Queue::queueBuffer(struct v4l2_buffer* v4l2Buffer) { } std::pair<bool, V4L2ReadableBufferRef> V4L2Queue::dequeueBuffer() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); // No need to dequeue if no buffers queued. if (queuedBuffersCount() == 0) return std::make_pair(true, nullptr); @@ -968,13 +968,13 @@ std::pair<bool, V4L2ReadableBufferRef> V4L2Queue::dequeueBuffer() { } bool V4L2Queue::isStreaming() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); return mIsStreaming; } bool V4L2Queue::streamon() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); if (mIsStreaming) return true; @@ -991,7 +991,7 @@ bool V4L2Queue::streamon() { } bool V4L2Queue::streamoff() { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); // We do not check the value of IsStreaming(), because we may have queued buffers to the queue // and wish to get them back - in such as case, we may need to do a VIDIOC_STREAMOFF on a @@ -1017,19 +1017,19 @@ bool V4L2Queue::streamoff() { } size_t V4L2Queue::allocatedBuffersCount() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); return mBuffers.size(); } size_t V4L2Queue::freeBuffersCount() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); return mFreeBuffers ? mFreeBuffers->size() : 0; } size_t V4L2Queue::queuedBuffersCount() const { - ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(mSequenceChecker); return mQueuedBuffers.size(); } @@ -1439,6 +1439,27 @@ int32_t V4L2Device::h264LevelIdcToV4L2H264Level(uint8_t levelIdc) { } // static +v4l2_mpeg_video_bitrate_mode V4L2Device::C2BitrateModeToV4L2BitrateMode( + C2Config::bitrate_mode_t bitrateMode) { + switch (bitrateMode) { + case C2Config::bitrate_mode_t::BITRATE_CONST_SKIP_ALLOWED: + ALOGW("BITRATE_CONST_SKIP_ALLOWED not supported, defaulting to BITRATE_CONST"); + FALLTHROUGH; + case C2Config::bitrate_mode_t::BITRATE_CONST: + return V4L2_MPEG_VIDEO_BITRATE_MODE_CBR; + case C2Config::bitrate_mode_t::BITRATE_VARIABLE_SKIP_ALLOWED: + ALOGW("BITRATE_VARIABLE_SKIP_ALLOWED not supported, defaulting to BITRATE_VARIABLE"); + FALLTHROUGH; + case C2Config::bitrate_mode_t::BITRATE_VARIABLE: + return V4L2_MPEG_VIDEO_BITRATE_MODE_VBR; + default: + ALOGW("Unsupported bitrate mode %u, defaulting to BITRATE_VARIABLE", + static_cast<uint32_t>(bitrateMode)); + return V4L2_MPEG_VIDEO_BITRATE_MODE_VBR; + } +} + +// static ui::Size V4L2Device::allocatedSizeFromV4L2Format(const struct v4l2_format& format) { ui::Size codedSize; ui::Size visibleSize; @@ -1509,7 +1530,7 @@ ui::Size V4L2Device::allocatedSizeFromV4L2Format(const struct v4l2_format& forma // Sanity checks. Calculated coded size has to contain given visible size and fulfill buffer // byte size requirements. - ALOG_ASSERT(Rect(codedSize).Contains(Rect(visibleSize))); + ALOG_ASSERT(contains(Rect(codedSize), Rect(visibleSize))); ALOG_ASSERT(sizeimage <= allocationSize(frameFormat, codedSize)); return codedSize; diff --git a/common/include/v4l2_codec2/common/EncodeHelpers.h b/common/include/v4l2_codec2/common/EncodeHelpers.h index bfbdd05..832ac91 100644 --- a/common/include/v4l2_codec2/common/EncodeHelpers.h +++ b/common/include/v4l2_codec2/common/EncodeHelpers.h @@ -42,12 +42,24 @@ uint8_t c2LevelToV4L2Level(C2Config::level_t level); // Get the specified graphics block in YCbCr format. android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block); +// Try to extract SPS and PPS NAL units from the specified H.264 |data| stream. If found the data +// will be copied (after resizing) into the provided |sps| and |pps| buffers. Returns whether +// extraction was successful. +bool extractSPSPPS(const uint8_t* data, size_t length, std::vector<uint8_t>* sps, + std::vector<uint8_t>* pps); + // When encoding a video the codec-specific data (CSD; e.g. SPS and PPS for H264 encoding) will be // concatenated to the first encoded slice. This function extracts the CSD out of the bitstream and -// stores it into |csd|. -void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, +// stores it into |csd|. Returns whether extracting CSD info was successful. +bool extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, size_t length); +// Prepend the specified |sps| and |pps| NAL units (without start codes) to the H.264 |data| stream. +// The result is copied into |dst|. The provided |sps| and |pps| data will be updated if an SPS or +// PPS NAL unit is encountered. Returns the size of the new data, will be 0 if an error occurred. +size_t prependSPSPPSToIDR(const uint8_t* src, size_t srcSize, uint8_t* dst, size_t dstSize, + std::vector<uint8_t>* sps, std::vector<uint8_t>* pps); + } // namespace android #endif // ANDROID_V4L2_CODEC2_COMMON_HELPERS_H diff --git a/common/include/v4l2_codec2/common/NalParser.h b/common/include/v4l2_codec2/common/NalParser.h index 69f56c3..ec8a876 100644 --- a/common/include/v4l2_codec2/common/NalParser.h +++ b/common/include/v4l2_codec2/common/NalParser.h @@ -12,6 +12,8 @@ namespace android { // Helper class to parse H264 NAL units from data. class NalParser { public: + // Type of a IDR Slice NAL unit. + static constexpr uint8_t kIDRType = 5; // Type of a SPS NAL unit. static constexpr uint8_t kSPSType = 7; // Type of a PPS NAL unit. @@ -19,9 +21,9 @@ public: // Parameters related to a video's color aspects. struct ColorAspects { - int32_t primaries; - int32_t transfer; - int32_t coeffs; + uint32_t primaries; + uint32_t transfer; + uint32_t coeffs; bool fullRange; }; diff --git a/common/include/v4l2_codec2/common/V4L2Device.h b/common/include/v4l2_codec2/common/V4L2Device.h index b4c909c..77d7ddb 100644 --- a/common/include/v4l2_codec2/common/V4L2Device.h +++ b/common/include/v4l2_codec2/common/V4L2Device.h @@ -349,6 +349,8 @@ public: // Convert required H264 profile and level to V4L2 enums. static int32_t c2ProfileToV4L2H264Profile(C2Config::profile_t profile); static int32_t h264LevelIdcToV4L2H264Level(uint8_t levelIdc); + static v4l2_mpeg_video_bitrate_mode C2BitrateModeToV4L2BitrateMode( + C2Config::bitrate_mode_t bitrateMode); // Converts v4l2_memory to a string. static const char* v4L2MemoryToString(const v4l2_memory memory); diff --git a/components/Android.bp b/components/Android.bp index 16c7d20..5bee73b 100644 --- a/components/Android.bp +++ b/components/Android.bp @@ -47,8 +47,6 @@ cc_library { "libstagefright_bufferqueue_helper", "libstagefright_foundation", "libui", - ], - static_libs: [ "libv4l2_codec2_common", ], diff --git a/components/V4L2DecodeComponent.cpp b/components/V4L2DecodeComponent.cpp index 400c765..456f3c4 100644 --- a/components/V4L2DecodeComponent.cpp +++ b/components/V4L2DecodeComponent.cpp @@ -35,6 +35,28 @@ namespace android { namespace { +// CCBC pauses sending input buffers to the component when all the output slots are filled by +// pending decoded buffers. If the available output buffers are exhausted before CCBC pauses sending +// input buffers, CCodec may timeout due to waiting for a available output buffer. +// This function returns the minimum number of output buffers to prevent the buffers from being +// exhausted before CCBC pauses sending input buffers. +size_t getMinNumOutputBuffers(VideoCodec codec) { + // The constant values copied from CCodecBufferChannel.cpp. + // (b/184020290): Check the value still sync when seeing error message from CCodec: + // "previous call to queue exceeded timeout". + constexpr size_t kSmoothnessFactor = 4; + constexpr size_t kRenderingDepth = 3; + // Extra number of needed output buffers for V4L2Decoder. + constexpr size_t kExtraNumOutputBuffersForDecoder = 2; + + // The total needed number of output buffers at pipeline are: + // - MediaCodec output slots: output delay + kSmoothnessFactor + // - Surface: kRenderingDepth + // - Component: kExtraNumOutputBuffersForDecoder + return V4L2DecodeInterface::getOutputDelay(codec) + kSmoothnessFactor + kRenderingDepth + + kExtraNumOutputBuffersForDecoder; +} + // Mask against 30 bits to avoid (undefined) wraparound on signed integer. int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) { return static_cast<int32_t>(frameIndex.peeku() & 0x3FFFFFFF); @@ -124,7 +146,7 @@ std::shared_ptr<C2Component> V4L2DecodeComponent::create( const std::string& name, c2_node_id_t id, const std::shared_ptr<C2ReflectorHelper>& helper, C2ComponentFactory::ComponentDeleter deleter) { static const int32_t kMaxConcurrentInstances = - property_get_int32("debug.v4l2_codec2.decode.concurrent-instances", -1); + property_get_int32("ro.vendor.v4l2_codec2.decode_concurrent_instances", -1); static std::mutex mutex; std::lock_guard<std::mutex> lock(mutex); @@ -180,7 +202,6 @@ c2_status_t V4L2DecodeComponent::start() { } mDecoderTaskRunner = mDecoderThread.task_runner(); mWeakThis = mWeakThisFactory.GetWeakPtr(); - mStdWeakThis = weak_from_this(); c2_status_t status = C2_CORRUPTED; ::base::WaitableEvent done; @@ -207,9 +228,11 @@ void V4L2DecodeComponent::startTask(c2_status_t* status, ::base::WaitableEvent* return; } const size_t inputBufferSize = mIntfImpl->getInputBufferSize(); + const size_t minNumOutputBuffers = getMinNumOutputBuffers(*codec); + // ::base::Unretained(this) is safe here because |mDecoder| is always destroyed before // |mDecoderThread| is stopped, so |*this| is always valid during |mDecoder|'s lifetime. - mDecoder = V4L2Decoder::Create(*codec, inputBufferSize, + mDecoder = V4L2Decoder::Create(*codec, inputBufferSize, minNumOutputBuffers, ::base::BindRepeating(&V4L2DecodeComponent::getVideoFramePool, ::base::Unretained(this)), ::base::BindRepeating(&V4L2DecodeComponent::onOutputFrameReady, @@ -237,7 +260,7 @@ std::unique_ptr<VideoFramePool> V4L2DecodeComponent::getVideoFramePool(const ui: ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); - auto sharedThis = mStdWeakThis.lock(); + auto sharedThis = weak_from_this().lock(); if (sharedThis == nullptr) { ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); return nullptr; @@ -324,7 +347,6 @@ void V4L2DecodeComponent::releaseTask() { ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); mWeakThisFactory.InvalidateWeakPtrs(); - mStdWeakThis.reset(); mDecoder = nullptr; } @@ -470,9 +492,8 @@ void V4L2DecodeComponent::pumpPendingWorks() { } } - std::unique_ptr<BitstreamBuffer> buffer = - std::make_unique<BitstreamBuffer>(bitstreamId, linearBlock.handle()->data[0], - linearBlock.offset(), linearBlock.size()); + std::unique_ptr<ConstBitstreamBuffer> buffer = std::make_unique<ConstBitstreamBuffer>( + bitstreamId, linearBlock, linearBlock.offset(), linearBlock.size()); if (!buffer) { reportError(C2_CORRUPTED); return; @@ -675,12 +696,6 @@ bool V4L2DecodeComponent::reportWork(std::unique_ptr<C2Work> work) { ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull()); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); - auto sharedThis = mStdWeakThis.lock(); - if (sharedThis == nullptr) { - ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); - return false; - } - if (!mListener) { ALOGE("mListener is nullptr, setListener_vb() not called?"); return false; @@ -688,7 +703,7 @@ bool V4L2DecodeComponent::reportWork(std::unique_ptr<C2Work> work) { std::list<std::unique_ptr<C2Work>> finishedWorks; finishedWorks.emplace_back(std::move(work)); - mListener->onWorkDone_nb(std::move(sharedThis), std::move(finishedWorks)); + mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorks)); return true; } @@ -725,12 +740,6 @@ void V4L2DecodeComponent::reportAbandonedWorks() { ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); - auto sharedThis = mStdWeakThis.lock(); - if (sharedThis == nullptr) { - ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); - return; - } - std::list<std::unique_ptr<C2Work>> abandonedWorks; while (!mPendingWorks.empty()) { abandonedWorks.emplace_back(std::move(mPendingWorks.front())); @@ -754,7 +763,7 @@ void V4L2DecodeComponent::reportAbandonedWorks() { ALOGE("mListener is nullptr, setListener_vb() not called?"); return; } - mListener->onWorkDone_nb(std::move(sharedThis), std::move(abandonedWorks)); + mListener->onWorkDone_nb(weak_from_this(), std::move(abandonedWorks)); } } @@ -828,12 +837,6 @@ void V4L2DecodeComponent::reportError(c2_status_t error) { ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error)); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); - auto sharedThis = mStdWeakThis.lock(); - if (sharedThis == nullptr) { - ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); - return; - } - if (mComponentState.load() == ComponentState::ERROR) return; mComponentState.store(ComponentState::ERROR); @@ -841,7 +844,7 @@ void V4L2DecodeComponent::reportError(c2_status_t error) { ALOGE("mListener is nullptr, setListener_vb() not called?"); return; } - mListener->onError_nb(std::move(sharedThis), static_cast<uint32_t>(error)); + mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error)); } c2_status_t V4L2DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) { diff --git a/components/V4L2Decoder.cpp b/components/V4L2Decoder.cpp index d694837..18d62d2 100644 --- a/components/V4L2Decoder.cpp +++ b/components/V4L2Decoder.cpp @@ -9,6 +9,7 @@ #include <stdint.h> +#include <algorithm> #include <vector> #include <base/bind.h> @@ -48,12 +49,13 @@ uint32_t VideoCodecToV4L2PixFmt(VideoCodec codec) { // static std::unique_ptr<VideoDecoder> V4L2Decoder::Create( - const VideoCodec& codec, const size_t inputBufferSize, GetPoolCB getPoolCb, - OutputCB outputCb, ErrorCB errorCb, scoped_refptr<::base::SequencedTaskRunner> taskRunner) { + const VideoCodec& codec, const size_t inputBufferSize, const size_t minNumOutputBuffers, + GetPoolCB getPoolCb, OutputCB outputCb, ErrorCB errorCb, + scoped_refptr<::base::SequencedTaskRunner> taskRunner) { std::unique_ptr<V4L2Decoder> decoder = ::base::WrapUnique<V4L2Decoder>(new V4L2Decoder(taskRunner)); - if (!decoder->start(codec, inputBufferSize, std::move(getPoolCb), std::move(outputCb), - std::move(errorCb))) { + if (!decoder->start(codec, inputBufferSize, minNumOutputBuffers, std::move(getPoolCb), + std::move(outputCb), std::move(errorCb))) { return nullptr; } return decoder; @@ -89,12 +91,14 @@ V4L2Decoder::~V4L2Decoder() { } } -bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, GetPoolCB getPoolCb, - OutputCB outputCb, ErrorCB errorCb) { - ALOGV("%s(codec=%s, inputBufferSize=%zu)", __func__, VideoCodecToString(codec), - inputBufferSize); +bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, + const size_t minNumOutputBuffers, GetPoolCB getPoolCb, OutputCB outputCb, + ErrorCB errorCb) { + ALOGV("%s(codec=%s, inputBufferSize=%zu, minNumOutputBuffers=%zu)", __func__, + VideoCodecToString(codec), inputBufferSize, minNumOutputBuffers); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + mMinNumOutputBuffers = minNumOutputBuffers; mGetPoolCb = std::move(getPoolCb); mOutputCb = std::move(outputCb); mErrorCb = std::move(errorCb); @@ -188,7 +192,7 @@ bool V4L2Decoder::setupInputFormat(const uint32_t inputPixelFormat, const size_t return true; } -void V4L2Decoder::decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) { +void V4L2Decoder::decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) { ALOGV("%s(id=%d)", __func__, buffer->id); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); @@ -296,7 +300,7 @@ void V4L2Decoder::pumpDecodeRequest() { inputBuffer->setPlaneDataOffset(0, request.buffer->offset); inputBuffer->setPlaneBytesUsed(0, request.buffer->offset + request.buffer->size); std::vector<int> fds; - fds.push_back(std::move(request.buffer->dmabuf_fd)); + fds.push_back(std::move(request.buffer->dmabuf.handle()->data[0])); if (!std::move(*inputBuffer).queueDMABuf(fds)) { ALOGE("%s(): Failed to QBUF to input queue, bitstreamId=%d", __func__, bitstreamId); onError(); @@ -330,6 +334,7 @@ void V4L2Decoder::flush() { } // Streamoff both V4L2 queues to drop input and output buffers. + const bool isOutputStreaming = mOutputQueue->isStreaming(); mDevice->stopPolling(); mOutputQueue->streamoff(); mFrameAtDevice.clear(); @@ -337,7 +342,9 @@ void V4L2Decoder::flush() { // Streamon both V4L2 queues. mInputQueue->streamon(); - mOutputQueue->streamon(); + if (isOutputStreaming) { + mOutputQueue->streamon(); + } // If there is no free buffer at mOutputQueue, tryFetchVideoFrame() should be triggerred after // a buffer is DQBUF from output queue. Now all the buffers are dropped at mOutputQueue, we @@ -496,6 +503,7 @@ bool V4L2Decoder::changeResolution() { if (!format || !numOutputBuffers) { return false; } + *numOutputBuffers = std::max(*numOutputBuffers, mMinNumOutputBuffers); const ui::Size codedSize(format->fmt.pix_mp.width, format->fmt.pix_mp.height); if (!setupOutputFormat(codedSize)) { diff --git a/components/V4L2EncodeComponent.cpp b/components/V4L2EncodeComponent.cpp index b4bbc0e..a1b46ab 100644 --- a/components/V4L2EncodeComponent.cpp +++ b/components/V4L2EncodeComponent.cpp @@ -40,6 +40,9 @@ namespace { const VideoPixelFormat kInputPixelFormat = VideoPixelFormat::NV12; +// The peak bitrate in function of the target bitrate, used when the bitrate mode is VBR. +constexpr uint32_t kPeakBitrateMultiplier = 2u; + // Get the video frame layout from the specified |inputBlock|. // TODO(dstaessens): Clean up code extracting layout from a C2GraphicBlock. std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGraphicBlock& block, @@ -186,6 +189,12 @@ std::unique_ptr<V4L2Encoder::InputFrame> CreateInputFrame(const C2ConstGraphicBl format, index, timestamp); } +// Check whether the specified |profile| is an H.264 profile. +bool IsH264Profile(C2Config::profile_t profile) { + return (profile >= C2Config::PROFILE_AVC_BASELINE && + profile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH); +} + } // namespace // static @@ -198,7 +207,7 @@ std::shared_ptr<C2Component> V4L2EncodeComponent::create( ALOGV("%s(%s)", __func__, name.c_str()); static const int32_t kMaxConcurrentInstances = - property_get_int32("debug.v4l2_codec2.encode.concurrent-instances", -1); + property_get_int32("ro.vendor.v4l2_codec2.encode_concurrent_instances", -1); static std::mutex mutex; std::lock_guard<std::mutex> lock(mutex); @@ -617,14 +626,16 @@ bool V4L2EncodeComponent::initializeEncoder() { ALOG_ASSERT(!mInputFormatConverter); ALOG_ASSERT(!mEncoder); - mCSDSubmitted = false; + mLastFrameTime = std::nullopt; // Get the requested profile and level. C2Config::profile_t outputProfile = mInterface->getOutputProfile(); + // CSD only needs to be extracted when using an H.264 profile. + mExtractCSD = IsH264Profile(outputProfile); + std::optional<uint8_t> h264Level; - if (outputProfile >= C2Config::PROFILE_AVC_BASELINE && - outputProfile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) { + if (IsH264Profile(outputProfile)) { h264Level = c2LevelToV4L2Level(mInterface->getOutputLevel()); } @@ -638,9 +649,15 @@ bool V4L2EncodeComponent::initializeEncoder() { return false; } + // Get the requested bitrate mode and bitrate. The C2 framework doesn't offer a parameter to + // configure the peak bitrate, so we use a multiple of the target bitrate. + mBitrateMode = mInterface->getBitrateMode(); + mBitrate = mInterface->getBitrate(); + mEncoder = V4L2Encoder::create( outputProfile, h264Level, mInterface->getInputVisibleSize(), *stride, - mInterface->getKeyFramePeriod(), + mInterface->getKeyFramePeriod(), mBitrateMode, mBitrate, + mBitrate * kPeakBitrateMultiplier, ::base::BindRepeating(&V4L2EncodeComponent::fetchOutputBlock, mWeakThis), ::base::BindRepeating(&V4L2EncodeComponent::onInputBufferDone, mWeakThis), ::base::BindRepeating(&V4L2EncodeComponent::onOutputBufferDone, mWeakThis), @@ -670,19 +687,10 @@ bool V4L2EncodeComponent::updateEncodingParameters() { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - // Query the interface for the encoding parameters requested by the codec 2.0 framework. - C2StreamBitrateInfo::output bitrateInfo; - C2StreamFrameRateInfo::output framerateInfo; - c2_status_t status = - mInterface->query({&bitrateInfo, &framerateInfo}, {}, C2_DONT_BLOCK, nullptr); - if (status != C2_OK) { - ALOGE("Failed to query interface for encoding parameters (error code: %d)", status); - reportError(status); - return false; - } - - // Ask device to change bitrate if it's different from the currently configured bitrate. - uint32_t bitrate = bitrateInfo.value; + // Ask device to change bitrate if it's different from the currently configured bitrate. The C2 + // framework doesn't offer a parameter to configure the peak bitrate, so we'll use a multiple of + // the target bitrate here. The peak bitrate is only used if the bitrate mode is set to VBR. + uint32_t bitrate = mInterface->getBitrate(); if (mBitrate != bitrate) { ALOG_ASSERT(bitrate > 0u); ALOGV("Setting bitrate to %u", bitrate); @@ -691,10 +699,17 @@ bool V4L2EncodeComponent::updateEncodingParameters() { return false; } mBitrate = bitrate; + + if (mBitrateMode == C2Config::BITRATE_VARIABLE) { + ALOGV("Setting peak bitrate to %u", bitrate * kPeakBitrateMultiplier); + // TODO(b/190336806): Our stack doesn't support dynamic peak bitrate changes yet, ignore + // errors for now. + mEncoder->setPeakBitrate(bitrate * kPeakBitrateMultiplier); + } } // Ask device to change framerate if it's different from the currently configured framerate. - uint32_t framerate = static_cast<uint32_t>(std::round(framerateInfo.value)); + uint32_t framerate = static_cast<uint32_t>(std::round(mInterface->getFramerate())); if (mFramerate != framerate) { ALOG_ASSERT(framerate > 0u); ALOGV("Setting framerate to %u", framerate); @@ -709,7 +724,7 @@ bool V4L2EncodeComponent::updateEncodingParameters() { // Check whether an explicit key frame was requested, if so reset the key frame counter to // immediately request a key frame. C2StreamRequestSyncFrameTuning::output requestKeyFrame; - status = mInterface->query({&requestKeyFrame}, {}, C2_DONT_BLOCK, nullptr); + c2_status_t status = mInterface->query({&requestKeyFrame}, {}, C2_DONT_BLOCK, nullptr); if (status != C2_OK) { ALOGE("Failed to query interface for key frame request (error code: %d)", status); reportError(status); @@ -738,6 +753,18 @@ bool V4L2EncodeComponent::encode(C2ConstGraphicBlock block, uint64_t index, int6 ALOGV("Encoding input block (index: %" PRIu64 ", timestamp: %" PRId64 ", size: %dx%d)", index, timestamp, block.width(), block.height()); + // Dynamically adjust framerate based on the frame's timestamp if required. + constexpr int64_t kMaxFramerateDiff = 5; + if (mLastFrameTime && (timestamp > *mLastFrameTime)) { + int64_t newFramerate = + static_cast<int64_t>(std::round(1000000.0 / (timestamp - *mLastFrameTime))); + if (abs(mFramerate - newFramerate) > kMaxFramerateDiff) { + ALOGV("Adjusting framerate to %" PRId64 " based on frame timestamps", newFramerate); + mInterface->setFramerate(static_cast<uint32_t>(newFramerate)); + } + } + mLastFrameTime = timestamp; + // Update dynamic encoding parameters (bitrate, framerate, key frame) if requested. if (!updateEncodingParameters()) return false; @@ -785,7 +812,7 @@ void V4L2EncodeComponent::flush() { mWorkQueue.pop_front(); } if (!abortedWorkItems.empty()) { - mListener->onWorkDone_nb(shared_from_this(), std::move(abortedWorkItems)); + mListener->onWorkDone_nb(weak_from_this(), std::move(abortedWorkItems)); } } @@ -803,13 +830,7 @@ void V4L2EncodeComponent::fetchOutputBlock(uint32_t size, reportError(status); } - // Store a reference to the block to keep the fds alive. - int fd = block->handle()->data[0]; - ALOG_ASSERT(!mOutputBuffersMap[fd]); - mOutputBuffersMap[fd] = std::move(block); - - // TODO(dstaessens) Store the C2LinearBlock directly into the BitstreamBuffer. - *buffer = std::make_unique<BitstreamBuffer>(fd, fd, 0, size); + *buffer = std::make_unique<BitstreamBuffer>(std::move(block), 0, size); } void V4L2EncodeComponent::onInputBufferDone(uint64_t index) { @@ -859,22 +880,19 @@ void V4L2EncodeComponent::onOutputBufferDone(size_t dataSize, int64_t timestamp, ALOGV("%s(): output buffer done (timestamp: %" PRId64 ", size: %zu, keyframe: %d)", __func__, timestamp, dataSize, keyFrame); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(buffer->dmabuf); - std::shared_ptr<C2LinearBlock> outputBlock = std::move(mOutputBuffersMap[buffer->id]); - mOutputBuffersMap.erase(buffer->id); - ALOG_ASSERT(outputBlock); - - C2ConstLinearBlock constBlock = outputBlock->share(outputBlock->offset(), dataSize, C2Fence()); + C2ConstLinearBlock constBlock = + buffer->dmabuf->share(buffer->dmabuf->offset(), dataSize, C2Fence()); // If no CSD (content-specific-data, e.g. SPS for H.264) has been submitted yet, we expect this // output block to contain CSD. We only submit the CSD once, even if it's attached to each key // frame. - if (!mCSDSubmitted) { + if (mExtractCSD) { ALOGV("No CSD submitted yet, extracting CSD"); std::unique_ptr<C2StreamInitDataInfo::output> csd; C2ReadView view = constBlock.map().get(); - extractCSDInfo(&csd, view.data(), view.capacity()); - if (!csd) { + if (!extractCSDInfo(&csd, view.data(), view.capacity())) { ALOGE("Failed to extract CSD"); reportError(C2_CORRUPTED); return; @@ -884,7 +902,7 @@ void V4L2EncodeComponent::onOutputBufferDone(size_t dataSize, int64_t timestamp, LOG_ASSERT(!mWorkQueue.empty()); C2Work* work = mWorkQueue.front().get(); work->worklets.front()->output.configUpdate.push_back(std::move(csd)); - mCSDSubmitted = true; + mExtractCSD = false; } // Get the work item associated with the timestamp. @@ -993,15 +1011,23 @@ void V4L2EncodeComponent::reportWork(std::unique_ptr<C2Work> work) { std::list<std::unique_ptr<C2Work>> finishedWorkList; finishedWorkList.emplace_back(std::move(work)); - mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorkList)); + mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorkList)); } bool V4L2EncodeComponent::getBlockPool() { + ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); + + auto sharedThis = weak_from_this().lock(); + if (!sharedThis) { + ALOGI("%s(): V4L2EncodeComponent instance is already destroyed", __func__); + return false; + } + C2BlockPool::local_id_t poolId = mInterface->getBlockPoolId(); if (poolId == C2BlockPool::BASIC_LINEAR) { ALOGW("Using unoptimized linear block pool"); } - c2_status_t status = GetCodec2BlockPool(poolId, shared_from_this(), &mOutputBlockPool); + c2_status_t status = GetCodec2BlockPool(poolId, std::move(sharedThis), &mOutputBlockPool); if (status != C2_OK || !mOutputBlockPool) { ALOGE("Failed to get output block pool, error: %d", status); return false; @@ -1017,7 +1043,7 @@ void V4L2EncodeComponent::reportError(c2_status_t error) { std::lock_guard<std::mutex> lock(mComponentLock); if (mComponentState != ComponentState::ERROR) { setComponentState(ComponentState::ERROR); - mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error)); + mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error)); } } diff --git a/components/V4L2EncodeInterface.cpp b/components/V4L2EncodeInterface.cpp index 7f0fb39..03d8c37 100644 --- a/components/V4L2EncodeInterface.cpp +++ b/components/V4L2EncodeInterface.cpp @@ -310,6 +310,15 @@ void V4L2EncodeInterface::Initialize(const C2String& name) { .withSetter(Setter<decltype(*mBitrate)>::StrictValueWithNoDeps) .build()); + addParameter( + DefineParam(mBitrateMode, C2_PARAMKEY_BITRATE_MODE) + .withDefault(new C2StreamBitrateModeTuning::output(0u, C2Config::BITRATE_CONST)) + .withFields( + {C2F(mBitrateMode, value) + .oneOf({C2Config::BITRATE_CONST, C2Config::BITRATE_VARIABLE})}) + .withSetter(Setter<decltype(*mBitrateMode)>::StrictValueWithNoDeps) + .build()); + std::string outputMime; if (getCodecFromComponentName(name) == VideoCodec::H264) { outputMime = MEDIA_MIMETYPE_VIDEO_AVC; @@ -329,8 +338,8 @@ void V4L2EncodeInterface::Initialize(const C2String& name) { C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2, C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1, C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4, - C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_5, - C2Config::LEVEL_AVC_5_1})}) + C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2, + C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1})}) .withSetter(H264ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate) .build()); } else if (getCodecFromComponentName(name) == VideoCodec::VP8) { diff --git a/components/V4L2Encoder.cpp b/components/V4L2Encoder.cpp index cdd2d89..cd20cb5 100644 --- a/components/V4L2Encoder.cpp +++ b/components/V4L2Encoder.cpp @@ -17,6 +17,7 @@ #include <log/log.h> #include <ui/Rect.h> +#include <v4l2_codec2/common/EncodeHelpers.h> #include <v4l2_codec2/common/Fourcc.h> #include <v4l2_codec2/common/V4L2Device.h> #include <v4l2_codec2/components/BitstreamBuffer.h> @@ -54,6 +55,7 @@ size_t GetMaxOutputBufferSize(const ui::Size& size) { std::unique_ptr<VideoEncoder> V4L2Encoder::create( C2Config::profile_t outputProfile, std::optional<uint8_t> level, const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod, + C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate, std::optional<uint32_t> peakBitrate, FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB inputBufferDoneCb, OutputBufferDoneCB outputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb, scoped_refptr<::base::SequencedTaskRunner> taskRunner) { @@ -62,7 +64,8 @@ std::unique_ptr<VideoEncoder> V4L2Encoder::create( std::unique_ptr<V4L2Encoder> encoder = ::base::WrapUnique<V4L2Encoder>(new V4L2Encoder( std::move(taskRunner), std::move(fetchOutputBufferCb), std::move(inputBufferDoneCb), std::move(outputBufferDoneCb), std::move(drainDoneCb), std::move(errorCb))); - if (!encoder->initialize(outputProfile, level, visibleSize, stride, keyFramePeriod)) { + if (!encoder->initialize(outputProfile, level, visibleSize, stride, keyFramePeriod, bitrateMode, + bitrate, peakBitrate)) { return nullptr; } return encoder; @@ -161,6 +164,19 @@ bool V4L2Encoder::setBitrate(uint32_t bitrate) { return true; } +bool V4L2Encoder::setPeakBitrate(uint32_t peakBitrate) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, peakBitrate)})) { + // TODO(b/190336806): Our stack doesn't support dynamic peak bitrate changes yet, ignore + // errors for now. + ALOGW("Setting peak bitrate to %u failed", peakBitrate); + } + return true; +} + bool V4L2Encoder::setFramerate(uint32_t framerate) { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); @@ -189,8 +205,9 @@ VideoPixelFormat V4L2Encoder::inputFormat() const { } bool V4L2Encoder::initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level, - const ui::Size& visibleSize, uint32_t stride, - uint32_t keyFramePeriod) { + const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod, + C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate, + std::optional<uint32_t> peakBitrate) { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); ALOG_ASSERT(keyFramePeriod > 0); @@ -238,6 +255,12 @@ bool V4L2Encoder::initialize(C2Config::profile_t outputProfile, std::optional<ui return false; } + // Configure the requested bitrate mode and bitrate on the device. + if (!configureBitrateMode(bitrateMode) || !setBitrate(bitrate)) return false; + + // If the bitrate mode is VBR we also need to configure the peak bitrate on the device. + if ((bitrateMode == C2Config::BITRATE_VARIABLE) && !setPeakBitrate(*peakBitrate)) return false; + // First try to configure the specified output format, as changing the output format can affect // the configured input format. if (!configureOutputFormat(outputProfile)) return false; @@ -582,7 +605,7 @@ bool V4L2Encoder::configureDevice(C2Config::profile_t outputProfile, V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0)}); // All controls below are H.264-specific, so we can return here if the profile is not H.264. - if (outputProfile >= C2Config::PROFILE_AVC_BASELINE || + if (outputProfile >= C2Config::PROFILE_AVC_BASELINE && outputProfile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) { return configureH264(outputProfile, outputH264Level); } @@ -594,17 +617,18 @@ bool V4L2Encoder::configureH264(C2Config::profile_t outputProfile, std::optional<const uint8_t> outputH264Level) { // When encoding H.264 we want to prepend SPS and PPS to each IDR for resilience. Some // devices support this through the V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR control. - // TODO(b/161495502): V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR is currently not supported - // yet, just log a warning if the operation was unsuccessful for now. + // Otherwise we have to cache the latest SPS and PPS and inject these manually. if (mDevice->isCtrlExposed(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR)) { if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR, 1)})) { ALOGE("Failed to configure device to prepend SPS and PPS to each IDR"); return false; } + mInjectParamsBeforeIDR = false; ALOGV("Device supports prepending SPS and PPS to each IDR"); } else { - ALOGW("Device doesn't support prepending SPS and PPS to IDR"); + mInjectParamsBeforeIDR = true; + ALOGV("Device doesn't support prepending SPS and PPS to IDR, injecting manually."); } std::vector<V4L2ExtCtrl> h264Ctrls; @@ -637,6 +661,21 @@ bool V4L2Encoder::configureH264(C2Config::profile_t outputProfile, return true; } +bool V4L2Encoder::configureBitrateMode(C2Config::bitrate_mode_t bitrateMode) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + v4l2_mpeg_video_bitrate_mode v4l2BitrateMode = + V4L2Device::C2BitrateModeToV4L2BitrateMode(bitrateMode); + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE_MODE, v4l2BitrateMode)})) { + // TODO(b/190336806): Our stack doesn't support bitrate mode changes yet. We default to CBR + // which is currently the only supported mode so we can safely ignore this for now. + ALOGW("Setting bitrate mode to %u failed", v4l2BitrateMode); + } + return true; +} + bool V4L2Encoder::startDevicePoll() { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); @@ -780,7 +819,7 @@ bool V4L2Encoder::enqueueOutputBuffer() { size_t bufferId = buffer->bufferId(); std::vector<int> fds; - fds.push_back(bitstreamBuffer->dmabuf_fd); + fds.push_back(bitstreamBuffer->dmabuf->handle()->data[0]); if (!std::move(*buffer).queueDMABuf(fds)) { ALOGE("Failed to queue output buffer using QueueDMABuf"); onError(); @@ -875,11 +914,51 @@ bool V4L2Encoder::dequeueOutputBuffer() { return false; } - std::unique_ptr<BitstreamBuffer> bitstream_buffer = + std::unique_ptr<BitstreamBuffer> bitstreamBuffer = std::move(mOutputBuffers[buffer->bufferId()]); if (encodedDataSize > 0) { - mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), buffer->isKeyframe(), - std::move(bitstream_buffer)); + if (!mInjectParamsBeforeIDR) { + // No need to inject SPS or PPS before IDR frames, we can just return the buffer as-is. + mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), + buffer->isKeyframe(), std::move(bitstreamBuffer)); + } else if (!buffer->isKeyframe()) { + // We need to inject SPS and PPS before IDR frames, but this frame is not a key frame. + // We can return the buffer as-is, but need to update our SPS and PPS cache if required. + C2ConstLinearBlock constBlock = bitstreamBuffer->dmabuf->share( + bitstreamBuffer->dmabuf->offset(), encodedDataSize, C2Fence()); + C2ReadView readView = constBlock.map().get(); + extractSPSPPS(readView.data(), encodedDataSize, &mCachedSPS, &mCachedPPS); + mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), + buffer->isKeyframe(), std::move(bitstreamBuffer)); + } else { + // We need to inject our cached SPS and PPS NAL units to the IDR frame. It's possible + // this frame already has SPS and PPS NAL units attached, in which case we only need to + // update our cached SPS and PPS. + C2ConstLinearBlock constBlock = bitstreamBuffer->dmabuf->share( + bitstreamBuffer->dmabuf->offset(), encodedDataSize, C2Fence()); + C2ReadView readView = constBlock.map().get(); + + // Allocate a new buffer to copy the data with prepended SPS and PPS into. + std::unique_ptr<BitstreamBuffer> prependedBitstreamBuffer; + mFetchOutputBufferCb.Run(mOutputBufferSize, &prependedBitstreamBuffer); + if (!prependedBitstreamBuffer) { + ALOGE("Failed to fetch output block"); + onError(); + return false; + } + C2WriteView writeView = prependedBitstreamBuffer->dmabuf->map().get(); + + // If there is not enough space in the output buffer just return the original buffer. + size_t newSize = prependSPSPPSToIDR(readView.data(), encodedDataSize, writeView.data(), + writeView.size(), &mCachedSPS, &mCachedPPS); + if (newSize > 0) { + mOutputBufferDoneCb.Run(newSize, timestamp.InMicroseconds(), buffer->isKeyframe(), + std::move(prependedBitstreamBuffer)); + } else { + mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), + buffer->isKeyframe(), std::move(bitstreamBuffer)); + } + } } // If the buffer is marked as last and we were flushing the encoder, flushing is now done. diff --git a/components/include/v4l2_codec2/components/BitstreamBuffer.h b/components/include/v4l2_codec2/components/BitstreamBuffer.h index d61e4f9..97cb203 100644 --- a/components/include/v4l2_codec2/components/BitstreamBuffer.h +++ b/components/include/v4l2_codec2/components/BitstreamBuffer.h @@ -7,18 +7,30 @@ #include <stdint.h> +#include <C2Buffer.h> + namespace android { -// The BitstreamBuffer class can be used to store encoded video data. -// Note: The BitstreamBuffer does not take ownership of the data. The file descriptor is not -// duplicated and the caller is responsible for keeping the data alive. +// The ConstBitstreamBuffer class can be used to store non-modifiable encoded video data. +struct ConstBitstreamBuffer { + ConstBitstreamBuffer(const int32_t id, C2ConstLinearBlock dmabuf, const size_t offset, + const size_t size) + : id(id), dmabuf(std::move(dmabuf)), offset(offset), size(size) {} + ~ConstBitstreamBuffer() = default; + + const int32_t id; + C2ConstLinearBlock dmabuf; + const size_t offset; + const size_t size; +}; + +// The BitstreamBuffer class can be used to store modifiable encoded video data. struct BitstreamBuffer { - BitstreamBuffer(const int32_t id, int dmabuf_fd, const size_t offset, const size_t size) - : id(id), dmabuf_fd(dmabuf_fd), offset(offset), size(size) {} + BitstreamBuffer(std::shared_ptr<C2LinearBlock> dmabuf, const size_t offset, const size_t size) + : dmabuf(std::move(dmabuf)), offset(offset), size(size) {} ~BitstreamBuffer() = default; - const int32_t id; - int dmabuf_fd; + std::shared_ptr<C2LinearBlock> dmabuf; const size_t offset; const size_t size; }; diff --git a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h index 1e98118..962f7d6 100644 --- a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h +++ b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h @@ -140,9 +140,6 @@ private: ::base::Thread mDecoderThread{"V4L2DecodeComponentDecoderThread"}; scoped_refptr<::base::SequencedTaskRunner> mDecoderTaskRunner; - // Hold a weak_ptr of |*this| when |mDecoderThread| is running. - std::weak_ptr<V4L2DecodeComponent> mStdWeakThis; - ::base::WeakPtrFactory<V4L2DecodeComponent> mWeakThisFactory{this}; ::base::WeakPtr<V4L2DecodeComponent> mWeakThis; }; diff --git a/components/include/v4l2_codec2/components/V4L2Decoder.h b/components/include/v4l2_codec2/components/V4L2Decoder.h index b65bd49..2ecb3bd 100644 --- a/components/include/v4l2_codec2/components/V4L2Decoder.h +++ b/components/include/v4l2_codec2/components/V4L2Decoder.h @@ -26,12 +26,12 @@ namespace android { class V4L2Decoder : public VideoDecoder { public: static std::unique_ptr<VideoDecoder> Create( - const VideoCodec& codec, const size_t inputBufferSize, GetPoolCB getPoolCB, - OutputCB outputCb, ErrorCB errorCb, + const VideoCodec& codec, const size_t inputBufferSize, const size_t minNumOutputBuffers, + GetPoolCB getPoolCB, OutputCB outputCb, ErrorCB errorCb, scoped_refptr<::base::SequencedTaskRunner> taskRunner); ~V4L2Decoder() override; - void decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) override; + void decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) override; void drain(DecodeCB drainCb) override; void flush() override; @@ -45,18 +45,19 @@ private: static const char* StateToString(State state); struct DecodeRequest { - DecodeRequest(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) + DecodeRequest(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) : buffer(std::move(buffer)), decodeCb(std::move(decodeCb)) {} DecodeRequest(DecodeRequest&&) = default; ~DecodeRequest() = default; - std::unique_ptr<BitstreamBuffer> buffer; // nullptr means Drain + std::unique_ptr<ConstBitstreamBuffer> buffer; // nullptr means Drain DecodeCB decodeCb; }; V4L2Decoder(scoped_refptr<::base::SequencedTaskRunner> taskRunner); - bool start(const VideoCodec& codec, const size_t inputBufferSize, GetPoolCB getPoolCb, - OutputCB outputCb, ErrorCB errorCb); + bool start(const VideoCodec& codec, const size_t inputBufferSize, + const size_t minNumOutputBuffers, GetPoolCB getPoolCb, OutputCB outputCb, + ErrorCB errorCb); bool setupInputFormat(const uint32_t inputPixelFormat, const size_t inputBufferSize); void pumpDecodeRequest(); @@ -85,6 +86,7 @@ private: std::queue<DecodeRequest> mDecodeRequests; std::map<int32_t, DecodeCB> mPendingDecodeCbs; + size_t mMinNumOutputBuffers = 0; GetPoolCB mGetPoolCb; OutputCB mOutputCb; DecodeCB mDrainCb; diff --git a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h index 4665ffa..0b150e4 100644 --- a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h +++ b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h @@ -155,18 +155,19 @@ private: // The bitrate currently configured on the v4l2 device. uint32_t mBitrate = 0; + // The bitrate mode currently configured on the v4l2 device. + C2Config::bitrate_mode_t mBitrateMode = C2Config::BITRATE_CONST; // The framerate currently configured on the v4l2 device. uint32_t mFramerate = 0; + // The timestamp of the last frame encoded, used to dynamically adjust the framerate. + std::optional<int64_t> mLastFrameTime; - // Whether we extracted and submitted CSD (codec-specific data, e.g. H.264 SPS) to the framework. - bool mCSDSubmitted = false; + // Whether we need to extract and submit CSD (codec-specific data, e.g. H.264 SPS). + bool mExtractCSD = false; // The queue of encode work items currently being processed. std::deque<std::unique_ptr<C2Work>> mWorkQueue; - // Map of buffer ids and associated C2LinearBlock buffers. The buffer's fds are used as id. - std::unordered_map<int32_t, std::shared_ptr<C2LinearBlock>> mOutputBuffersMap; - // The output block pool. std::shared_ptr<C2BlockPool> mOutputBlockPool; diff --git a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h index 2efbfcc..fefebf0 100644 --- a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h +++ b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h @@ -39,8 +39,18 @@ public: return ui::Size(mInputVisibleSize->width, mInputVisibleSize->height); } C2BlockPool::local_id_t getBlockPoolId() const { return mOutputBlockPoolIds->m.values[0]; } + // Get sync key-frame period in frames. uint32_t getKeyFramePeriod() const; + // Get the requested bitrate mode. + C2Config::bitrate_mode_t getBitrateMode() const { return mBitrateMode->value; } + // Get the requested bitrate. + uint32_t getBitrate() const { return mBitrate->value; } + // Get the requested framerate. + float getFramerate() const { return mFrameRate->value; } + + // Request changing the framerate to the specified value. + void setFramerate(uint32_t framerate) { mFrameRate->value = framerate; } protected: void Initialize(const C2String& name); @@ -95,6 +105,8 @@ protected: // The requested bitrate of the encoded output stream, in bits per second. std::shared_ptr<C2StreamBitrateInfo::output> mBitrate; + // The requested bitrate mode. + std::shared_ptr<C2StreamBitrateModeTuning::output> mBitrateMode; // The requested framerate, in frames per second. std::shared_ptr<C2StreamFrameRateInfo::output> mFrameRate; // The switch-type parameter that will be set to true while client requests keyframe. It diff --git a/components/include/v4l2_codec2/components/V4L2Encoder.h b/components/include/v4l2_codec2/components/V4L2Encoder.h index 5abee8f..d7b55c0 100644 --- a/components/include/v4l2_codec2/components/V4L2Encoder.h +++ b/components/include/v4l2_codec2/components/V4L2Encoder.h @@ -33,9 +33,10 @@ public: static std::unique_ptr<VideoEncoder> create( C2Config::profile_t profile, std::optional<uint8_t> level, const ui::Size& visibleSize, - uint32_t stride, uint32_t keyFramePeriod, FetchOutputBufferCB fetchOutputBufferCb, - InputBufferDoneCB inputBufferDoneCb, OutputBufferDoneCB outputBufferDoneCb, - DrainDoneCB drainDoneCb, ErrorCB errorCb, + uint32_t stride, uint32_t keyFramePeriod, C2Config::bitrate_mode_t bitrateMode, + uint32_t bitrate, std::optional<uint32_t> peakBitrate, + FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB inputBufferDoneCb, + OutputBufferDoneCB outputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb, scoped_refptr<::base::SequencedTaskRunner> taskRunner); ~V4L2Encoder() override; @@ -44,6 +45,7 @@ public: void flush() override; bool setBitrate(uint32_t bitrate) override; + bool setPeakBitrate(uint32_t peakBitrate) override; bool setFramerate(uint32_t framerate) override; void requestKeyframe() override; @@ -80,7 +82,9 @@ private: // Initialize the V4L2 encoder for specified parameters. bool initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level, - const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod); + const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod, + C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate, + std::optional<uint32_t> peakBitrate); // Handle the next encode request on the queue. void handleEncodeRequest(); @@ -101,6 +105,8 @@ private: // Configure required and optional H.264 controls on the V4L2 device. bool configureH264(C2Config::profile_t outputProfile, std::optional<const uint8_t> outputH264Level); + // Configure the specified bitrate mode on the V4L2 device. + bool configureBitrateMode(C2Config::bitrate_mode_t bitrateMode); // Attempt to start the V4L2 device poller. bool startDevicePoll(); @@ -158,6 +164,12 @@ private: // Key frame counter, a key frame will be requested each time it reaches zero. uint32_t mKeyFrameCounter = 0; + // Whether we need to manually cache and prepend SPS and PPS to IDR frames. + bool mInjectParamsBeforeIDR = false; + // The latest cached SPS and PPS (without H.264 start code). + std::vector<uint8_t> mCachedSPS; + std::vector<uint8_t> mCachedPPS; + // The V4L2 device and associated queues used to interact with the device. scoped_refptr<V4L2Device> mDevice; scoped_refptr<V4L2Queue> mInputQueue; diff --git a/components/include/v4l2_codec2/components/VideoDecoder.h b/components/include/v4l2_codec2/components/VideoDecoder.h index 9a48562..5b2da41 100644 --- a/components/include/v4l2_codec2/components/VideoDecoder.h +++ b/components/include/v4l2_codec2/components/VideoDecoder.h @@ -34,7 +34,7 @@ public: virtual ~VideoDecoder(); - virtual void decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) = 0; + virtual void decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) = 0; virtual void drain(DecodeCB drainCb) = 0; virtual void flush() = 0; }; diff --git a/components/include/v4l2_codec2/components/VideoEncoder.h b/components/include/v4l2_codec2/components/VideoEncoder.h index 46bcad1..5f23541 100644 --- a/components/include/v4l2_codec2/components/VideoEncoder.h +++ b/components/include/v4l2_codec2/components/VideoEncoder.h @@ -64,8 +64,12 @@ public: // Flush the encoder, pending drain operations will be aborted. virtual void flush() = 0; - // Set the bitrate to the specified value, will affect all non-processed frames. + // Set the target bitrate to the specified value, will affect all non-processed frames. virtual bool setBitrate(uint32_t bitrate) = 0; + // Set the peak bitrate to the specified value. The peak bitrate must be larger or equal to the + // target bitrate and is ignored if the bitrate mode is constant. + virtual bool setPeakBitrate(uint32_t peakBitrate) = 0; + // Set the framerate to the specified value, will affect all non-processed frames. virtual bool setFramerate(uint32_t framerate) = 0; // Request the next frame encoded to be a key frame, will affect the next non-processed frame. diff --git a/plugin_store/C2VdaPooledBlockPool.cpp b/plugin_store/C2VdaPooledBlockPool.cpp index 48cc2e5..2b9104b 100644 --- a/plugin_store/C2VdaPooledBlockPool.cpp +++ b/plugin_store/C2VdaPooledBlockPool.cpp @@ -14,19 +14,6 @@ #include <log/log.h> namespace android { -namespace { -// The wait time for another try to fetch a buffer from bufferpool. -const int64_t kFetchRetryDelayUs = 10 * 1000; - -int64_t GetNowUs() { - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = 0; - clock_gettime(CLOCK_MONOTONIC, &t); - int64_t nsecs = static_cast<int64_t>(t.tv_sec) * 1000000000LL + t.tv_nsec; - return nsecs / 1000ll; -} -} // namespace using android::hardware::media::bufferpool::BufferPoolData; @@ -57,14 +44,6 @@ c2_status_t C2VdaPooledBlockPool::fetchGraphicBlock(uint32_t width, uint32_t hei ALOG_ASSERT(block != nullptr); std::lock_guard<std::mutex> lock(mMutex); - if (mNextFetchTimeUs != 0) { - int delayUs = GetNowUs() - mNextFetchTimeUs; - if (delayUs > 0) { - ::usleep(delayUs); - } - mNextFetchTimeUs = 0; - } - std::shared_ptr<C2GraphicBlock> fetchBlock; c2_status_t err = C2PooledBlockPool::fetchGraphicBlock(width, height, format, usage, &fetchBlock); @@ -89,7 +68,6 @@ c2_status_t C2VdaPooledBlockPool::fetchGraphicBlock(uint32_t width, uint32_t hei return C2_OK; } ALOGV("No buffer could be recycled now, wait for another try..."); - mNextFetchTimeUs = GetNowUs() + kFetchRetryDelayUs; return C2_TIMED_OUT; } diff --git a/plugin_store/DrmGrallocHelpers.cpp b/plugin_store/DrmGrallocHelpers.cpp index 0565a69..0e2cca9 100644 --- a/plugin_store/DrmGrallocHelpers.cpp +++ b/plugin_store/DrmGrallocHelpers.cpp @@ -8,21 +8,58 @@ #include <v4l2_codec2/plugin_store/DrmGrallocHelpers.h> #include <fcntl.h> -#include <string.h> +#include <glob.h> +#include <string> +#include <vector> + +#include <cutils/properties.h> #include <drm/drm.h> #include <log/log.h> namespace android { +namespace { + +// Return a list of paths that matches |pattern|, the unix style pathname pattern. +std::vector<std::string> glob(const std::string& pattern) { + glob_t glob_result; + memset(&glob_result, 0, sizeof(glob_result)); + + std::vector<std::string> paths; + if (glob(pattern.c_str(), GLOB_ERR | GLOB_NOESCAPE, NULL, &glob_result) == 0) { + for (size_t i = 0; i < glob_result.gl_pathc; ++i) { + paths.emplace_back(glob_result.gl_pathv[i]); + } + } + + globfree(&glob_result); + return paths; +} + +std::optional<std::string> propertyGetString(const char* key) { + char buf[PROPERTY_VALUE_MAX]; + int len = property_get(key, buf, nullptr); + if (len == 0) { + return std::nullopt; + } + return std::string(buf, len); +} + +} // namespace std::optional<int> openRenderFd() { - const char kVirglName[] = "virtio_gpu"; + constexpr char kDevNamePropertyKey[] = "ro.vendor.v4l2_codec2.drm_device_name"; + constexpr char kDevPathPropertyKey[] = "ro.vendor.v4l2_codec2.drm_device_path"; - for (uint32_t i = 128; i < 192; i++) { - char devName[32]; - snprintf(devName, sizeof(devName), "/dev/dri/renderD%d", i); + const auto devName = propertyGetString(kDevNamePropertyKey); + if (!devName) { + ALOGE("Failed to get DRM device name from Android property"); + return std::nullopt; + } - int fd = open(devName, O_RDWR | O_CLOEXEC); + const auto devPath = propertyGetString(kDevPathPropertyKey).value_or("/dev/dri/renderD*"); + for (const auto filePath : glob(devPath)) { + int fd = open(filePath.c_str(), O_RDWR | O_CLOEXEC); if (fd < 0) { continue; } @@ -37,7 +74,7 @@ std::optional<int> openRenderFd() { close(fd); continue; } - if (v.name_len != sizeof(kVirglName) - 1 || memcmp(name, kVirglName, v.name_len)) { + if (devName->size() != v.name_len || *devName != name) { close(fd); continue; } diff --git a/plugin_store/include/v4l2_codec2/plugin_store/C2VdaPooledBlockPool.h b/plugin_store/include/v4l2_codec2/plugin_store/C2VdaPooledBlockPool.h index 5603046..749ff47 100644 --- a/plugin_store/include/v4l2_codec2/plugin_store/C2VdaPooledBlockPool.h +++ b/plugin_store/include/v4l2_codec2/plugin_store/C2VdaPooledBlockPool.h @@ -47,9 +47,6 @@ private: std::set<uint32_t> mBufferIds GUARDED_BY(mMutex); // The maximum count of allocated buffers. size_t mBufferCount GUARDED_BY(mMutex){0}; - // The timestamp for the next fetchGraphicBlock() call. - // Set when the previous fetchGraphicBlock() call timed out. - int64_t mNextFetchTimeUs GUARDED_BY(mMutex){0}; }; } // namespace android |