diff options
157 files changed, 8084 insertions, 22964 deletions
diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..7f53aec --- /dev/null +++ b/Android.bp @@ -0,0 +1,44 @@ +// +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + +// Added automatically by a large-scale-change that took the approach of +// 'apply every license found to every target'. While this makes sure we respect +// every license restriction, it may not be entirely correct. +// +// e.g. GPL in an MIT project might only apply to the contrib/ directory. +// +// Please consider splitting the single license below into multiple licenses, +// taking care not to lose any license_kind information, and overriding the +// default license using the 'licenses: [...]' property on targets as needed. +// +// For unused files, consider creating a 'fileGroup' with "//visibility:private" +// to attach the license to, and including a comment whether the files may be +// used in the current project. +// See: http://go/android-license-faq +license { + name: "external_v4l2_codec2_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-BSD", + "legacy_unencumbered", + ], + license_text: [ + "NOTICE", + ], +} diff --git a/METADATA b/METADATA new file mode 100644 index 0000000..d97975c --- /dev/null +++ b/METADATA @@ -0,0 +1,3 @@ +third_party { + license_type: NOTICE +} @@ -6,14 +6,11 @@ Core V4L2 API and codec utilities, ported from Chromium project. * common/ -Common helper classes for both components/ and store/. +Common helper classes for components. * components/ -The C2Component implementations based on V4L2 API. - -* store/ -The implementation of C2ComponentStore. It is used for creating all the -C2Components implemented at components/ folder. +The C2Component implementations based on V4L2 API, and the implementation of +C2ComponentStore for creating all the C2Components. * service/ The Codec2's V4L2 IComponentStore service. The service initiates the component diff --git a/accel/.clang-format b/accel/.clang-format deleted file mode 100644 index d174c9d..0000000 --- a/accel/.clang-format +++ /dev/null @@ -1,4 +0,0 @@ -# The codes in this directory are ported from Chromium Project. -# Therefore, they are obviously based on Chromium coding style -# and shouldn't be formatted by Android Coding Style -BasedOnStyle: Chromium diff --git a/accel/Android.bp b/accel/Android.bp deleted file mode 100644 index 1bf4805..0000000 --- a/accel/Android.bp +++ /dev/null @@ -1,64 +0,0 @@ -cc_library { - name: "libv4l2_codec2_accel", - vendor: true, - - srcs: [ - "bit_reader.cc", - "bit_reader_core.cc", - "bitstream_buffer.cc", - "color_plane_layout.cc", - "fourcc.cc", - "h264_bit_reader.cc", - "h264_decoder.cc", - "h264_dpb.cc", - "h264_parser.cc", - "generic_v4l2_device.cc", - "native_pixmap_handle.cc", - "picture.cc", - "ranges.cc", - "shared_memory_region.cc", - "v4l2_device.cc", - "v4l2_device_poller.cc", - "v4l2_video_decode_accelerator.cc", - "video_codecs.cc", - "video_decode_accelerator.cc", - "video_encode_accelerator.cc", - "video_frame.cc", - "video_frame_layout.cc", - "video_frame_metadata.cc", - "video_pixel_format.cc", - "vp8_bool_decoder.cc", - "vp8_decoder.cc", - "vp8_parser.cc", - "vp8_picture.cc", - "vp9_bool_decoder.cc", - "vp9_compressed_header_parser.cc", - "vp9_decoder.cc", - "vp9_parser.cc", - "vp9_picture.cc", - "vp9_raw_bits_reader.cc", - "vp9_uncompressed_header_parser.cc", - ], - - shared_libs: ["libchrome"], - // -Wno-unused-parameter is needed for libchrome/base codes - cflags: [ - "-Wall", - "-Werror", - "-Wno-unused-parameter", - "-Wno-implicit-fallthrough", // at h264_decoder.cc:1374 - ], - clang: true, - sanitize: { - misc_undefined: [ - "unsigned-integer-overflow", - "signed-integer-overflow", - ], - }, - - ldflags: [ - "-Wl", - "-Bsymbolic", - ], - export_include_dirs: ["."], -} diff --git a/accel/accelerated_video_decoder.h b/accel/accelerated_video_decoder.h deleted file mode 100644 index 238e34d..0000000 --- a/accel/accelerated_video_decoder.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 77118c9 - -#ifndef ACCELERATED_VIDEO_DECODER_H_ -#define ACCELERATED_VIDEO_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include "base/macros.h" -#include "size.h" - -namespace media { - -// An AcceleratedVideoDecoder is a video decoder that requires support from an -// external accelerator (typically a hardware accelerator) to partially offload -// the decode process after parsing stream headers, and performing reference -// frame and state management. -class AcceleratedVideoDecoder { - public: - AcceleratedVideoDecoder() {} - virtual ~AcceleratedVideoDecoder() {} - - virtual void SetStream(const uint8_t* ptr, size_t size) = 0; - - // Have the decoder flush its state and trigger output of all previously - // decoded surfaces. Return false on failure. - virtual bool Flush() WARN_UNUSED_RESULT = 0; - - // Stop (pause) decoding, discarding all remaining inputs and outputs, - // but do not flush decoder state, so that playback can be resumed later, - // possibly from a different location. - // To be called during decoding. - virtual void Reset() = 0; - - enum DecodeResult { - kDecodeError, // Error while decoding. - // TODO(posciak): unsupported streams are currently treated as error - // in decoding; in future it could perhaps be possible to fall back - // to software decoding instead. - // kStreamError, // Error in stream. - kAllocateNewSurfaces, // Need a new set of surfaces to be allocated. - kRanOutOfStreamData, // Need more stream data to proceed. - kRanOutOfSurfaces, // Waiting for the client to free up output surfaces. - kNeedContextUpdate, // Waiting for the client to update decoding context - // with data acquired from the accelerator. - }; - - // Try to decode more of the stream, returning decoded frames asynchronously. - // Return when more stream is needed, when we run out of free surfaces, when - // we need a new set of them, or when an error occurs. - virtual DecodeResult Decode() WARN_UNUSED_RESULT = 0; - - // Return dimensions/required number of output surfaces that client should - // be ready to provide for the decoder to function properly. - // To be used after Decode() returns kAllocateNewSurfaces. - virtual Size GetPicSize() const = 0; - virtual size_t GetRequiredNumOfPictures() const = 0; - - private: - DISALLOW_COPY_AND_ASSIGN(AcceleratedVideoDecoder); -}; - -} // namespace media - -#endif // ACCELERATED_VIDEO_DECODER_H_ diff --git a/accel/bit_reader.cc b/accel/bit_reader.cc deleted file mode 100644 index 95e7634..0000000 --- a/accel/bit_reader.cc +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "bit_reader.h" - -namespace media { - -BitReader::BitReader(const uint8_t* data, int size) - : initial_size_(size), - data_(data), - bytes_left_(size), - bit_reader_core_(this) { - DCHECK(data != NULL); - DCHECK_GE(size, 0); -} - -BitReader::~BitReader() = default; - -bool BitReader::ReadString(int num_bits, std::string* str) { - DCHECK_EQ(num_bits % 8, 0); - DCHECK_GT(num_bits, 0); - DCHECK(str); - int num_bytes = num_bits / 8; - str->resize(num_bytes); - char* ptr = &str->front(); - while (num_bytes--) { - if (!ReadBits(8, ptr++)) - return false; - } - return true; -} - -int BitReader::GetBytes(int max_nbytes, const uint8_t** out) { - DCHECK_GE(max_nbytes, 0); - DCHECK(out); - - int nbytes = max_nbytes; - if (nbytes > bytes_left_) - nbytes = bytes_left_; - - *out = data_; - data_ += nbytes; - bytes_left_ -= nbytes; - return nbytes; -} - -} // namespace media diff --git a/accel/bit_reader.h b/accel/bit_reader.h deleted file mode 100644 index dfc2b0b..0000000 --- a/accel/bit_reader.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 43ddd7a - -#ifndef BIT_READER_H_ -#define BIT_READER_H_ - -#include <stdint.h> -#include <string> - -#include "base/compiler_specific.h" -#include "base/macros.h" -#include "bit_reader_core.h" - -namespace media { - -class BitReader : private BitReaderCore::ByteStreamProvider { - public: - // Initialize the reader to start reading at |data|, |size| being size - // of |data| in bytes. - BitReader(const uint8_t* data, int size); - ~BitReader() override; - - template<typename T> bool ReadBits(int num_bits, T* out) { - return bit_reader_core_.ReadBits(num_bits, out); - } - - bool ReadFlag(bool* flag) { - return bit_reader_core_.ReadFlag(flag); - } - - // Read |num_bits| of binary data into |str|. |num_bits| must be a positive - // multiple of 8. This is not efficient for extracting large strings. - // If false is returned, |str| may not be valid. - bool ReadString(int num_bits, std::string* str); - - bool SkipBits(int num_bits) { - return bit_reader_core_.SkipBits(num_bits); - } - - int bits_available() const { - return initial_size_ * 8 - bits_read(); - } - - int bits_read() const { - return bit_reader_core_.bits_read(); - } - - private: - // BitReaderCore::ByteStreamProvider implementation. - int GetBytes(int max_n, const uint8_t** out) override; - - // Total number of bytes that was initially passed to BitReader. - const int initial_size_; - - // Pointer to the next unread byte in the stream. - const uint8_t* data_; - - // Bytes left in the stream. - int bytes_left_; - - BitReaderCore bit_reader_core_; - - DISALLOW_COPY_AND_ASSIGN(BitReader); -}; - -} // namespace media - -#endif // BIT_READER_H_ diff --git a/accel/bit_reader_core.cc b/accel/bit_reader_core.cc deleted file mode 100644 index 92b3211..0000000 --- a/accel/bit_reader_core.cc +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "bit_reader_core.h" - -#include <stdint.h> - -#include "base/sys_byteorder.h" - -namespace { -const int kRegWidthInBits = sizeof(uint64_t) * 8; -} - -namespace media { - -BitReaderCore::ByteStreamProvider::ByteStreamProvider() = default; - -BitReaderCore::ByteStreamProvider::~ByteStreamProvider() = default; - -BitReaderCore::BitReaderCore(ByteStreamProvider* byte_stream_provider) - : byte_stream_provider_(byte_stream_provider), - bits_read_(0), - nbits_(0), - reg_(0), - nbits_next_(0), - reg_next_(0) { -} - -BitReaderCore::~BitReaderCore() = default; - -bool BitReaderCore::ReadFlag(bool* flag) { - if (nbits_ == 0 && !Refill(1)) - return false; - - *flag = (reg_ & (UINT64_C(1) << (kRegWidthInBits - 1))) != 0; - reg_ <<= 1; - nbits_--; - bits_read_++; - return true; -} - -int BitReaderCore::PeekBitsMsbAligned(int num_bits, uint64_t* out) { - // Try to have at least |num_bits| in the bit register. - if (nbits_ < num_bits) - Refill(num_bits); - - *out = reg_; - return nbits_; -} - -bool BitReaderCore::SkipBitsSmall(int num_bits) { - DCHECK_GE(num_bits, 0); - uint64_t dummy; - while (num_bits >= kRegWidthInBits) { - if (!ReadBitsInternal(kRegWidthInBits, &dummy)) - return false; - num_bits -= kRegWidthInBits; - } - return ReadBitsInternal(num_bits, &dummy); -} - -bool BitReaderCore::SkipBits(int num_bits) { - DCHECK_GE(num_bits, 0); - - const int remaining_bits = nbits_ + nbits_next_; - if (remaining_bits >= num_bits) - return SkipBitsSmall(num_bits); - - // Skip first the remaining available bits. - num_bits -= remaining_bits; - bits_read_ += remaining_bits; - nbits_ = 0; - reg_ = 0; - nbits_next_ = 0; - reg_next_ = 0; - - // Next, skip an integer number of bytes. - const int nbytes = num_bits / 8; - if (nbytes > 0) { - const uint8_t* byte_stream_window; - const int window_size = - byte_stream_provider_->GetBytes(nbytes, &byte_stream_window); - DCHECK_GE(window_size, 0); - DCHECK_LE(window_size, nbytes); - if (window_size < nbytes) { - // Note that some bytes were consumed. - bits_read_ += 8 * window_size; - return false; - } - num_bits -= 8 * nbytes; - bits_read_ += 8 * nbytes; - } - - // Skip the remaining bits. - return SkipBitsSmall(num_bits); -} - -int BitReaderCore::bits_read() const { - return bits_read_; -} - -bool BitReaderCore::ReadBitsInternal(int num_bits, uint64_t* out) { - DCHECK_GE(num_bits, 0); - - if (num_bits == 0) { - *out = 0; - return true; - } - - if (num_bits > nbits_ && !Refill(num_bits)) { - // Any subsequent ReadBits should fail: - // empty the current bit register for that purpose. - nbits_ = 0; - reg_ = 0; - return false; - } - - bits_read_ += num_bits; - - if (num_bits == kRegWidthInBits) { - // Special case needed since for example for a 64 bit integer "a" - // "a << 64" is not defined by the C/C++ standard. - *out = reg_; - reg_ = 0; - nbits_ = 0; - return true; - } - - *out = reg_ >> (kRegWidthInBits - num_bits); - reg_ <<= num_bits; - nbits_ -= num_bits; - return true; -} - -bool BitReaderCore::Refill(int min_nbits) { - DCHECK_LE(min_nbits, kRegWidthInBits); - - // Transfer from the next to the current register. - RefillCurrentRegister(); - if (min_nbits <= nbits_) - return true; - DCHECK_EQ(nbits_next_, 0); - DCHECK_EQ(reg_next_, 0u); - - // Max number of bytes to refill. - int max_nbytes = sizeof(reg_next_); - - // Refill. - const uint8_t* byte_stream_window; - int window_size = - byte_stream_provider_->GetBytes(max_nbytes, &byte_stream_window); - DCHECK_GE(window_size, 0); - DCHECK_LE(window_size, max_nbytes); - if (window_size == 0) - return false; - - reg_next_ = 0; - memcpy(®_next_, byte_stream_window, window_size); - reg_next_ = base::NetToHost64(reg_next_); - nbits_next_ = window_size * 8; - - // Transfer from the next to the current register. - RefillCurrentRegister(); - - return (nbits_ >= min_nbits); -} - -void BitReaderCore::RefillCurrentRegister() { - // No refill possible if the destination register is full - // or the source register is empty. - if (nbits_ == kRegWidthInBits || nbits_next_ == 0) - return; - - reg_ |= (reg_next_ >> nbits_); - - int free_nbits = kRegWidthInBits - nbits_; - if (free_nbits >= nbits_next_) { - nbits_ += nbits_next_; - reg_next_ = 0; - nbits_next_ = 0; - return; - } - - nbits_ += free_nbits; - reg_next_ <<= free_nbits; - nbits_next_ -= free_nbits; -} - -} // namespace media diff --git a/accel/bit_reader_core.h b/accel/bit_reader_core.h deleted file mode 100644 index 62a21e2..0000000 --- a/accel/bit_reader_core.h +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 1323b9c - -#ifndef BIT_READER_CORE_H_ -#define BIT_READER_CORE_H_ - -#include <stdint.h> - -#include "base/logging.h" -#include "base/macros.h" - -namespace media { - -class BitReaderCore { - public: - class ByteStreamProvider { - public: - ByteStreamProvider(); - virtual ~ByteStreamProvider(); - - // Consume at most the following |max_n| bytes of the stream - // and return the number n of bytes actually consumed. - // Set |*array| to point to a memory buffer containing those n bytes. - // Note: |*array| must be valid until the next call to GetBytes - // but there is no guarantee it is valid after. - virtual int GetBytes(int max_n, const uint8_t** array) = 0; - }; - - // Lifetime of |byte_stream_provider| must be longer than BitReaderCore. - explicit BitReaderCore(ByteStreamProvider* byte_stream_provider); - ~BitReaderCore(); - - // Read one bit from the stream and return it as a boolean in |*out|. - // Remark: we do not use the template version for reading a bool - // since it generates some optimization warnings during compilation - // on Windows platforms. - bool ReadBits(int num_bits, bool* out) { - DCHECK_EQ(num_bits, 1); - return ReadFlag(out); - } - - // Read |num_bits| next bits from stream and return in |*out|, first bit - // from the stream starting at |num_bits| position in |*out|, - // bits of |*out| whose position is strictly greater than |num_bits| - // are all set to zero. - // Notes: - // - |num_bits| cannot be larger than the bits the type can hold. - // - From the above description, passing a signed type in |T| does not - // mean the first bit read from the stream gives the sign of the value. - // Return false if the given number of bits cannot be read (not enough - // bits in the stream), true otherwise. When return false, the stream will - // enter a state where further ReadBits/SkipBits operations will always - // return false unless |num_bits| is 0. The type |T| has to be a primitive - // integer type. - template<typename T> bool ReadBits(int num_bits, T* out) { - DCHECK_LE(num_bits, static_cast<int>(sizeof(T) * 8)); - uint64_t temp; - bool ret = ReadBitsInternal(num_bits, &temp); - *out = static_cast<T>(temp); - return ret; - } - - // Read one bit from the stream and return it as a boolean in |*flag|. - bool ReadFlag(bool* flag); - - // Retrieve some bits without actually consuming them. - // Bits returned in |*out| are shifted so the most significant bit contains - // the next bit that can be read from the stream. - // Return the number of bits actually written in |out|. - // Note: |num_bits| is just a suggestion of how many bits the caller - // wish to get in |*out| and must be less than 64: - // - The number of bits returned can be more than |num_bits|. - // - However, it will be strictly less than |num_bits| - // if and only if there are not enough bits left in the stream. - int PeekBitsMsbAligned(int num_bits, uint64_t* out); - - // Skip |num_bits| next bits from stream. Return false if the given number of - // bits cannot be skipped (not enough bits in the stream), true otherwise. - // When return false, the stream will enter a state where further - // ReadBits/ReadFlag/SkipBits operations - // will always return false unless |num_bits| is 0. - bool SkipBits(int num_bits); - - // Returns the number of bits read so far. - int bits_read() const; - - private: - // This function can skip any number of bits but is more efficient - // for small numbers. Return false if the given number of bits cannot be - // skipped (not enough bits in the stream), true otherwise. - bool SkipBitsSmall(int num_bits); - - // Help function used by ReadBits to avoid inlining the bit reading logic. - bool ReadBitsInternal(int num_bits, uint64_t* out); - - // Refill bit registers to have at least |min_nbits| bits available. - // Return true if the mininimum bit count condition is met after the refill. - bool Refill(int min_nbits); - - // Refill the current bit register from the next bit register. - void RefillCurrentRegister(); - - ByteStreamProvider* const byte_stream_provider_; - - // Number of bits read so far. - int bits_read_; - - // Number of bits in |reg_| that have not been consumed yet. - // Note: bits are consumed from MSB to LSB. - int nbits_; - uint64_t reg_; - - // Number of bits in |reg_next_| that have not been consumed yet. - // Note: bits are consumed from MSB to LSB. - int nbits_next_; - uint64_t reg_next_; - - DISALLOW_COPY_AND_ASSIGN(BitReaderCore); -}; - -} // namespace media - -#endif // BIT_READER_CORE_H_ diff --git a/accel/bitstream_buffer.cc b/accel/bitstream_buffer.cc deleted file mode 100644 index 36b8d06..0000000 --- a/accel/bitstream_buffer.cc +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "bitstream_buffer.h" - -namespace media { - -BitstreamBuffer::BitstreamBuffer() - : BitstreamBuffer(-1, base::SharedMemoryHandle(), 0) {} - -BitstreamBuffer::BitstreamBuffer(int32_t id, - base::SharedMemoryHandle handle, - size_t size, - off_t offset, - base::TimeDelta presentation_timestamp) - : id_(id), - handle_(handle), - size_(size), - offset_(offset), - presentation_timestamp_(presentation_timestamp) {} - -BitstreamBuffer::BitstreamBuffer(const BitstreamBuffer& other) = default; - -BitstreamBuffer::~BitstreamBuffer() = default; - -} // namespace media diff --git a/accel/bitstream_buffer.h b/accel/bitstream_buffer.h deleted file mode 100644 index 3a267a0..0000000 --- a/accel/bitstream_buffer.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 39a7f93 - -#ifndef MEDIA_BASE_BITSTREAM_BUFFER_H_ -#define MEDIA_BASE_BITSTREAM_BUFFER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include "base/macros.h" -#include "base/memory/shared_memory.h" -#include "base/time/time.h" - -namespace media { - -// Indicates an invalid or missing timestamp. -constexpr base::TimeDelta kNoTimestamp = - base::TimeDelta::FromMicroseconds(std::numeric_limits<int64_t>::min()); - -// Class for passing bitstream buffers around. Does not take ownership of the -// data. This is the media-namespace equivalent of PP_VideoBitstreamBuffer_Dev. -class BitstreamBuffer { - public: - BitstreamBuffer(); - - // Constructs a new BitstreamBuffer. The content of the bitstream is located - // at |offset| bytes away from the start of the shared memory and the payload - // is |size| bytes. When not provided, the default value for |offset| is 0. - // |presentation_timestamp| is when the decoded frame should be displayed. - // When not provided, |presentation_timestamp| will be - // |media::kNoTimestamp|. - BitstreamBuffer(int32_t id, - base::SharedMemoryHandle handle, - size_t size, - off_t offset = 0, - base::TimeDelta presentation_timestamp = kNoTimestamp); - - BitstreamBuffer(const BitstreamBuffer& other); - - ~BitstreamBuffer(); - - int32_t id() const { return id_; } - base::SharedMemoryHandle handle() const { return handle_; } - - // The number of bytes of the actual bitstream data. It is the size of the - // content instead of the whole shared memory. - size_t size() const { return size_; } - - // The offset to the start of actual bitstream data in the shared memory. - off_t offset() const { return offset_; } - - // The timestamp is only valid if it's not equal to |media::kNoTimestamp|. - base::TimeDelta presentation_timestamp() const { - return presentation_timestamp_; - } - - void set_handle(const base::SharedMemoryHandle& handle) { handle_ = handle; } - - private: - int32_t id_; - base::SharedMemoryHandle handle_; - size_t size_; - off_t offset_; - - // This is only set when necessary. For example, AndroidVideoDecodeAccelerator - // needs the timestamp because the underlying decoder may require it to - // determine the output order. - base::TimeDelta presentation_timestamp_; - - // Allow compiler-generated copy & assign constructors. -}; - -} // namespace media - -#endif // MEDIA_BASE_BITSTREAM_BUFFER_H_ diff --git a/accel/color_plane_layout.cc b/accel/color_plane_layout.cc deleted file mode 100644 index 3a19798..0000000 --- a/accel/color_plane_layout.cc +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 57ec858cddff - -#include "color_plane_layout.h" - -namespace media { - -ColorPlaneLayout::ColorPlaneLayout() = default; - -ColorPlaneLayout::ColorPlaneLayout(int32_t stride, size_t offset, size_t size) - : stride(stride), offset(offset), size(size) {} - -ColorPlaneLayout::~ColorPlaneLayout() = default; - -bool ColorPlaneLayout::operator==(const ColorPlaneLayout& rhs) const { - return stride == rhs.stride && offset == rhs.offset && size == rhs.size; -} - -bool ColorPlaneLayout::operator!=(const ColorPlaneLayout& rhs) const { - return !(*this == rhs); -} - -std::ostream& operator<<(std::ostream& ostream, const ColorPlaneLayout& plane) { - ostream << "(" << plane.stride << ", " << plane.offset << ", " << plane.size - << ")"; - return ostream; -} - -} // namespace media diff --git a/accel/color_plane_layout.h b/accel/color_plane_layout.h deleted file mode 100644 index 829c0ec..0000000 --- a/accel/color_plane_layout.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 57ec858cddff - -#ifndef MEDIA_COLOR_PLANE_LAYOUT_H_ -#define MEDIA_COLOR_PLANE_LAYOUT_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <ostream> - -namespace media { - -// Encapsulates a color plane's memory layout: (stride, offset, size) -// stride: in bytes of a plane. Note that stride can be negative if the image -// layout is bottom-up. -// offset: in bytes of a plane, which stands for the offset of a start point of -// a color plane from a buffer FD. -// size: in bytes of a plane. This |size| bytes data must contain all the data -// a decoder will access (e.g. visible area and padding). -struct ColorPlaneLayout { - ColorPlaneLayout(); - ColorPlaneLayout(int32_t stride, size_t offset, size_t size); - ~ColorPlaneLayout(); - - bool operator==(const ColorPlaneLayout& rhs) const; - bool operator!=(const ColorPlaneLayout& rhs) const; - - int32_t stride = 0; - size_t offset = 0; - size_t size = 0; -}; - -// Outputs ColorPlaneLayout to stream. -std::ostream& operator<<(std::ostream& ostream, const ColorPlaneLayout& plane); - -} // namespace media - -#endif // MEDIA_COLOR_PLANE_LAYOUT_H_ diff --git a/accel/fourcc.cc b/accel/fourcc.cc deleted file mode 100644 index d340ddf..0000000 --- a/accel/fourcc.cc +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 27c98933749f - -#include "fourcc.h" - -#include <linux/videodev2.h> - -#include "base/logging.h" -#include "base/strings/stringprintf.h" - -#include "macros.h" - -namespace media { - -Fourcc::Fourcc(Fourcc::Value fourcc) : value_(fourcc) {} -Fourcc::~Fourcc() = default; -Fourcc& Fourcc::operator=(const Fourcc& other) = default; - -// static -base::Optional<Fourcc> Fourcc::FromUint32(uint32_t fourcc) { - switch (fourcc) { - case AR24: - case AB24: - case XR24: - case XB24: - case RGB4: - case YU12: - case YV12: - case YM12: - case YM21: - case YUYV: - case NV12: - case NV21: - case NM12: - case NM21: - case YM16: - case MT21: - case MM21: - return Fourcc(static_cast<Value>(fourcc)); - } - DVLOGF(3) << "Unmapped fourcc: " << FourccToString(fourcc); - return base::nullopt; -} - -// static -base::Optional<Fourcc> Fourcc::FromVideoPixelFormat( - VideoPixelFormat pixel_format, - bool single_planar) { - if (single_planar) { - switch (pixel_format) { - case PIXEL_FORMAT_ARGB: - return Fourcc(AR24); - case PIXEL_FORMAT_ABGR: - return Fourcc(AB24); - case PIXEL_FORMAT_XRGB: - return Fourcc(XR24); - case PIXEL_FORMAT_XBGR: - return Fourcc(XB24); - case PIXEL_FORMAT_BGRA: - return Fourcc(RGB4); - case PIXEL_FORMAT_I420: - return Fourcc(YU12); - case PIXEL_FORMAT_YV12: - return Fourcc(YV12); - case PIXEL_FORMAT_YUY2: - return Fourcc(YUYV); - case PIXEL_FORMAT_NV12: - return Fourcc(NV12); - case PIXEL_FORMAT_NV21: - return Fourcc(NV21); - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_P016LE: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - case PIXEL_FORMAT_UNKNOWN: - break; - } - } else { - switch (pixel_format) { - case PIXEL_FORMAT_I420: - return Fourcc(YM12); - case PIXEL_FORMAT_YV12: - return Fourcc(YM21); - case PIXEL_FORMAT_NV12: - return Fourcc(NM12); - case PIXEL_FORMAT_I422: - return Fourcc(YM16); - case PIXEL_FORMAT_NV21: - return Fourcc(NM21); - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_P016LE: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - case PIXEL_FORMAT_BGRA: - case PIXEL_FORMAT_UNKNOWN: - break; - } - } - DVLOGF(3) << "Unmapped " << VideoPixelFormatToString(pixel_format) << " for " - << (single_planar ? "single-planar" : "multi-planar"); - return base::nullopt; -} - -VideoPixelFormat Fourcc::ToVideoPixelFormat() const { - switch (value_) { - case AR24: - return PIXEL_FORMAT_ARGB; - case AB24: - return PIXEL_FORMAT_ABGR; - case XR24: - return PIXEL_FORMAT_XRGB; - case XB24: - return PIXEL_FORMAT_XBGR; - case RGB4: - return PIXEL_FORMAT_BGRA; - case YU12: - case YM12: - return PIXEL_FORMAT_I420; - case YV12: - case YM21: - return PIXEL_FORMAT_YV12; - case YUYV: - return PIXEL_FORMAT_YUY2; - case NV12: - case NM12: - return PIXEL_FORMAT_NV12; - case NV21: - case NM21: - return PIXEL_FORMAT_NV21; - case YM16: - return PIXEL_FORMAT_I422; - // V4L2_PIX_FMT_MT21C is only used for MT8173 hardware video decoder output - // and should be converted by MT8173 image processor for compositor to - // render. Since it is an intermediate format for video decoder, - // VideoPixelFormat shall not have its mapping. However, we need to create a - // VideoFrameLayout for the format to process the intermediate frame. Hence - // we map V4L2_PIX_FMT_MT21C to PIXEL_FORMAT_NV12 as their layout are the - // same. - case MT21: - // V4L2_PIX_FMT_MM21 is used for MT8183 hardware video decoder. It is - // similar to V4L2_PIX_FMT_MT21C but is not compressed ; thus it can also - // be mapped to PIXEL_FORMAT_NV12. - case MM21: - return PIXEL_FORMAT_NV12; - } - NOTREACHED() << "Unmapped Fourcc: " << ToString(); - return PIXEL_FORMAT_UNKNOWN; -} - -// static -base::Optional<Fourcc> Fourcc::FromV4L2PixFmt(uint32_t v4l2_pix_fmt) { - // We can do that because we adopt the same internal definition of Fourcc as - // V4L2. - return FromUint32(v4l2_pix_fmt); -} - -uint32_t Fourcc::ToV4L2PixFmt() const { - // Note that we can do that because we adopt the same internal definition of - // Fourcc as V4L2. - return static_cast<uint32_t>(value_); -} - -base::Optional<Fourcc> Fourcc::ToSinglePlanar() const { - switch (value_) { - case AR24: - case AB24: - case XR24: - case XB24: - case RGB4: - case YU12: - case YV12: - case YUYV: - case NV12: - case NV21: - return Fourcc(value_); - case YM12: - return Fourcc(YU12); - case YM21: - return Fourcc(YV12); - case NM12: - return Fourcc(NV12); - case NM21: - return Fourcc(NV21); - case YM16: - case MT21: - case MM21: - return base::nullopt; - } -} - -bool operator!=(const Fourcc& lhs, const Fourcc& rhs) { - return !(lhs == rhs); -} - -bool Fourcc::IsMultiPlanar() const { - switch (value_) { - case AR24: - case AB24: - case XR24: - case XB24: - case RGB4: - case YU12: - case YV12: - case YUYV: - case NV12: - case NV21: - return false; - case YM12: - case YM21: - case NM12: - case NM21: - case YM16: - case MT21: - case MM21: - return true; - } -} - -std::string Fourcc::ToString() const { - return FourccToString(static_cast<uint32_t>(value_)); -} - -static_assert(Fourcc::AR24 == V4L2_PIX_FMT_ABGR32, "Mismatch Fourcc"); -#ifdef V4L2_PIX_FMT_RGBA32 -// V4L2_PIX_FMT_RGBA32 is defined since v5.2 -static_assert(Fourcc::AB24 == V4L2_PIX_FMT_RGBA32, "Mismatch Fourcc"); -#endif // V4L2_PIX_FMT_RGBA32 -static_assert(Fourcc::XR24 == V4L2_PIX_FMT_XBGR32, "Mismatch Fourcc"); -#ifdef V4L2_PIX_FMT_RGBX32 -// V4L2_PIX_FMT_RGBX32 is defined since v5.2 -static_assert(Fourcc::XB24 == V4L2_PIX_FMT_RGBX32, "Mismatch Fourcc"); -#endif // V4L2_PIX_FMT_RGBX32 -static_assert(Fourcc::RGB4 == V4L2_PIX_FMT_RGB32, "Mismatch Fourcc"); -static_assert(Fourcc::YU12 == V4L2_PIX_FMT_YUV420, "Mismatch Fourcc"); -static_assert(Fourcc::YV12 == V4L2_PIX_FMT_YVU420, "Mismatch Fourcc"); -static_assert(Fourcc::YM12 == V4L2_PIX_FMT_YUV420M, "Mismatch Fourcc"); -static_assert(Fourcc::YM21 == V4L2_PIX_FMT_YVU420M, "Mismatch Fourcc"); -static_assert(Fourcc::YUYV == V4L2_PIX_FMT_YUYV, "Mismatch Fourcc"); -static_assert(Fourcc::NV12 == V4L2_PIX_FMT_NV12, "Mismatch Fourcc"); -static_assert(Fourcc::NV21 == V4L2_PIX_FMT_NV21, "Mismatch Fourcc"); -static_assert(Fourcc::NM12 == V4L2_PIX_FMT_NV12M, "Mismatch Fourcc"); -static_assert(Fourcc::NM21 == V4L2_PIX_FMT_NV21M, "Mismatch Fourcc"); -static_assert(Fourcc::YM16 == V4L2_PIX_FMT_YUV422M, "Mismatch Fourcc"); -static_assert(Fourcc::MT21 == V4L2_PIX_FMT_MT21C, "Mismatch Fourcc"); -#ifdef V4L2_PIX_FMT_MM21 -// V4L2_PIX_FMT_MM21 is not yet upstreamed. -static_assert(Fourcc::MM21 == V4L2_PIX_FMT_MM21, "Mismatch Fourcc"); -#endif // V4L2_PIX_FMT_MM21 -} // namespace media diff --git a/accel/fourcc.h b/accel/fourcc.h deleted file mode 100644 index 408b845..0000000 --- a/accel/fourcc.h +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 27c98933749f - -#ifndef FOURCC_H_ -#define FOURCC_H_ - -#include <stdint.h> -#include <string> - -#include "base/optional.h" -#include "video_pixel_format.h" - -namespace media { - -// Composes a Fourcc value. -constexpr uint32_t ComposeFourcc(char a, char b, char c, char d) { - return static_cast<uint32_t>(a) | (static_cast<uint32_t>(b) << 8) | - (static_cast<uint32_t>(c) << 16) | (static_cast<uint32_t>(d) << 24); -} - -// Fourcc enum holder and converters. -// Usage: -// Fourcc f1(Fourcc::AR24); -// EXPECT_EQ("AR24", f1.ToString()); -// Fourcc f2 = Fourcc::FromVideoPixelFormat(PIXEL_FORMAT_ARGB); -// EXPECT_EQ(f2, f1); -class Fourcc { - public: - enum Value : uint32_t { - // RGB formats. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-rgb.html - // Maps to PIXEL_FORMAT_ARGB, V4L2_PIX_FMT_ABGR32, VA_FOURCC_BGRA. - // 32bpp BGRA (byte-order), 1 plane. - AR24 = ComposeFourcc('A', 'R', '2', '4'), - - // Maps to PIXEL_FORMAT_ABGR, V4L2_PIX_FMT_RGBA32, VA_FOURCC_RGBA. - // 32bpp RGBA (byte-order), 1 plane - AB24 = ComposeFourcc('A', 'B', '2', '4'), - - // Maps to PIXEL_FORMAT_XRGB, V4L2_PIX_FMT_XBGR32, VA_FOURCC_BGRX. - // 32bpp BGRX (byte-order), 1 plane. - XR24 = ComposeFourcc('X', 'R', '2', '4'), - - // Maps to PIXEL_FORMAT_XBGR, V4L2_PIX_FMT_RGBX32, VA_FOURCC_RGBX. - // 32bpp RGBX (byte-order), 1 plane. - XB24 = ComposeFourcc('X', 'B', '2', '4'), - - // Maps to PIXEL_FORMAT_BGRA, V4L2_PIX_FMT_RGB32, VA_FOURCC_ARGB. - // 32bpp ARGB (byte-order), 1 plane. - // Note that V4L2_PIX_FMT_RGB32("RGB4") is deprecated and replaced by - // V4L2_PIX_FMT_ARGB32("BA24"), however, some board relies on the fourcc - // mapping so we keep it as-is. - RGB4 = ComposeFourcc('R', 'G', 'B', '4'), - - // YUV420 single-planar formats. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv420.html - // Maps to PIXEL_FORMAT_I420, V4L2_PIX_FMT_YUV420, VA_FOURCC_I420. - // 12bpp YUV planar 1x1 Y, 2x2 UV samples. - YU12 = ComposeFourcc('Y', 'U', '1', '2'), - // Maps to PIXEL_FORMAT_YV12, V4L2_PIX_FMT_YVU420, VA_FOURCC_YV12. - // 12bpp YVU planar 1x1 Y, 2x2 VU samples. - YV12 = ComposeFourcc('Y', 'V', '1', '2'), - - // YUV420 multi-planar format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv420m.htm - // Maps to PIXEL_FORMAT_I420, V4L2_PIX_FMT_YUV420M. - YM12 = ComposeFourcc('Y', 'M', '1', '2'), - // Maps to PIXEL_FORMAT_YV12, V4L2_PIX_FMT_YVU420M. - YM21 = ComposeFourcc('Y', 'M', '2', '1'), - - // YUYV format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuyv.html - // Maps to PIXEL_FORMAT_YUY2, V4L2_PIX_FMT_YUYV, VA_FOURCC_YUY2. - // 16bpp YUV planar (YUV 4:2:2), YUYV (byte-order), 1 plane. - YUYV = ComposeFourcc('Y', 'U', 'Y', 'V'), - - // NV12 single-planar format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-nv12.html - // Maps to PIXEL_FORMAT_NV12, V4L2_PIX_FMT_NV12, VA_FOURCC_NV12. - // 12bpp with Y plane followed by a 2x2 interleaved UV plane. - NV12 = ComposeFourcc('N', 'V', '1', '2'), - // Maps to PIXEL_FORMAT_NV21, V4L2_PIX_FMT_NV21, VA_FOURCC_NV21. - // 12bpp with Y plane followed by a 2x2 interleaved VU plane. - NV21 = ComposeFourcc('N', 'V', '2', '1'), - - // NV12 multi-planar format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-nv12m.html - // Maps to PIXEL_FORMAT_NV12, V4L2_PIX_FMT_NV12M, - NM12 = ComposeFourcc('N', 'M', '1', '2'), - // Maps to PIXEL_FORMAT_NV21, V4L2_PIX_FMT_NV21M. - NM21 = ComposeFourcc('N', 'M', '2', '1'), - - // YUV422 multi-planar format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv422m.html - // Maps to PIXEL_FORMAT_I422, V4L2_PIX_FMT_YUV422M - // 16bpp YUV planar 1x1 Y, 2x1 UV samples. - YM16 = ComposeFourcc('Y', 'M', '1', '6'), - - // V4L2 proprietary format. - // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-reserved.html - // Maps to V4L2_PIX_FMT_MT21C. - // It is used for MT8173 hardware video decoder output and should be - // converted by MT8173 image processor for compositor to render. - MT21 = ComposeFourcc('M', 'T', '2', '1'), - // Maps to V4L2_PIX_FMT_MM21. - // It is used for MT8183 hardware video decoder. - MM21 = ComposeFourcc('M', 'M', '2', '1'), - }; - - explicit Fourcc(Fourcc::Value fourcc); - Fourcc& operator=(const Fourcc& fourcc); - ~Fourcc(); - - bool operator==(const Fourcc& rhs) const { return value_ == rhs.value_; } - - // Factory methods: - - // Builds a Fourcc from a given fourcc code. This will return a valid - // Fourcc if the argument is part of the |Value| enum, or nullopt otherwise. - static base::Optional<Fourcc> FromUint32(uint32_t fourcc); - - // Converts a VideoPixelFormat to Fourcc. - // Returns nullopt for invalid input. - // Note that a VideoPixelFormat may have two Fourcc counterparts. Caller has - // to specify if it is for single-planar or multi-planar format. - static base::Optional<Fourcc> FromVideoPixelFormat( - VideoPixelFormat pixel_format, - bool single_planar = true); - // Converts a V4L2PixFmt to Fourcc. - // Returns nullopt for invalid input. - static base::Optional<Fourcc> FromV4L2PixFmt(uint32_t v4l2_pix_fmt); - - // Value getters: - // Returns the VideoPixelFormat counterpart of the value. - // Returns PIXEL_FORMAT_UNKNOWN if no mapping is found. - VideoPixelFormat ToVideoPixelFormat() const; - // Returns the V4L2PixFmt counterpart of the value. - // Returns 0 if no mapping is found. - uint32_t ToV4L2PixFmt() const; - - // Returns the single-planar Fourcc of the value. If value is a single-planar, - // returns the same Fourcc. Returns nullopt if no mapping is found. - base::Optional<Fourcc> ToSinglePlanar() const; - - // Returns whether |value_| is multi planar format. - bool IsMultiPlanar() const; - - // Outputs human readable fourcc string, e.g. "NV12". - std::string ToString() const; - - private: - Value value_; -}; - -bool operator!=(const Fourcc& lhs, const Fourcc& rhs); - -} // namespace media - -#endif // FOURCC_H_ diff --git a/accel/generic_v4l2_device.cc b/accel/generic_v4l2_device.cc deleted file mode 100644 index 8dea028..0000000 --- a/accel/generic_v4l2_device.cc +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 8c9190713ed9 - -#include "generic_v4l2_device.h" - -#include <errno.h> -#include <fcntl.h> -#include <linux/videodev2.h> -#include <poll.h> -#include <string.h> -#include <sys/eventfd.h> -#include <sys/ioctl.h> -#include <sys/mman.h> - -#include <algorithm> -#include <memory> - -#include "base/files/scoped_file.h" -#include "base/posix/eintr_wrapper.h" -#include "base/stl_util.h" -#include "base/strings/stringprintf.h" - -#include "macros.h" - -namespace media { - -GenericV4L2Device::GenericV4L2Device() {} - -GenericV4L2Device::~GenericV4L2Device() { - CloseDevice(); -} - -int GenericV4L2Device::Ioctl(int request, void* arg) { - DCHECK(device_fd_.is_valid()); - return HANDLE_EINTR(ioctl(device_fd_.get(), request, arg)); -} - -bool GenericV4L2Device::Poll(bool poll_device, bool* event_pending) { - struct pollfd pollfds[2]; - nfds_t nfds; - int pollfd = -1; - - pollfds[0].fd = device_poll_interrupt_fd_.get(); - pollfds[0].events = POLLIN | POLLERR; - nfds = 1; - - if (poll_device) { - DVLOGF(5) << "adding device fd to poll() set"; - pollfds[nfds].fd = device_fd_.get(); - pollfds[nfds].events = POLLIN | POLLOUT | POLLERR | POLLPRI; - pollfd = nfds; - nfds++; - } - - if (HANDLE_EINTR(poll(pollfds, nfds, -1)) == -1) { - VPLOGF(1) << "poll() failed"; - return false; - } - *event_pending = (pollfd != -1 && pollfds[pollfd].revents & POLLPRI); - return true; -} - -void* GenericV4L2Device::Mmap(void* addr, - unsigned int len, - int prot, - int flags, - unsigned int offset) { - DCHECK(device_fd_.is_valid()); - return mmap(addr, len, prot, flags, device_fd_.get(), offset); -} - -void GenericV4L2Device::Munmap(void* addr, unsigned int len) { - munmap(addr, len); -} - -bool GenericV4L2Device::SetDevicePollInterrupt() { - DVLOGF(4); - - const uint64_t buf = 1; - if (HANDLE_EINTR(write(device_poll_interrupt_fd_.get(), &buf, sizeof(buf))) == - -1) { - VPLOGF(1) << "write() failed"; - return false; - } - return true; -} - -bool GenericV4L2Device::ClearDevicePollInterrupt() { - DVLOGF(5); - - uint64_t buf; - if (HANDLE_EINTR(read(device_poll_interrupt_fd_.get(), &buf, sizeof(buf))) == - -1) { - if (errno == EAGAIN) { - // No interrupt flag set, and we're reading nonblocking. Not an error. - return true; - } else { - VPLOGF(1) << "read() failed"; - return false; - } - } - return true; -} - -bool GenericV4L2Device::Initialize() { - DVLOGF(3); - static bool v4l2_functions_initialized = PostSandboxInitialization(); - if (!v4l2_functions_initialized) { - VLOGF(1) << "Failed to initialize LIBV4L2 libs"; - return false; - } - - return true; -} - -bool GenericV4L2Device::Open(Type type, uint32_t v4l2_pixfmt) { - DVLOGF(3); - std::string path = GetDevicePathFor(type, v4l2_pixfmt); - - if (path.empty()) { - VLOGF(1) << "No devices supporting " << FourccToString(v4l2_pixfmt) - << " for type: " << static_cast<int>(type); - return false; - } - - if (!OpenDevicePath(path, type)) { - VLOGF(1) << "Failed opening " << path; - return false; - } - - device_poll_interrupt_fd_.reset(eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)); - if (!device_poll_interrupt_fd_.is_valid()) { - VLOGF(1) << "Failed creating a poll interrupt fd"; - return false; - } - - return true; -} - -std::vector<base::ScopedFD> GenericV4L2Device::GetDmabufsForV4L2Buffer( - int index, - size_t num_planes, - enum v4l2_buf_type buf_type) { - DVLOGF(3); - DCHECK(V4L2_TYPE_IS_MULTIPLANAR(buf_type)); - - std::vector<base::ScopedFD> dmabuf_fds; - for (size_t i = 0; i < num_planes; ++i) { - struct v4l2_exportbuffer expbuf; - memset(&expbuf, 0, sizeof(expbuf)); - expbuf.type = buf_type; - expbuf.index = index; - expbuf.plane = i; - expbuf.flags = O_CLOEXEC; - if (Ioctl(VIDIOC_EXPBUF, &expbuf) != 0) { - dmabuf_fds.clear(); - break; - } - - dmabuf_fds.push_back(base::ScopedFD(expbuf.fd)); - } - - return dmabuf_fds; -} - -std::vector<uint32_t> GenericV4L2Device::PreferredInputFormat(Type type) { - if (type == Type::kEncoder) - return {V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_NV12}; - - return {}; -} - -std::vector<uint32_t> GenericV4L2Device::GetSupportedImageProcessorPixelformats( - v4l2_buf_type buf_type) { - std::vector<uint32_t> supported_pixelformats; - - Type type = Type::kImageProcessor; - const auto& devices = GetDevicesForType(type); - for (const auto& device : devices) { - if (!OpenDevicePath(device.first, type)) { - VLOGF(1) << "Failed opening " << device.first; - continue; - } - - std::vector<uint32_t> pixelformats = - EnumerateSupportedPixelformats(buf_type); - - supported_pixelformats.insert(supported_pixelformats.end(), - pixelformats.begin(), pixelformats.end()); - CloseDevice(); - } - - return supported_pixelformats; -} - -VideoDecodeAccelerator::SupportedProfiles -GenericV4L2Device::GetSupportedDecodeProfiles(const size_t num_formats, - const uint32_t pixelformats[]) { - VideoDecodeAccelerator::SupportedProfiles supported_profiles; - - Type type = Type::kDecoder; - const auto& devices = GetDevicesForType(type); - for (const auto& device : devices) { - if (!OpenDevicePath(device.first, type)) { - VLOGF(1) << "Failed opening " << device.first; - continue; - } - - const auto& profiles = - EnumerateSupportedDecodeProfiles(num_formats, pixelformats); - supported_profiles.insert(supported_profiles.end(), profiles.begin(), - profiles.end()); - CloseDevice(); - } - - return supported_profiles; -} - -VideoEncodeAccelerator::SupportedProfiles -GenericV4L2Device::GetSupportedEncodeProfiles() { - VideoEncodeAccelerator::SupportedProfiles supported_profiles; - - Type type = Type::kEncoder; - const auto& devices = GetDevicesForType(type); - for (const auto& device : devices) { - if (!OpenDevicePath(device.first, type)) { - VLOGF(1) << "Failed opening " << device.first; - continue; - } - - const auto& profiles = EnumerateSupportedEncodeProfiles(); - supported_profiles.insert(supported_profiles.end(), profiles.begin(), - profiles.end()); - CloseDevice(); - } - - return supported_profiles; -} - -bool GenericV4L2Device::IsImageProcessingSupported() { - const auto& devices = GetDevicesForType(Type::kImageProcessor); - return !devices.empty(); -} - -bool GenericV4L2Device::IsJpegDecodingSupported() { - const auto& devices = GetDevicesForType(Type::kJpegDecoder); - return !devices.empty(); -} - -bool GenericV4L2Device::IsJpegEncodingSupported() { - const auto& devices = GetDevicesForType(Type::kJpegEncoder); - return !devices.empty(); -} - -bool GenericV4L2Device::OpenDevicePath(const std::string& path, Type type) { - DCHECK(!device_fd_.is_valid()); - - device_fd_.reset( - HANDLE_EINTR(open(path.c_str(), O_RDWR | O_NONBLOCK | O_CLOEXEC))); - if (!device_fd_.is_valid()) - return false; - - return true; -} - -void GenericV4L2Device::CloseDevice() { - DVLOGF(3); - device_fd_.reset(); -} - -// static -bool GenericV4L2Device::PostSandboxInitialization() { - return true; -} - -void GenericV4L2Device::EnumerateDevicesForType(Type type) { - // video input/output devices are registered as /dev/videoX in V4L2. - static const std::string kVideoDevicePattern = "/dev/video"; - - std::string device_pattern; - v4l2_buf_type buf_type; - switch (type) { - case Type::kDecoder: - device_pattern = kVideoDevicePattern; - buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - break; - case Type::kEncoder: - device_pattern = kVideoDevicePattern; - buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - break; - default: - LOG(ERROR) << "Only decoder and encoder types are supported!!"; - return; - } - - std::vector<std::string> candidate_paths; - - // TODO(posciak): Remove this legacy unnumbered device once - // all platforms are updated to use numbered devices. - candidate_paths.push_back(device_pattern); - - // We are sandboxed, so we can't query directory contents to check which - // devices are actually available. Try to open the first 10; if not present, - // we will just fail to open immediately. - for (int i = 0; i < 10; ++i) { - candidate_paths.push_back( - base::StringPrintf("%s%d", device_pattern.c_str(), i)); - } - - Devices devices; - for (const auto& path : candidate_paths) { - if (!OpenDevicePath(path, type)) - continue; - - const auto& supported_pixelformats = - EnumerateSupportedPixelformats(buf_type); - if (!supported_pixelformats.empty()) { - DVLOGF(3) << "Found device: " << path; - devices.push_back(std::make_pair(path, supported_pixelformats)); - } - - CloseDevice(); - } - - DCHECK_EQ(devices_by_type_.count(type), 0u); - devices_by_type_[type] = devices; -} - -const GenericV4L2Device::Devices& GenericV4L2Device::GetDevicesForType( - Type type) { - if (devices_by_type_.count(type) == 0) - EnumerateDevicesForType(type); - - DCHECK_NE(devices_by_type_.count(type), 0u); - return devices_by_type_[type]; -} - -std::string GenericV4L2Device::GetDevicePathFor(Type type, uint32_t pixfmt) { - const Devices& devices = GetDevicesForType(type); - - for (const auto& device : devices) { - if (std::find(device.second.begin(), device.second.end(), pixfmt) != - device.second.end()) - return device.first; - } - - return std::string(); -} - -} // namespace media diff --git a/accel/generic_v4l2_device.h b/accel/generic_v4l2_device.h deleted file mode 100644 index 9567b43..0000000 --- a/accel/generic_v4l2_device.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains the implementation of GenericV4L2Device used on -// platforms, which provide generic V4L2 video codec devices. -// Note: ported from Chromium commit head: 9a075af92855 -// Note: removed all references to 'USE_LIBV4L2'. -// Note: removed GL-related functionality. - -#ifndef V4L2_GENERIC_V4L2_DEVICE_H_ -#define V4L2_GENERIC_V4L2_DEVICE_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <map> -#include <vector> - -#include "base/files/scoped_file.h" -#include "base/macros.h" -#include "v4l2_device.h" - -namespace media { - -class GenericV4L2Device : public V4L2Device { - public: - GenericV4L2Device(); - - // V4L2Device implementation. - bool Open(Type type, uint32_t v4l2_pixfmt) override; - int Ioctl(int request, void* arg) override; - bool Poll(bool poll_device, bool* event_pending) override; - bool SetDevicePollInterrupt() override; - bool ClearDevicePollInterrupt() override; - void* Mmap(void* addr, - unsigned int len, - int prot, - int flags, - unsigned int offset) override; - void Munmap(void* addr, unsigned int len) override; - - std::vector<base::ScopedFD> GetDmabufsForV4L2Buffer( - int index, - size_t num_planes, - enum v4l2_buf_type buf_type) override; - - std::vector<uint32_t> PreferredInputFormat(Type type) override; - - std::vector<uint32_t> GetSupportedImageProcessorPixelformats( - v4l2_buf_type buf_type) override; - - VideoDecodeAccelerator::SupportedProfiles GetSupportedDecodeProfiles( - const size_t num_formats, - const uint32_t pixelformats[]) override; - - VideoEncodeAccelerator::SupportedProfiles GetSupportedEncodeProfiles() - override; - - bool IsImageProcessingSupported() override; - - bool IsJpegDecodingSupported() override; - bool IsJpegEncodingSupported() override; - - protected: - ~GenericV4L2Device() override; - - bool Initialize() override; - - private: - // Vector of video device node paths and corresponding pixelformats supported - // by each device node. - using Devices = std::vector<std::pair<std::string, std::vector<uint32_t>>>; - - // Open device node for |path| as a device of |type|. - bool OpenDevicePath(const std::string& path, Type type); - - // Close the currently open device. - void CloseDevice(); - - // Enumerate all V4L2 devices on the system for |type| and store the results - // under devices_by_type_[type]. - void EnumerateDevicesForType(V4L2Device::Type type); - - // Return device information for all devices of |type| available in the - // system. Enumerates and queries devices on first run and caches the results - // for subsequent calls. - const Devices& GetDevicesForType(V4L2Device::Type type); - - // Return device node path for device of |type| supporting |pixfmt|, or - // an empty string if the given combination is not supported by the system. - std::string GetDevicePathFor(V4L2Device::Type type, uint32_t pixfmt); - - // Stores information for all devices available on the system - // for each device Type. - std::map<V4L2Device::Type, Devices> devices_by_type_; - - // The actual device fd. - base::ScopedFD device_fd_; - - // eventfd fd to signal device poll thread when its poll() should be - // interrupted. - base::ScopedFD device_poll_interrupt_fd_; - - DISALLOW_COPY_AND_ASSIGN(GenericV4L2Device); - - // Lazily initialize static data after sandbox is enabled. Return false on - // init failure. - static bool PostSandboxInitialization(); -}; -} // namespace media - -#endif // V4L2_GENERIC_V4L2_DEVICE_H_ diff --git a/accel/h264_bit_reader.cc b/accel/h264_bit_reader.cc deleted file mode 100644 index 6713655..0000000 --- a/accel/h264_bit_reader.cc +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "base/logging.h" -#include "h264_bit_reader.h" - -namespace media { - -H264BitReader::H264BitReader() - : data_(NULL), - bytes_left_(0), - curr_byte_(0), - num_remaining_bits_in_curr_byte_(0), - prev_two_bytes_(0), - emulation_prevention_bytes_(0) {} - -H264BitReader::~H264BitReader() = default; - -bool H264BitReader::Initialize(const uint8_t* data, off_t size) { - DCHECK(data); - - if (size < 1) - return false; - - data_ = data; - bytes_left_ = size; - num_remaining_bits_in_curr_byte_ = 0; - // Initially set to 0xffff to accept all initial two-byte sequences. - prev_two_bytes_ = 0xffff; - emulation_prevention_bytes_ = 0; - - return true; -} - -bool H264BitReader::UpdateCurrByte() { - if (bytes_left_ < 1) - return false; - - // Emulation prevention three-byte detection. - // If a sequence of 0x000003 is found, skip (ignore) the last byte (0x03). - if (*data_ == 0x03 && (prev_two_bytes_ & 0xffff) == 0) { - // Detected 0x000003, skip last byte. - ++data_; - --bytes_left_; - ++emulation_prevention_bytes_; - // Need another full three bytes before we can detect the sequence again. - prev_two_bytes_ = 0xffff; - - if (bytes_left_ < 1) - return false; - } - - // Load a new byte and advance pointers. - curr_byte_ = *data_++ & 0xff; - --bytes_left_; - num_remaining_bits_in_curr_byte_ = 8; - - prev_two_bytes_ = ((prev_two_bytes_ & 0xff) << 8) | curr_byte_; - - return true; -} - -// Read |num_bits| (1 to 31 inclusive) from the stream and return them -// in |out|, with first bit in the stream as MSB in |out| at position -// (|num_bits| - 1). -bool H264BitReader::ReadBits(int num_bits, int* out) { - int bits_left = num_bits; - *out = 0; - DCHECK(num_bits <= 31); - - while (num_remaining_bits_in_curr_byte_ < bits_left) { - // Take all that's left in current byte, shift to make space for the rest. - *out |= (curr_byte_ << (bits_left - num_remaining_bits_in_curr_byte_)); - bits_left -= num_remaining_bits_in_curr_byte_; - - if (!UpdateCurrByte()) - return false; - } - - *out |= (curr_byte_ >> (num_remaining_bits_in_curr_byte_ - bits_left)); - *out &= ((1u << num_bits) - 1u); - num_remaining_bits_in_curr_byte_ -= bits_left; - - return true; -} - -off_t H264BitReader::NumBitsLeft() { - return (num_remaining_bits_in_curr_byte_ + bytes_left_ * 8); -} - -bool H264BitReader::HasMoreRBSPData() { - // Make sure we have more bits, if we are at 0 bits in current byte and - // updating current byte fails, we don't have more data anyway. - if (num_remaining_bits_in_curr_byte_ == 0 && !UpdateCurrByte()) - return false; - - // If there is no more RBSP data, then |curr_byte_| contains the stop bit and - // zero padding. Check to see if there is other data instead. - // (We don't actually check for the stop bit itself, instead treating the - // invalid case of all trailing zeros identically). - if ((curr_byte_ & ((1 << (num_remaining_bits_in_curr_byte_ - 1)) - 1)) != 0) - return true; - - // While the spec disallows it (7.4.1: "The last byte of the NAL unit shall - // not be equal to 0x00"), some streams have trailing null bytes anyway. We - // don't handle emulation prevention sequences because HasMoreRBSPData() is - // not used when parsing slices (where cabac_zero_word elements are legal). - for (off_t i = 0; i < bytes_left_; i++) { - if (data_[i] != 0) - return true; - } - - bytes_left_ = 0; - return false; -} - -size_t H264BitReader::NumEmulationPreventionBytesRead() { - return emulation_prevention_bytes_; -} - -} // namespace media diff --git a/accel/h264_bit_reader.h b/accel/h264_bit_reader.h deleted file mode 100644 index aa162ce..0000000 --- a/accel/h264_bit_reader.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of an H264 Annex-B video stream parser. -// Note: ported from Chromium commit head: 77be7ae - -#ifndef H264_BIT_READER_H_ -#define H264_BIT_READER_H_ - -#include <stddef.h> -#include <stdint.h> -#include <sys/types.h> - -#include "base/macros.h" - -namespace media { - -// A class to provide bit-granularity reading of H.264 streams. -// This is not a generic bit reader class, as it takes into account -// H.264 stream-specific constraints, such as skipping emulation-prevention -// bytes and stop bits. See spec for more details. -class H264BitReader { - public: - H264BitReader(); - ~H264BitReader(); - - // Initialize the reader to start reading at |data|, |size| being size - // of |data| in bytes. - // Return false on insufficient size of stream.. - // TODO(posciak,fischman): consider replacing Initialize() with - // heap-allocating and creating bit readers on demand instead. - bool Initialize(const uint8_t* data, off_t size); - - // Read |num_bits| next bits from stream and return in |*out|, first bit - // from the stream starting at |num_bits| position in |*out|. - // |num_bits| may be 1-32, inclusive. - // Return false if the given number of bits cannot be read (not enough - // bits in the stream), true otherwise. - bool ReadBits(int num_bits, int* out); - - // Return the number of bits left in the stream. - off_t NumBitsLeft(); - - // See the definition of more_rbsp_data() in spec. - bool HasMoreRBSPData(); - - // Return the number of emulation prevention bytes already read. - size_t NumEmulationPreventionBytesRead(); - - private: - // Advance to the next byte, loading it into curr_byte_. - // Return false on end of stream. - bool UpdateCurrByte(); - - // Pointer to the next unread (not in curr_byte_) byte in the stream. - const uint8_t* data_; - - // Bytes left in the stream (without the curr_byte_). - off_t bytes_left_; - - // Contents of the current byte; first unread bit starting at position - // 8 - num_remaining_bits_in_curr_byte_ from MSB. - int curr_byte_; - - // Number of bits remaining in curr_byte_ - int num_remaining_bits_in_curr_byte_; - - // Used in emulation prevention three byte detection (see spec). - // Initially set to 0xffff to accept all initial two-byte sequences. - int prev_two_bytes_; - - // Number of emulation preventation bytes (0x000003) we met. - size_t emulation_prevention_bytes_; - - DISALLOW_COPY_AND_ASSIGN(H264BitReader); -}; - -} // namespace media - -#endif // H264_BIT_READER_H_ diff --git a/accel/h264_decoder.cc b/accel/h264_decoder.cc deleted file mode 100644 index abaaac5..0000000 --- a/accel/h264_decoder.cc +++ /dev/null @@ -1,1459 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: c3bd64c - -#include <algorithm> -#include <limits> - -#include "base/bind.h" -#include "base/bind_helpers.h" -#include "base/callback_helpers.h" -#include "base/macros.h" -#include "base/numerics/safe_conversions.h" -#include "base/optional.h" -#include "base/stl_util.h" -#include "h264_decoder.h" - -namespace media { - -H264Decoder::H264Accelerator::H264Accelerator() = default; - -H264Decoder::H264Accelerator::~H264Accelerator() = default; - -H264Decoder::H264Decoder(H264Accelerator* accelerator) - : state_(kNeedStreamMetadata), - max_frame_num_(0), - max_pic_num_(0), - max_long_term_frame_idx_(0), - max_num_reorder_frames_(0), - accelerator_(accelerator) { - DCHECK(accelerator_); - Reset(); -} - -H264Decoder::~H264Decoder() = default; - -void H264Decoder::Reset() { - curr_pic_ = nullptr; - curr_nalu_ = nullptr; - curr_slice_hdr_ = nullptr; - curr_sps_id_ = -1; - curr_pps_id_ = -1; - - prev_frame_num_ = -1; - prev_ref_frame_num_ = -1; - prev_frame_num_offset_ = -1; - prev_has_memmgmnt5_ = false; - - prev_ref_has_memmgmnt5_ = false; - prev_ref_top_field_order_cnt_ = -1; - prev_ref_pic_order_cnt_msb_ = -1; - prev_ref_pic_order_cnt_lsb_ = -1; - prev_ref_field_ = H264Picture::FIELD_NONE; - - ref_pic_list_p0_.clear(); - ref_pic_list_b0_.clear(); - ref_pic_list_b1_.clear(); - dpb_.Clear(); - parser_.Reset(); - accelerator_->Reset(); - last_output_poc_ = std::numeric_limits<int>::min(); - - // If we are in kDecoding, we can resume without processing an SPS. - if (state_ == kDecoding) - state_ = kAfterReset; -} - -void H264Decoder::PrepareRefPicLists(const H264SliceHeader* slice_hdr) { - ConstructReferencePicListsP(slice_hdr); - ConstructReferencePicListsB(slice_hdr); -} - -bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr, - H264Picture::Vector* ref_pic_list0, - H264Picture::Vector* ref_pic_list1) { - ref_pic_list0->clear(); - ref_pic_list1->clear(); - - // Fill reference picture lists for B and S/SP slices. - if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) { - *ref_pic_list0 = ref_pic_list_p0_; - return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0); - } else if (slice_hdr->IsBSlice()) { - *ref_pic_list0 = ref_pic_list_b0_; - *ref_pic_list1 = ref_pic_list_b1_; - return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) && - ModifyReferencePicList(slice_hdr, 1, ref_pic_list1); - } - - return true; -} - -bool H264Decoder::DecodePicture() { - DCHECK(curr_pic_.get()); - - DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt; - return accelerator_->SubmitDecode(curr_pic_); -} - -bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic, - int frame_num) { - pic->nonexisting = true; - pic->nal_ref_idc = 1; - pic->frame_num = pic->pic_num = frame_num; - pic->adaptive_ref_pic_marking_mode_flag = false; - pic->ref = true; - pic->long_term_reference_flag = false; - pic->field = H264Picture::FIELD_NONE; - - return CalculatePicOrderCounts(pic); -} - -bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) { - DCHECK(curr_pic_.get()); - - curr_pic_->idr = slice_hdr->idr_pic_flag; - if (curr_pic_->idr) - curr_pic_->idr_pic_id = slice_hdr->idr_pic_id; - - if (slice_hdr->field_pic_flag) { - curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM - : H264Picture::FIELD_TOP; - } else { - curr_pic_->field = H264Picture::FIELD_NONE; - } - - if (curr_pic_->field != H264Picture::FIELD_NONE) { - DVLOG(1) << "Interlaced video not supported."; - return false; - } - - curr_pic_->nal_ref_idc = slice_hdr->nal_ref_idc; - curr_pic_->ref = slice_hdr->nal_ref_idc != 0; - // This assumes non-interlaced stream. - curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num; - - DCHECK_NE(curr_sps_id_, -1); - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - curr_pic_->pic_order_cnt_type = sps->pic_order_cnt_type; - switch (curr_pic_->pic_order_cnt_type) { - case 0: - curr_pic_->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; - curr_pic_->delta_pic_order_cnt_bottom = - slice_hdr->delta_pic_order_cnt_bottom; - break; - - case 1: - curr_pic_->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt0; - curr_pic_->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt1; - break; - - case 2: - break; - - default: - NOTREACHED(); - return false; - } - - if (!CalculatePicOrderCounts(curr_pic_)) - return false; - - curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag; - curr_pic_->adaptive_ref_pic_marking_mode_flag = - slice_hdr->adaptive_ref_pic_marking_mode_flag; - - // If the slice header indicates we will have to perform reference marking - // process after this picture is decoded, store required data for that - // purpose. - if (slice_hdr->adaptive_ref_pic_marking_mode_flag) { - static_assert(sizeof(curr_pic_->ref_pic_marking) == - sizeof(slice_hdr->ref_pic_marking), - "Array sizes of ref pic marking do not match."); - memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking, - sizeof(curr_pic_->ref_pic_marking)); - } - - curr_pic_->visible_rect = visible_rect_; - - return true; -} - -bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) { - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - switch (pic->pic_order_cnt_type) { - case 0: { - // See spec 8.2.1.1. - int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; - - if (pic->idr) { - prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; - } else { - if (prev_ref_has_memmgmnt5_) { - if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) { - prev_pic_order_cnt_msb = 0; - prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_; - } else { - prev_pic_order_cnt_msb = 0; - prev_pic_order_cnt_lsb = 0; - } - } else { - prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_; - prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_; - } - } - - int max_pic_order_cnt_lsb = - 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); - DCHECK_NE(max_pic_order_cnt_lsb, 0); - if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && - (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >= - max_pic_order_cnt_lsb / 2)) { - pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb; - } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) && - (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb > - max_pic_order_cnt_lsb / 2)) { - pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb; - } else { - pic->pic_order_cnt_msb = prev_pic_order_cnt_msb; - } - - if (pic->field != H264Picture::FIELD_BOTTOM) { - pic->top_field_order_cnt = - pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; - } - - if (pic->field != H264Picture::FIELD_TOP) { - if (pic->field == H264Picture::FIELD_NONE) { - pic->bottom_field_order_cnt = - pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom; - } else { - pic->bottom_field_order_cnt = - pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; - } - } - break; - } - - case 1: { - // See spec 8.2.1.2. - if (prev_has_memmgmnt5_) - prev_frame_num_offset_ = 0; - - if (pic->idr) - pic->frame_num_offset = 0; - else if (prev_frame_num_ > pic->frame_num) - pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; - else - pic->frame_num_offset = prev_frame_num_offset_; - - int abs_frame_num = 0; - if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) - abs_frame_num = pic->frame_num_offset + pic->frame_num; - else - abs_frame_num = 0; - - if (pic->nal_ref_idc == 0 && abs_frame_num > 0) - --abs_frame_num; - - int expected_pic_order_cnt = 0; - if (abs_frame_num > 0) { - if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { - DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle " - << "in stream"; - return false; - } - - int pic_order_cnt_cycle_cnt = - (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle; - int frame_num_in_pic_order_cnt_cycle = - (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle; - - expected_pic_order_cnt = pic_order_cnt_cycle_cnt * - sps->expected_delta_per_pic_order_cnt_cycle; - // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser - for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) - expected_pic_order_cnt += sps->offset_for_ref_frame[i]; - } - - if (!pic->nal_ref_idc) - expected_pic_order_cnt += sps->offset_for_non_ref_pic; - - if (pic->field == H264Picture::FIELD_NONE) { - pic->top_field_order_cnt = - expected_pic_order_cnt + pic->delta_pic_order_cnt0; - pic->bottom_field_order_cnt = pic->top_field_order_cnt + - sps->offset_for_top_to_bottom_field + - pic->delta_pic_order_cnt1; - } else if (pic->field != H264Picture::FIELD_BOTTOM) { - pic->top_field_order_cnt = - expected_pic_order_cnt + pic->delta_pic_order_cnt0; - } else { - pic->bottom_field_order_cnt = expected_pic_order_cnt + - sps->offset_for_top_to_bottom_field + - pic->delta_pic_order_cnt0; - } - break; - } - - case 2: { - // See spec 8.2.1.3. - if (prev_has_memmgmnt5_) - prev_frame_num_offset_ = 0; - - if (pic->idr) - pic->frame_num_offset = 0; - else if (prev_frame_num_ > pic->frame_num) - pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; - else - pic->frame_num_offset = prev_frame_num_offset_; - - int temp_pic_order_cnt; - if (pic->idr) { - temp_pic_order_cnt = 0; - } else if (!pic->nal_ref_idc) { - temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1; - } else { - temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num); - } - - if (pic->field == H264Picture::FIELD_NONE) { - pic->top_field_order_cnt = temp_pic_order_cnt; - pic->bottom_field_order_cnt = temp_pic_order_cnt; - } else if (pic->field == H264Picture::FIELD_BOTTOM) { - pic->bottom_field_order_cnt = temp_pic_order_cnt; - } else { - pic->top_field_order_cnt = temp_pic_order_cnt; - } - break; - } - - default: - DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type; - return false; - } - - switch (pic->field) { - case H264Picture::FIELD_NONE: - pic->pic_order_cnt = - std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt); - break; - case H264Picture::FIELD_TOP: - pic->pic_order_cnt = pic->top_field_order_cnt; - break; - case H264Picture::FIELD_BOTTOM: - pic->pic_order_cnt = pic->bottom_field_order_cnt; - break; - } - - return true; -} - -void H264Decoder::UpdatePicNums(int frame_num) { - for (auto& pic : dpb_) { - if (!pic->ref) - continue; - - // 8.2.4.1. Assumes non-interlaced stream. - DCHECK_EQ(pic->field, H264Picture::FIELD_NONE); - if (pic->long_term) { - pic->long_term_pic_num = pic->long_term_frame_idx; - } else { - if (pic->frame_num > frame_num) - pic->frame_num_wrap = pic->frame_num - max_frame_num_; - else - pic->frame_num_wrap = pic->frame_num; - - pic->pic_num = pic->frame_num_wrap; - } - } -} - -struct PicNumDescCompare { - bool operator()(const scoped_refptr<H264Picture>& a, - const scoped_refptr<H264Picture>& b) const { - return a->pic_num > b->pic_num; - } -}; - -struct LongTermPicNumAscCompare { - bool operator()(const scoped_refptr<H264Picture>& a, - const scoped_refptr<H264Picture>& b) const { - return a->long_term_pic_num < b->long_term_pic_num; - } -}; - -void H264Decoder::ConstructReferencePicListsP( - const H264SliceHeader* slice_hdr) { - // RefPicList0 (8.2.4.2.1) [[1] [2]], where: - // [1] shortterm ref pics sorted by descending pic_num, - // [2] longterm ref pics by ascending long_term_pic_num. - ref_pic_list_p0_.clear(); - - // First get the short ref pics... - dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_); - size_t num_short_refs = ref_pic_list_p0_.size(); - - // and sort them to get [1]. - std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(), - PicNumDescCompare()); - - // Now get long term pics and sort them by long_term_pic_num to get [2]. - dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_); - std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(), - LongTermPicNumAscCompare()); -} - -struct POCAscCompare { - bool operator()(const scoped_refptr<H264Picture>& a, - const scoped_refptr<H264Picture>& b) const { - return a->pic_order_cnt < b->pic_order_cnt; - } -}; - -struct POCDescCompare { - bool operator()(const scoped_refptr<H264Picture>& a, - const scoped_refptr<H264Picture>& b) const { - return a->pic_order_cnt > b->pic_order_cnt; - } -}; - -void H264Decoder::ConstructReferencePicListsB( - const H264SliceHeader* slice_hdr) { - // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where: - // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC, - // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC, - // [3] longterm ref pics by ascending long_term_pic_num. - ref_pic_list_b0_.clear(); - ref_pic_list_b1_.clear(); - dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_); - size_t num_short_refs = ref_pic_list_b0_.size(); - - // First sort ascending, this will put [1] in right place and finish [2]. - std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare()); - - // Find first with POC > curr_pic's POC to get first element in [2]... - H264Picture::Vector::iterator iter; - iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), - curr_pic_.get(), POCAscCompare()); - - // and sort [1] descending, thus finishing sequence [1] [2]. - std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare()); - - // Now add [3] and sort by ascending long_term_pic_num. - dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_); - std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(), - LongTermPicNumAscCompare()); - - // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where: - // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, - // [2] shortterm ref pics with POC < curr_pic's POC by descending POC, - // [3] longterm ref pics by ascending long_term_pic_num. - - dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); - num_short_refs = ref_pic_list_b1_.size(); - - // First sort by descending POC. - std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare()); - - // Find first with POC < curr_pic's POC to get first element in [2]... - iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), - curr_pic_.get(), POCDescCompare()); - - // and sort [1] ascending. - std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare()); - - // Now add [3] and sort by ascending long_term_pic_num - dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); - std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(), - LongTermPicNumAscCompare()); - - // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3) - if (ref_pic_list_b1_.size() > 1 && - std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), - ref_pic_list_b1_.begin())) - std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]); -} - -// See 8.2.4 -int H264Decoder::PicNumF(const scoped_refptr<H264Picture>& pic) { - if (!pic) - return -1; - - if (!pic->long_term) - return pic->pic_num; - else - return max_pic_num_; -} - -// See 8.2.4 -int H264Decoder::LongTermPicNumF(const scoped_refptr<H264Picture>& pic) { - if (pic->ref && pic->long_term) - return pic->long_term_pic_num; - else - return 2 * (max_long_term_frame_idx_ + 1); -} - -// Shift elements on the |v| starting from |from| to |to|, inclusive, -// one position to the right and insert pic at |from|. -static void ShiftRightAndInsert(H264Picture::Vector* v, - int from, - int to, - const scoped_refptr<H264Picture>& pic) { - // Security checks, do not disable in Debug mode. - CHECK(from <= to); - CHECK(to <= std::numeric_limits<int>::max() - 2); - // Additional checks. Debug mode ok. - DCHECK(v); - DCHECK(pic); - DCHECK((to + 1 == static_cast<int>(v->size())) || - (to + 2 == static_cast<int>(v->size()))); - - v->resize(to + 2); - - for (int i = to + 1; i > from; --i) - (*v)[i] = (*v)[i - 1]; - - (*v)[from] = pic; -} - -bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr, - int list, - H264Picture::Vector* ref_pic_listx) { - bool ref_pic_list_modification_flag_lX; - int num_ref_idx_lX_active_minus1; - const H264ModificationOfPicNum* list_mod; - - // This can process either ref_pic_list0 or ref_pic_list1, depending on - // the list argument. Set up pointers to proper list to be processed here. - if (list == 0) { - ref_pic_list_modification_flag_lX = - slice_hdr->ref_pic_list_modification_flag_l0; - num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1; - list_mod = slice_hdr->ref_list_l0_modifications; - } else { - ref_pic_list_modification_flag_lX = - slice_hdr->ref_pic_list_modification_flag_l1; - num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1; - list_mod = slice_hdr->ref_list_l1_modifications; - } - - // Resize the list to the size requested in the slice header. - // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to - // indicate there should be more ref pics on list than we constructed. - // Those superfluous ones should be treated as non-reference and will be - // initialized to nullptr, which must be handled by clients. - DCHECK_GE(num_ref_idx_lX_active_minus1, 0); - ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); - - if (!ref_pic_list_modification_flag_lX) - return true; - - // Spec 8.2.4.3: - // Reorder pictures on the list in a way specified in the stream. - int pic_num_lx_pred = curr_pic_->pic_num; - int ref_idx_lx = 0; - int pic_num_lx_no_wrap; - int pic_num_lx; - bool done = false; - scoped_refptr<H264Picture> pic; - for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) { - switch (list_mod->modification_of_pic_nums_idc) { - case 0: - case 1: - // Modify short reference picture position. - if (list_mod->modification_of_pic_nums_idc == 0) { - // Subtract given value from predicted PicNum. - pic_num_lx_no_wrap = - pic_num_lx_pred - - (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); - // Wrap around max_pic_num_ if it becomes < 0 as result - // of subtraction. - if (pic_num_lx_no_wrap < 0) - pic_num_lx_no_wrap += max_pic_num_; - } else { - // Add given value to predicted PicNum. - pic_num_lx_no_wrap = - pic_num_lx_pred + - (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); - // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result - // of the addition. - if (pic_num_lx_no_wrap >= max_pic_num_) - pic_num_lx_no_wrap -= max_pic_num_; - } - - // For use in next iteration. - pic_num_lx_pred = pic_num_lx_no_wrap; - - if (pic_num_lx_no_wrap > curr_pic_->pic_num) - pic_num_lx = pic_num_lx_no_wrap - max_pic_num_; - else - pic_num_lx = pic_num_lx_no_wrap; - - DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, - H264SliceHeader::kRefListModSize); - pic = dpb_.GetShortRefPicByPicNum(pic_num_lx); - if (!pic) { - DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx; - return false; - } - ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, - num_ref_idx_lX_active_minus1, pic); - ref_idx_lx++; - - for (int src = ref_idx_lx, dst = ref_idx_lx; - src <= num_ref_idx_lX_active_minus1 + 1; ++src) { - if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx) - (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; - } - break; - - case 2: - // Modify long term reference picture position. - DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, - H264SliceHeader::kRefListModSize); - pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num); - if (!pic) { - DVLOG(1) << "Malformed stream, no pic num " - << list_mod->long_term_pic_num; - return false; - } - ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, - num_ref_idx_lX_active_minus1, pic); - ref_idx_lx++; - - for (int src = ref_idx_lx, dst = ref_idx_lx; - src <= num_ref_idx_lX_active_minus1 + 1; ++src) { - if (LongTermPicNumF((*ref_pic_listx)[src]) != - static_cast<int>(list_mod->long_term_pic_num)) - (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; - } - break; - - case 3: - // End of modification list. - done = true; - break; - - default: - // May be recoverable. - DVLOG(1) << "Invalid modification_of_pic_nums_idc=" - << list_mod->modification_of_pic_nums_idc - << " in position " << i; - break; - } - - ++list_mod; - } - - // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is - // temporarily made one element longer than the required final list. - // Resize the list back to its required size. - ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); - - return true; -} - -void H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) { - DCHECK(!pic->outputted); - pic->outputted = true; - - if (pic->nonexisting) { - DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num; - return; - } - - DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_) - << "Outputting out of order, likely a broken stream: " - << last_output_poc_ << " -> " << pic->pic_order_cnt; - last_output_poc_ = pic->pic_order_cnt; - - DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt; - accelerator_->OutputPicture(pic); -} - -void H264Decoder::ClearDPB() { - // Clear DPB contents, marking the pictures as unused first. - dpb_.Clear(); - last_output_poc_ = std::numeric_limits<int>::min(); -} - -bool H264Decoder::OutputAllRemainingPics() { - // Output all pictures that are waiting to be outputted. - FinishPrevFrameIfPresent(); - H264Picture::Vector to_output; - dpb_.GetNotOutputtedPicsAppending(&to_output); - // Sort them by ascending POC to output in order. - std::sort(to_output.begin(), to_output.end(), POCAscCompare()); - - for (auto& pic : to_output) - OutputPic(pic); - - return true; -} - -bool H264Decoder::Flush() { - DVLOG(2) << "Decoder flush"; - - if (!OutputAllRemainingPics()) - return false; - - ClearDPB(); - DVLOG(2) << "Decoder flush finished"; - return true; -} - -bool H264Decoder::StartNewFrame(const H264SliceHeader* slice_hdr) { - // TODO posciak: add handling of max_num_ref_frames per spec. - CHECK(curr_pic_.get()); - DCHECK(slice_hdr); - - curr_pps_id_ = slice_hdr->pic_parameter_set_id; - const H264PPS* pps = parser_.GetPPS(curr_pps_id_); - if (!pps) - return false; - - curr_sps_id_ = pps->seq_parameter_set_id; - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4); - int frame_num = slice_hdr->frame_num; - if (slice_hdr->idr_pic_flag) - prev_ref_frame_num_ = 0; - - // 7.4.3 - if (frame_num != prev_ref_frame_num_ && - frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) { - if (!HandleFrameNumGap(frame_num)) - return false; - } - - if (!InitCurrPicture(slice_hdr)) - return false; - - UpdatePicNums(frame_num); - PrepareRefPicLists(slice_hdr); - - if (!accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_, - ref_pic_list_b0_, ref_pic_list_b1_, - curr_pic_.get())) - return false; - - return true; -} - -bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) { - // 8.2.5.4 - for (size_t i = 0; i < arraysize(pic->ref_pic_marking); ++i) { - // Code below does not support interlaced stream (per-field pictures). - H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i]; - scoped_refptr<H264Picture> to_mark; - int pic_num_x; - - switch (ref_pic_marking->memory_mgmnt_control_operation) { - case 0: - // Normal end of operations' specification. - return true; - - case 1: - // Mark a short term reference picture as unused so it can be removed - // if outputted. - pic_num_x = - pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); - to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); - if (to_mark) { - to_mark->ref = false; - } else { - DVLOG(1) << "Invalid short ref pic num to unmark"; - return false; - } - break; - - case 2: - // Mark a long term reference picture as unused so it can be removed - // if outputted. - to_mark = dpb_.GetLongRefPicByLongTermPicNum( - ref_pic_marking->long_term_pic_num); - if (to_mark) { - to_mark->ref = false; - } else { - DVLOG(1) << "Invalid long term ref pic num to unmark"; - return false; - } - break; - - case 3: - // Mark a short term reference picture as long term reference. - pic_num_x = - pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); - to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); - if (to_mark) { - DCHECK(to_mark->ref && !to_mark->long_term); - to_mark->long_term = true; - to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; - } else { - DVLOG(1) << "Invalid short term ref pic num to mark as long ref"; - return false; - } - break; - - case 4: { - // Unmark all reference pictures with long_term_frame_idx over new max. - max_long_term_frame_idx_ = - ref_pic_marking->max_long_term_frame_idx_plus1 - 1; - H264Picture::Vector long_terms; - dpb_.GetLongTermRefPicsAppending(&long_terms); - for (size_t i = 0; i < long_terms.size(); ++i) { - scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; - DCHECK(long_term_pic->ref && long_term_pic->long_term); - // Ok to cast, max_long_term_frame_idx is much smaller than 16bit. - if (long_term_pic->long_term_frame_idx > - static_cast<int>(max_long_term_frame_idx_)) - long_term_pic->ref = false; - } - break; - } - - case 5: - // Unmark all reference pictures. - dpb_.MarkAllUnusedForRef(); - max_long_term_frame_idx_ = -1; - pic->mem_mgmt_5 = true; - break; - - case 6: { - // Replace long term reference pictures with current picture. - // First unmark if any existing with this long_term_frame_idx... - H264Picture::Vector long_terms; - dpb_.GetLongTermRefPicsAppending(&long_terms); - for (size_t i = 0; i < long_terms.size(); ++i) { - scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; - DCHECK(long_term_pic->ref && long_term_pic->long_term); - // Ok to cast, long_term_frame_idx is much smaller than 16bit. - if (long_term_pic->long_term_frame_idx == - static_cast<int>(ref_pic_marking->long_term_frame_idx)) - long_term_pic->ref = false; - } - - // and mark the current one instead. - pic->ref = true; - pic->long_term = true; - pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; - break; - } - - default: - // Would indicate a bug in parser. - NOTREACHED(); - } - } - - return true; -} - -// This method ensures that DPB does not overflow, either by removing -// reference pictures as specified in the stream, or using a sliding window -// procedure to remove the oldest one. -// It also performs marking and unmarking pictures as reference. -// See spac 8.2.5.1. -bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) { - // If the current picture is an IDR, all reference pictures are unmarked. - if (pic->idr) { - dpb_.MarkAllUnusedForRef(); - - if (pic->long_term_reference_flag) { - pic->long_term = true; - pic->long_term_frame_idx = 0; - max_long_term_frame_idx_ = 0; - } else { - pic->long_term = false; - max_long_term_frame_idx_ = -1; - } - - return true; - } - - // Not an IDR. If the stream contains instructions on how to discard pictures - // from DPB and how to mark/unmark existing reference pictures, do so. - // Otherwise, fall back to default sliding window process. - if (pic->adaptive_ref_pic_marking_mode_flag) { - DCHECK(!pic->nonexisting); - return HandleMemoryManagementOps(pic); - } else { - return SlidingWindowPictureMarking(); - } -} - -bool H264Decoder::SlidingWindowPictureMarking() { - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture. - int num_ref_pics = dpb_.CountRefPics(); - DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1)); - if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) { - // Max number of reference pics reached, need to remove one of the short - // term ones. Find smallest frame_num_wrap short reference picture and mark - // it as unused. - scoped_refptr<H264Picture> to_unmark = - dpb_.GetLowestFrameNumWrapShortRefPic(); - if (!to_unmark) { - DVLOG(1) << "Couldn't find a short ref picture to unmark"; - return false; - } - - to_unmark->ref = false; - } - - return true; -} - -bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) { - // Finish processing the picture. - // Start by storing previous picture data for later use. - if (pic->ref) { - ReferencePictureMarking(pic); - prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5; - prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt; - prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb; - prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb; - prev_ref_field_ = pic->field; - prev_ref_frame_num_ = pic->frame_num; - } - prev_frame_num_ = pic->frame_num; - prev_has_memmgmnt5_ = pic->mem_mgmt_5; - prev_frame_num_offset_ = pic->frame_num_offset; - - // Remove unused (for reference or later output) pictures from DPB, marking - // them as such. - dpb_.DeleteUnused(); - - DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num - << ", entries in DPB: " << dpb_.size(); - - // The ownership of pic will either be transferred to DPB - if the picture is - // still needed (for output and/or reference) - or we will release it - // immediately if we manage to output it here and won't have to store it for - // future reference. - - // Get all pictures that haven't been outputted yet. - H264Picture::Vector not_outputted; - dpb_.GetNotOutputtedPicsAppending(¬_outputted); - // Include the one we've just decoded. - not_outputted.push_back(pic); - - // Sort in output order. - std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare()); - - // Try to output as many pictures as we can. A picture can be output, - // if the number of decoded and not yet outputted pictures that would remain - // in DPB afterwards would at least be equal to max_num_reorder_frames. - // If the outputted picture is not a reference picture, it doesn't have - // to remain in the DPB and can be removed. - H264Picture::Vector::iterator output_candidate = not_outputted.begin(); - size_t num_remaining = not_outputted.size(); - while (num_remaining > max_num_reorder_frames_ || - // If the condition below is used, this is an invalid stream. We should - // not be forced to output beyond max_num_reorder_frames in order to - // make room in DPB to store the current picture (if we need to do so). - // However, if this happens, ignore max_num_reorder_frames and try - // to output more. This may cause out-of-order output, but is not - // fatal, and better than failing instead. - ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) { - DVLOG_IF(1, num_remaining <= max_num_reorder_frames_) - << "Invalid stream: max_num_reorder_frames not preserved"; - - OutputPic(*output_candidate); - - if (!(*output_candidate)->ref) { - // Current picture hasn't been inserted into DPB yet, so don't remove it - // if we managed to output it immediately. - int outputted_poc = (*output_candidate)->pic_order_cnt; - if (outputted_poc != pic->pic_order_cnt) - dpb_.DeleteByPOC(outputted_poc); - } - - ++output_candidate; - --num_remaining; - } - - // If we haven't managed to output the picture that we just decoded, or if - // it's a reference picture, we have to store it in DPB. - if (!pic->outputted || pic->ref) { - if (dpb_.IsFull()) { - // If we haven't managed to output anything to free up space in DPB - // to store this picture, it's an error in the stream. - DVLOG(1) << "Could not free up space in DPB!"; - return false; - } - - dpb_.StorePic(pic); - } - - return true; -} - -static int LevelToMaxDpbMbs(int level) { - // See table A-1 in spec. - switch (level) { - case 10: - return 396; - case 11: - return 900; - case 12: // fallthrough - case 13: // fallthrough - case 20: - return 2376; - case 21: - return 4752; - case 22: // fallthrough - case 30: - return 8100; - case 31: - return 18000; - case 32: - return 20480; - case 40: // fallthrough - case 41: - return 32768; - case 42: - return 34816; - case 50: - return 110400; - case 51: // fallthrough - case 52: - return 184320; - default: - DVLOG(1) << "Invalid codec level (" << level << ")"; - return 0; - } -} - -bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) { - if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) { - max_num_reorder_frames_ = - base::checked_cast<size_t>(sps->max_num_reorder_frames); - if (max_num_reorder_frames_ > dpb_.max_num_pics()) { - DVLOG(1) - << "max_num_reorder_frames present, but larger than MaxDpbFrames (" - << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")"; - max_num_reorder_frames_ = 0; - return false; - } - return true; - } - - // max_num_reorder_frames not present, infer from profile/constraints - // (see VUI semantics in spec). - if (sps->constraint_set3_flag) { - switch (sps->profile_idc) { - case 44: - case 86: - case 100: - case 110: - case 122: - case 244: - max_num_reorder_frames_ = 0; - break; - default: - max_num_reorder_frames_ = dpb_.max_num_pics(); - break; - } - } else { - max_num_reorder_frames_ = dpb_.max_num_pics(); - } - - return true; -} - -bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) { - DVLOG(4) << "Processing SPS id:" << sps_id; - - const H264SPS* sps = parser_.GetSPS(sps_id); - if (!sps) - return false; - - *need_new_buffers = false; - - if (sps->frame_mbs_only_flag == 0) { - DVLOG(1) << "frame_mbs_only_flag != 1 not supported"; - return false; - } - - Size new_pic_size = sps->GetCodedSize().value_or(Size()); - if (new_pic_size.IsEmpty()) { - DVLOG(1) << "Invalid picture size"; - return false; - } - - int width_mb = new_pic_size.width() / 16; - int height_mb = new_pic_size.height() / 16; - - // Verify that the values are not too large before multiplying. - if (std::numeric_limits<int>::max() / width_mb < height_mb) { - DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString(); - return false; - } - - int level = sps->level_idc; - int max_dpb_mbs = LevelToMaxDpbMbs(level); - if (max_dpb_mbs == 0) - return false; - - // MaxDpbFrames from level limits per spec. - size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb), - static_cast<int>(H264DPB::kDPBMaxSize)); - DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames - << ", max_num_ref_frames: " << sps->max_num_ref_frames - << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering; - - // Set DPB size to at least the level limit, or what the stream requires. - size_t max_dpb_size = - std::max(static_cast<int>(max_dpb_frames), - std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering)); - // Some non-conforming streams specify more frames are needed than the current - // level limit. Allow this, but only up to the maximum number of reference - // frames allowed per spec. - DVLOG_IF(1, max_dpb_size > max_dpb_frames) - << "Invalid stream, DPB size > MaxDpbFrames"; - if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) { - DVLOG(1) << "Invalid DPB size: " << max_dpb_size; - return false; - } - - if ((pic_size_ != new_pic_size) || (dpb_.max_num_pics() != max_dpb_size)) { - if (!Flush()) - return false; - DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size - << ", Picture size: " << new_pic_size.ToString(); - *need_new_buffers = true; - pic_size_ = new_pic_size; - dpb_.set_max_num_pics(max_dpb_size); - } - - Rect new_visible_rect = sps->GetVisibleRect().value_or(Rect()); - if (visible_rect_ != new_visible_rect) { - DVLOG(2) << "New visible rect: " << new_visible_rect.ToString(); - visible_rect_ = new_visible_rect; - } - - if (!UpdateMaxNumReorderFrames(sps)) - return false; - DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_; - - return true; -} - -bool H264Decoder::FinishPrevFrameIfPresent() { - // If we already have a frame waiting to be decoded, decode it and finish. - if (curr_pic_) { - if (!DecodePicture()) - return false; - - scoped_refptr<H264Picture> pic = curr_pic_; - curr_pic_ = nullptr; - return FinishPicture(pic); - } - - return true; -} - -bool H264Decoder::HandleFrameNumGap(int frame_num) { - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - if (!sps->gaps_in_frame_num_value_allowed_flag) { - DVLOG(1) << "Invalid frame_num: " << frame_num; - return false; - } - - DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->" - << frame_num; - - // 7.4.3/7-23 - int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_; - while (unused_short_term_frame_num != frame_num) { - scoped_refptr<H264Picture> pic = new H264Picture(); - if (!InitNonexistingPicture(pic, unused_short_term_frame_num)) - return false; - - UpdatePicNums(unused_short_term_frame_num); - - if (!FinishPicture(pic)) - return false; - - unused_short_term_frame_num++; - unused_short_term_frame_num %= max_frame_num_; - } - - return true; -} - -bool H264Decoder::IsNewPrimaryCodedPicture( - const H264SliceHeader* slice_hdr) const { - if (!curr_pic_) - return true; - - // 7.4.1.2.4, assumes non-interlaced. - if (slice_hdr->frame_num != curr_pic_->frame_num || - slice_hdr->pic_parameter_set_id != curr_pps_id_ || - slice_hdr->nal_ref_idc != curr_pic_->nal_ref_idc || - slice_hdr->idr_pic_flag != curr_pic_->idr || - (slice_hdr->idr_pic_flag && - (slice_hdr->idr_pic_id != curr_pic_->idr_pic_id || - // If we have two consecutive IDR slices, and the second one has - // first_mb_in_slice == 0, treat it as a new picture. - // Per spec, idr_pic_id should not be equal in this case (and we should - // have hit the condition above instead, see spec 7.4.3 on idr_pic_id), - // but some encoders neglect changing idr_pic_id for two consecutive - // IDRs. Work around this by checking if the next slice contains the - // zeroth macroblock, i.e. data that belongs to the next picture. - slice_hdr->first_mb_in_slice == 0))) - return true; - - const H264SPS* sps = parser_.GetSPS(curr_sps_id_); - if (!sps) - return false; - - if (sps->pic_order_cnt_type == curr_pic_->pic_order_cnt_type) { - if (curr_pic_->pic_order_cnt_type == 0) { - if (slice_hdr->pic_order_cnt_lsb != curr_pic_->pic_order_cnt_lsb || - slice_hdr->delta_pic_order_cnt_bottom != - curr_pic_->delta_pic_order_cnt_bottom) - return true; - } else if (curr_pic_->pic_order_cnt_type == 1) { - if (slice_hdr->delta_pic_order_cnt0 != curr_pic_->delta_pic_order_cnt0 || - slice_hdr->delta_pic_order_cnt1 != curr_pic_->delta_pic_order_cnt1) - return true; - } - } - - return false; -} - -bool H264Decoder::PreprocessCurrentSlice() { - const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); - DCHECK(slice_hdr); - - if (IsNewPrimaryCodedPicture(slice_hdr)) { - // New picture, so first finish the previous one before processing it. - if (!FinishPrevFrameIfPresent()) - return false; - - DCHECK(!curr_pic_); - - if (slice_hdr->first_mb_in_slice != 0) { - DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: " - << slice_hdr->first_mb_in_slice; - return false; - } - - // If the new picture is an IDR, flush DPB. - if (slice_hdr->idr_pic_flag) { - // Output all remaining pictures, unless we are explicitly instructed - // not to do so. - if (!slice_hdr->no_output_of_prior_pics_flag) { - if (!Flush()) - return false; - } - dpb_.Clear(); - last_output_poc_ = std::numeric_limits<int>::min(); - } - } - - return true; -} - -bool H264Decoder::ProcessCurrentSlice() { - DCHECK(curr_pic_); - - const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); - DCHECK(slice_hdr); - - if (slice_hdr->field_pic_flag == 0) - max_pic_num_ = max_frame_num_; - else - max_pic_num_ = 2 * max_frame_num_; - - H264Picture::Vector ref_pic_list0, ref_pic_list1; - if (!ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1)) - return false; - - const H264PPS* pps = parser_.GetPPS(curr_pps_id_); - if (!pps) - return false; - - if (!accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1, - curr_pic_.get(), slice_hdr->nalu_data, - slice_hdr->nalu_size)) - return false; - - return true; -} - -#define SET_ERROR_AND_RETURN() \ - do { \ - DVLOG(1) << "Error during decode"; \ - state_ = kError; \ - return H264Decoder::kDecodeError; \ - } while (0) - -void H264Decoder::SetStream(const uint8_t* ptr, size_t size) { - DCHECK(ptr); - DCHECK(size); - - DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size; - parser_.SetStream(ptr, size); -} - -H264Decoder::DecodeResult H264Decoder::Decode() { - if (state_ == kError) { - DVLOG(1) << "Decoder in error state"; - return kDecodeError; - } - - while (1) { - H264Parser::Result par_res; - - if (!curr_nalu_) { - curr_nalu_.reset(new H264NALU()); - par_res = parser_.AdvanceToNextNALU(curr_nalu_.get()); - if (par_res == H264Parser::kEOStream) - return kRanOutOfStreamData; - else if (par_res != H264Parser::kOk) - SET_ERROR_AND_RETURN(); - - DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type); - } - - switch (curr_nalu_->nal_unit_type) { - case H264NALU::kNonIDRSlice: - // We can't resume from a non-IDR slice. - if (state_ != kDecoding) - break; - - // else fallthrough - case H264NALU::kIDRSlice: { - // TODO(posciak): the IDR may require an SPS that we don't have - // available. For now we'd fail if that happens, but ideally we'd like - // to keep going until the next SPS in the stream. - if (state_ == kNeedStreamMetadata) { - // We need an SPS, skip this IDR and keep looking. - break; - } - - // If after reset, we should be able to recover from an IDR. - state_ = kDecoding; - - if (!curr_slice_hdr_) { - curr_slice_hdr_.reset(new H264SliceHeader()); - par_res = - parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get()); - if (par_res != H264Parser::kOk) - SET_ERROR_AND_RETURN(); - - if (!PreprocessCurrentSlice()) - SET_ERROR_AND_RETURN(); - } - - if (!curr_pic_) { - // New picture/finished previous one, try to start a new one - // or tell the client we need more surfaces. - curr_pic_ = accelerator_->CreateH264Picture(); - if (!curr_pic_) - return kRanOutOfSurfaces; - - if (!StartNewFrame(curr_slice_hdr_.get())) - SET_ERROR_AND_RETURN(); - } - - if (!ProcessCurrentSlice()) - SET_ERROR_AND_RETURN(); - - curr_slice_hdr_.reset(); - break; - } - - case H264NALU::kSPS: { - int sps_id; - - if (!FinishPrevFrameIfPresent()) - SET_ERROR_AND_RETURN(); - - par_res = parser_.ParseSPS(&sps_id); - if (par_res != H264Parser::kOk) - SET_ERROR_AND_RETURN(); - - bool need_new_buffers = false; - if (!ProcessSPS(sps_id, &need_new_buffers)) - SET_ERROR_AND_RETURN(); - - if (state_ == kNeedStreamMetadata) - state_ = kAfterReset; - - if (need_new_buffers) { - curr_pic_ = nullptr; - curr_nalu_ = nullptr; - ref_pic_list_p0_.clear(); - ref_pic_list_b0_.clear(); - ref_pic_list_b1_.clear(); - - return kAllocateNewSurfaces; - } - break; - } - - case H264NALU::kPPS: { - int pps_id; - - if (!FinishPrevFrameIfPresent()) - SET_ERROR_AND_RETURN(); - - par_res = parser_.ParsePPS(&pps_id); - if (par_res != H264Parser::kOk) - SET_ERROR_AND_RETURN(); - - break; - } - - case H264NALU::kAUD: - case H264NALU::kEOSeq: - case H264NALU::kEOStream: - if (state_ != kDecoding) - break; - - if (!FinishPrevFrameIfPresent()) - SET_ERROR_AND_RETURN(); - - break; - - default: - DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type; - break; - } - - DVLOG(4) << "NALU done"; - curr_nalu_.reset(); - } -} - -Size H264Decoder::GetPicSize() const { - return pic_size_; -} - -size_t H264Decoder::GetRequiredNumOfPictures() const { - return dpb_.max_num_pics() + kPicsInPipeline; -} - -} // namespace media diff --git a/accel/h264_decoder.h b/accel/h264_decoder.h deleted file mode 100644 index 82ab98f..0000000 --- a/accel/h264_decoder.h +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 77be7ae - -#ifndef H264_DECODER_H_ -#define H264_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> -#include <vector> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "accelerated_video_decoder.h" -#include "h264_dpb.h" -#include "h264_parser.h" -#include "rect.h" -#include "size.h" - -namespace media { - -// Clients of this class are expected to pass H264 Annex-B byte stream -// and are expected to provide an implementation of H264Accelerator for -// offloading final steps of the decoding process. -// -// This class must be created, called and destroyed on a single thread, and -// does nothing internally on any other thread. -class H264Decoder : public AcceleratedVideoDecoder { - public: - class H264Accelerator { - public: - H264Accelerator(); - virtual ~H264Accelerator(); - - // Create a new H264Picture that the decoder client can use for decoding - // and pass back to this accelerator for decoding or reference. - // When the picture is no longer needed by decoder, it will just drop - // its reference to it, and it may do so at any time. - // Note that this may return nullptr if accelerator is not able to provide - // any new pictures at given time. The decoder is expected to handle - // this situation as normal and return from Decode() with kRanOutOfSurfaces. - virtual scoped_refptr<H264Picture> CreateH264Picture() = 0; - - // Submit metadata for the current frame, providing the current |sps| and - // |pps| for it, |dpb| has to contain all the pictures in DPB for current - // frame, and |ref_pic_p0/b0/b1| as specified in the H264 spec. Note that - // depending on the frame type, either p0, or b0 and b1 are used. |pic| - // contains information about the picture for the current frame. - // Note that this does not run decode in the accelerator and the decoder - // is expected to follow this call with one or more SubmitSlice() calls - // before calling SubmitDecode(). - // Return true if successful. - virtual bool SubmitFrameMetadata(const H264SPS* sps, - const H264PPS* pps, - const H264DPB& dpb, - const H264Picture::Vector& ref_pic_listp0, - const H264Picture::Vector& ref_pic_listb0, - const H264Picture::Vector& ref_pic_listb1, - const scoped_refptr<H264Picture>& pic) = 0; - - // Submit one slice for the current frame, passing the current |pps| and - // |pic| (same as in SubmitFrameMetadata()), the parsed header for the - // current slice in |slice_hdr|, and the reordered |ref_pic_listX|, - // as per H264 spec. - // |data| pointing to the full slice (including the unparsed header| of - // |size| in bytes. - // This must be called one or more times per frame, before SubmitDecode(). - // Note that |data| does not have to remain valid after this call returns. - // Return true if successful. - virtual bool SubmitSlice(const H264PPS* pps, - const H264SliceHeader* slice_hdr, - const H264Picture::Vector& ref_pic_list0, - const H264Picture::Vector& ref_pic_list1, - const scoped_refptr<H264Picture>& pic, - const uint8_t* data, - size_t size) = 0; - - // Execute the decode in hardware for |pic|, using all the slices and - // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since - // the previous call to SubmitDecode(). - // Return true if successful. - virtual bool SubmitDecode(const scoped_refptr<H264Picture>& pic) = 0; - - // Schedule output (display) of |pic|. Note that returning from this - // method does not mean that |pic| has already been outputted (displayed), - // but guarantees that all pictures will be outputted in the same order - // as this method was called for them. Decoder may drop its reference - // to |pic| after calling this method. - // Return true if successful. - virtual bool OutputPicture(const scoped_refptr<H264Picture>& pic) = 0; - - // Reset any current state that may be cached in the accelerator, dropping - // any cached parameters/slices that have not been committed yet. - virtual void Reset() = 0; - - private: - DISALLOW_COPY_AND_ASSIGN(H264Accelerator); - }; - - H264Decoder(H264Accelerator* accelerator); - ~H264Decoder() override; - - // AcceleratedVideoDecoder implementation. - bool Flush() override WARN_UNUSED_RESULT; - void Reset() override; - void SetStream(const uint8_t* ptr, size_t size) override; - DecodeResult Decode() override WARN_UNUSED_RESULT; - Size GetPicSize() const override; - size_t GetRequiredNumOfPictures() const override; - - private: - // We need to keep at most kDPBMaxSize pictures in DPB for - // reference/to display later and an additional one for the one currently - // being decoded. We also ask for some additional ones since VDA needs - // to accumulate a few ready-to-output pictures before it actually starts - // displaying and giving them back. +2 instead of +1 because of subjective - // smoothness improvement during testing. - enum { - // TODO(johnylin): see if we could get rid of kMaxVideoFrames. - kMaxVideoFrames = 4, - kPicsInPipeline = kMaxVideoFrames + 2, - kMaxNumReqPictures = H264DPB::kDPBMaxSize + kPicsInPipeline, - }; - - // Internal state of the decoder. - enum State { - kNeedStreamMetadata, // After initialization, need an SPS. - kDecoding, // Ready to decode from any point. - kAfterReset, // After Reset(), need a resume point. - kError, // Error in decode, can't continue. - }; - - // Process H264 stream structures. - bool ProcessSPS(int sps_id, bool* need_new_buffers); - // Process current slice header to discover if we need to start a new picture, - // finishing up the current one. - bool PreprocessCurrentSlice(); - // Process current slice as a slice of the current picture. - bool ProcessCurrentSlice(); - - // Return true if we need to start a new picture. - bool IsNewPrimaryCodedPicture(const H264SliceHeader* slice_hdr) const; - - // Initialize the current picture according to data in |slice_hdr|. - bool InitCurrPicture(const H264SliceHeader* slice_hdr); - - // Initialize |pic| as a "non-existing" picture (see spec) with |frame_num|, - // to be used for frame gap concealment. - bool InitNonexistingPicture(scoped_refptr<H264Picture> pic, int frame_num); - - // Calculate picture order counts for |pic| on initialization - // of a new frame (see spec). - bool CalculatePicOrderCounts(scoped_refptr<H264Picture> pic); - - // Update PicNum values in pictures stored in DPB on creation of - // a picture with |frame_num|. - void UpdatePicNums(int frame_num); - - bool UpdateMaxNumReorderFrames(const H264SPS* sps); - - // Prepare reference picture lists for the current frame. - void PrepareRefPicLists(const H264SliceHeader* slice_hdr); - // Prepare reference picture lists for the given slice. - bool ModifyReferencePicLists(const H264SliceHeader* slice_hdr, - H264Picture::Vector* ref_pic_list0, - H264Picture::Vector* ref_pic_list1); - - // Construct initial reference picture lists for use in decoding of - // P and B pictures (see 8.2.4 in spec). - void ConstructReferencePicListsP(const H264SliceHeader* slice_hdr); - void ConstructReferencePicListsB(const H264SliceHeader* slice_hdr); - - // Helper functions for reference list construction, per spec. - int PicNumF(const scoped_refptr<H264Picture>& pic); - int LongTermPicNumF(const scoped_refptr<H264Picture>& pic); - - // Perform the reference picture lists' modification (reordering), as - // specified in spec (8.2.4). - // - // |list| indicates list number and should be either 0 or 1. - bool ModifyReferencePicList(const H264SliceHeader* slice_hdr, - int list, - H264Picture::Vector* ref_pic_listx); - - // Perform reference picture memory management operations (marking/unmarking - // of reference pictures, long term picture management, discarding, etc.). - // See 8.2.5 in spec. - bool HandleMemoryManagementOps(scoped_refptr<H264Picture> pic); - bool ReferencePictureMarking(scoped_refptr<H264Picture> pic); - bool SlidingWindowPictureMarking(); - - // Handle a gap in frame_num in the stream up to |frame_num|, by creating - // "non-existing" pictures (see spec). - bool HandleFrameNumGap(int frame_num); - - // Start processing a new frame. - bool StartNewFrame(const H264SliceHeader* slice_hdr); - - // All data for a frame received, process it and decode. - bool FinishPrevFrameIfPresent(); - - // Called after we are done processing |pic|. Performs all operations to be - // done after decoding, including DPB management, reference picture marking - // and memory management operations. - // This will also output pictures if any have become ready to be outputted - // after processing |pic|. - bool FinishPicture(scoped_refptr<H264Picture> pic); - - // Clear DPB contents and remove all surfaces in DPB from *in_use_ list. - // Cleared pictures will be made available for decode, unless they are - // at client waiting to be displayed. - void ClearDPB(); - - // Commits all pending data for HW decoder and starts HW decoder. - bool DecodePicture(); - - // Notifies client that a picture is ready for output. - void OutputPic(scoped_refptr<H264Picture> pic); - - // Output all pictures in DPB that have not been outputted yet. - bool OutputAllRemainingPics(); - - // Decoder state. - State state_; - - // Parser in use. - H264Parser parser_; - - // DPB in use. - H264DPB dpb_; - - // Picture currently being processed/decoded. - scoped_refptr<H264Picture> curr_pic_; - - // Reference picture lists, constructed for each frame. - H264Picture::Vector ref_pic_list_p0_; - H264Picture::Vector ref_pic_list_b0_; - H264Picture::Vector ref_pic_list_b1_; - - // Global state values, needed in decoding. See spec. - int max_frame_num_; - int max_pic_num_; - int max_long_term_frame_idx_; - size_t max_num_reorder_frames_; - - int prev_frame_num_; - int prev_ref_frame_num_; - int prev_frame_num_offset_; - bool prev_has_memmgmnt5_; - - // Values related to previously decoded reference picture. - bool prev_ref_has_memmgmnt5_; - int prev_ref_top_field_order_cnt_; - int prev_ref_pic_order_cnt_msb_; - int prev_ref_pic_order_cnt_lsb_; - H264Picture::Field prev_ref_field_; - - // Currently active SPS and PPS. - int curr_sps_id_; - int curr_pps_id_; - - // Current NALU and slice header being processed. - std::unique_ptr<H264NALU> curr_nalu_; - std::unique_ptr<H264SliceHeader> curr_slice_hdr_; - - // Output picture size. - Size pic_size_; - // Output visible cropping rect. - Rect visible_rect_; - - // PicOrderCount of the previously outputted frame. - int last_output_poc_; - - H264Accelerator* accelerator_; - - DISALLOW_COPY_AND_ASSIGN(H264Decoder); -}; - -} // namespace media - -#endif // H264_DECODER_H_ diff --git a/accel/h264_dpb.cc b/accel/h264_dpb.cc deleted file mode 100644 index af0b5e0..0000000 --- a/accel/h264_dpb.cc +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include <string.h> - -#include <algorithm> - -#include "base/logging.h" -#include "base/stl_util.h" -#include "h264_dpb.h" - -namespace media { - -H264Picture::H264Picture() - : pic_order_cnt_type(0), - top_field_order_cnt(0), - bottom_field_order_cnt(0), - pic_order_cnt(0), - pic_order_cnt_msb(0), - pic_order_cnt_lsb(0), - delta_pic_order_cnt_bottom(0), - delta_pic_order_cnt0(0), - delta_pic_order_cnt1(0), - pic_num(0), - long_term_pic_num(0), - frame_num(0), - frame_num_offset(0), - frame_num_wrap(0), - long_term_frame_idx(0), - type(H264SliceHeader::kPSlice), - nal_ref_idc(0), - idr(false), - idr_pic_id(0), - ref(false), - long_term(false), - outputted(false), - mem_mgmt_5(false), - nonexisting(false), - field(FIELD_NONE), - long_term_reference_flag(false), - adaptive_ref_pic_marking_mode_flag(false), - dpb_position(0) { - memset(&ref_pic_marking, 0, sizeof(ref_pic_marking)); -} - -H264Picture::~H264Picture() = default; - -V4L2H264Picture* H264Picture::AsV4L2H264Picture() { - return nullptr; -} - -H264DPB::H264DPB() : max_num_pics_(0) {} -H264DPB::~H264DPB() = default; - -void H264DPB::Clear() { - pics_.clear(); -} - -void H264DPB::set_max_num_pics(size_t max_num_pics) { - DCHECK_LE(max_num_pics, static_cast<size_t>(kDPBMaxSize)); - max_num_pics_ = max_num_pics; - if (pics_.size() > max_num_pics_) - pics_.resize(max_num_pics_); -} - -void H264DPB::UpdatePicPositions() { - size_t i = 0; - for (auto& pic : pics_) { - pic->dpb_position = i; - ++i; - } -} - -void H264DPB::DeleteByPOC(int poc) { - for (H264Picture::Vector::iterator it = pics_.begin(); it != pics_.end(); - ++it) { - if ((*it)->pic_order_cnt == poc) { - pics_.erase(it); - UpdatePicPositions(); - return; - } - } - NOTREACHED() << "Missing POC: " << poc; -} - -void H264DPB::DeleteUnused() { - for (H264Picture::Vector::iterator it = pics_.begin(); it != pics_.end();) { - if ((*it)->outputted && !(*it)->ref) - it = pics_.erase(it); - else - ++it; - } - UpdatePicPositions(); -} - -void H264DPB::StorePic(const scoped_refptr<H264Picture>& pic) { - DCHECK_LT(pics_.size(), max_num_pics_); - DVLOG(3) << "Adding PicNum: " << pic->pic_num << " ref: " << (int)pic->ref - << " longterm: " << (int)pic->long_term << " to DPB"; - pic->dpb_position = pics_.size(); - pics_.push_back(pic); -} - -int H264DPB::CountRefPics() { - int ret = 0; - for (size_t i = 0; i < pics_.size(); ++i) { - if (pics_[i]->ref) - ++ret; - } - return ret; -} - -void H264DPB::MarkAllUnusedForRef() { - for (size_t i = 0; i < pics_.size(); ++i) - pics_[i]->ref = false; -} - -scoped_refptr<H264Picture> H264DPB::GetShortRefPicByPicNum(int pic_num) { - for (const auto& pic : pics_) { - if (pic->ref && !pic->long_term && pic->pic_num == pic_num) - return pic; - } - - DVLOG(1) << "Missing short ref pic num: " << pic_num; - return nullptr; -} - -scoped_refptr<H264Picture> H264DPB::GetLongRefPicByLongTermPicNum(int pic_num) { - for (const auto& pic : pics_) { - if (pic->ref && pic->long_term && pic->long_term_pic_num == pic_num) - return pic; - } - - DVLOG(1) << "Missing long term pic num: " << pic_num; - return nullptr; -} - -scoped_refptr<H264Picture> H264DPB::GetLowestFrameNumWrapShortRefPic() { - scoped_refptr<H264Picture> ret; - for (const auto& pic : pics_) { - if (pic->ref && !pic->long_term && - (!ret || pic->frame_num_wrap < ret->frame_num_wrap)) - ret = pic; - } - return ret; -} - -void H264DPB::GetNotOutputtedPicsAppending(H264Picture::Vector* out) { - for (const auto& pic : pics_) { - if (!pic->outputted) - out->push_back(pic); - } -} - -void H264DPB::GetShortTermRefPicsAppending(H264Picture::Vector* out) { - for (const auto& pic : pics_) { - if (pic->ref && !pic->long_term) - out->push_back(pic); - } -} - -void H264DPB::GetLongTermRefPicsAppending(H264Picture::Vector* out) { - for (const auto& pic : pics_) { - if (pic->ref && pic->long_term) - out->push_back(pic); - } -} - -} // namespace media diff --git a/accel/h264_dpb.h b/accel/h264_dpb.h deleted file mode 100644 index 3da284e..0000000 --- a/accel/h264_dpb.h +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of an H.264 Decoded Picture Buffer -// used in H264 decoders. -// Note: ported from Chromium commit head: 70340ce - -#ifndef H264_DPB_H_ -#define H264_DPB_H_ - -#include <stddef.h> - -#include <vector> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "h264_parser.h" -#include "rect.h" - -namespace media { - -class V4L2H264Picture; - -// A picture (a frame or a field) in the H.264 spec sense. -// See spec at http://www.itu.int/rec/T-REC-H.264 -class H264Picture : public base::RefCountedThreadSafe<H264Picture> { - public: - using Vector = std::vector<scoped_refptr<H264Picture>>; - - enum Field { - FIELD_NONE, - FIELD_TOP, - FIELD_BOTTOM, - }; - - H264Picture(); - - virtual V4L2H264Picture* AsV4L2H264Picture(); - - // Values calculated per H.264 specification or taken from slice header. - // See spec for more details on each (some names have been converted from - // CamelCase in spec to Chromium-style names). - int pic_order_cnt_type; - int top_field_order_cnt; - int bottom_field_order_cnt; - int pic_order_cnt; - int pic_order_cnt_msb; - int pic_order_cnt_lsb; - int delta_pic_order_cnt_bottom; - int delta_pic_order_cnt0; - int delta_pic_order_cnt1; - - int pic_num; - int long_term_pic_num; - int frame_num; // from slice header - int frame_num_offset; - int frame_num_wrap; - int long_term_frame_idx; - - H264SliceHeader::Type type; - int nal_ref_idc; - bool idr; // IDR picture? - int idr_pic_id; // Valid only if idr == true. - bool ref; // reference picture? - bool long_term; // long term reference picture? - bool outputted; - // Does memory management op 5 needs to be executed after this - // picture has finished decoding? - bool mem_mgmt_5; - - // Created by the decoding process for gaps in frame_num. - // Not for decode or output. - bool nonexisting; - - Field field; - - // Values from slice_hdr to be used during reference marking and - // memory management after finishing this picture. - bool long_term_reference_flag; - bool adaptive_ref_pic_marking_mode_flag; - H264DecRefPicMarking ref_pic_marking[H264SliceHeader::kRefListSize]; - - // Position in DPB (i.e. index in DPB). - int dpb_position; - - // The visible size of picture. This could be either parsed from SPS, or set - // to Rect(0, 0) for indicating invalid values or not available. - Rect visible_rect; - - protected: - friend class base::RefCountedThreadSafe<H264Picture>; - virtual ~H264Picture(); - - private: - DISALLOW_COPY_AND_ASSIGN(H264Picture); -}; - -// DPB - Decoded Picture Buffer. -// Stores decoded pictures that will be used for future display -// and/or reference. -class H264DPB { - public: - H264DPB(); - ~H264DPB(); - - void set_max_num_pics(size_t max_num_pics); - size_t max_num_pics() const { return max_num_pics_; } - - // Remove unused (not reference and already outputted) pictures from DPB - // and free it. - void DeleteUnused(); - - // Remove a picture by its pic_order_cnt and free it. - void DeleteByPOC(int poc); - - // Clear DPB. - void Clear(); - - // Store picture in DPB. DPB takes ownership of its resources. - void StorePic(const scoped_refptr<H264Picture>& pic); - - // Return the number of reference pictures in DPB. - int CountRefPics(); - - // Mark all pictures in DPB as unused for reference. - void MarkAllUnusedForRef(); - - // Return a short-term reference picture by its pic_num. - scoped_refptr<H264Picture> GetShortRefPicByPicNum(int pic_num); - - // Return a long-term reference picture by its long_term_pic_num. - scoped_refptr<H264Picture> GetLongRefPicByLongTermPicNum(int pic_num); - - // Return the short reference picture with lowest frame_num. Used for sliding - // window memory management. - scoped_refptr<H264Picture> GetLowestFrameNumWrapShortRefPic(); - - // Append all pictures that have not been outputted yet to the passed |out| - // vector, sorted by lowest pic_order_cnt (in output order). - void GetNotOutputtedPicsAppending(H264Picture::Vector* out); - - // Append all short term reference pictures to the passed |out| vector. - void GetShortTermRefPicsAppending(H264Picture::Vector* out); - - // Append all long term reference pictures to the passed |out| vector. - void GetLongTermRefPicsAppending(H264Picture::Vector* out); - - // Iterators for direct access to DPB contents. - // Will be invalidated after any of Remove* calls. - H264Picture::Vector::iterator begin() { return pics_.begin(); } - H264Picture::Vector::iterator end() { return pics_.end(); } - H264Picture::Vector::const_iterator begin() const { return pics_.begin(); } - H264Picture::Vector::const_iterator end() const { return pics_.end(); } - H264Picture::Vector::const_reverse_iterator rbegin() const { - return pics_.rbegin(); - } - H264Picture::Vector::const_reverse_iterator rend() const { - return pics_.rend(); - } - - size_t size() const { return pics_.size(); } - bool IsFull() const { return pics_.size() == max_num_pics_; } - - // Per H264 spec, increase to 32 if interlaced video is supported. - enum { - kDPBMaxSize = 16, - }; - - private: - void UpdatePicPositions(); - - H264Picture::Vector pics_; - size_t max_num_pics_; - - DISALLOW_COPY_AND_ASSIGN(H264DPB); -}; - -} // namespace media - -#endif // H264_DPB_H_ diff --git a/accel/h264_parser.cc b/accel/h264_parser.cc deleted file mode 100644 index 0e24473..0000000 --- a/accel/h264_parser.cc +++ /dev/null @@ -1,1612 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 600904374759 -// Note: GetColorSpace() is not ported. - -#include "h264_parser.h" -#include "subsample_entry.h" - -#include <limits> -#include <memory> - -#include "base/logging.h" -#include "base/numerics/safe_math.h" -#include "base/stl_util.h" - -namespace media { - -namespace { -// Converts [|start|, |end|) range with |encrypted_ranges| into a vector of -// SubsampleEntry. |encrypted_ranges| must be with in the range defined by -// |start| and |end|. -// It is OK to pass in empty |encrypted_ranges|; this will return a vector -// with single SubsampleEntry with clear_bytes set to the size of the buffer. -std::vector<SubsampleEntry> EncryptedRangesToSubsampleEntry( - const uint8_t* start, - const uint8_t* end, - const Ranges<const uint8_t*>& encrypted_ranges) { - std::vector<SubsampleEntry> subsamples; - const uint8_t* cur = start; - for (size_t i = 0; i < encrypted_ranges.size(); ++i) { - SubsampleEntry subsample = {}; - - const uint8_t* encrypted_start = encrypted_ranges.start(i); - DCHECK_GE(encrypted_start, cur) - << "Encrypted range started before the current buffer pointer."; - subsample.clear_bytes = encrypted_start - cur; - - const uint8_t* encrypted_end = encrypted_ranges.end(i); - subsample.cypher_bytes = encrypted_end - encrypted_start; - - subsamples.push_back(subsample); - cur = encrypted_end; - DCHECK_LE(cur, end) << "Encrypted range is outside the buffer range."; - } - - // If there is more data in the buffer but not covered by encrypted_ranges, - // then it must be in the clear. - if (cur < end) { - SubsampleEntry subsample = {}; - subsample.clear_bytes = end - cur; - subsamples.push_back(subsample); - } - return subsamples; -} -} // namespace - -bool H264SliceHeader::IsPSlice() const { - return (slice_type % 5 == kPSlice); -} - -bool H264SliceHeader::IsBSlice() const { - return (slice_type % 5 == kBSlice); -} - -bool H264SliceHeader::IsISlice() const { - return (slice_type % 5 == kISlice); -} - -bool H264SliceHeader::IsSPSlice() const { - return (slice_type % 5 == kSPSlice); -} - -bool H264SliceHeader::IsSISlice() const { - return (slice_type % 5 == kSISlice); -} - -H264NALU::H264NALU() { - memset(this, 0, sizeof(*this)); -} - -// static -void H264SPS::GetLevelConfigFromProfileLevel(VideoCodecProfile profile, - uint8_t level, - int* level_idc, - bool* constraint_set3_flag) { - // Spec A.3.1. - // Note: we always use h264_output_level = 9 to indicate Level 1b in - // VideoEncodeAccelerator::Config, in order to tell apart from Level 1.1 - // which level IDC is also 11. - // For Baseline and Main profile, if requested level is Level 1b, set - // level_idc to 11 and constraint_set3_flag to true. Otherwise, set level_idc - // to 9 for Level 1b, and ten times level number for others. - if ((profile == H264PROFILE_BASELINE || profile == H264PROFILE_MAIN) && - level == kLevelIDC1B) { - *level_idc = 11; - *constraint_set3_flag = true; - } else { - *level_idc = level; - } -} - -H264SPS::H264SPS() { - memset(this, 0, sizeof(*this)); -} - -// Based on T-REC-H.264 7.4.2.1.1, "Sequence parameter set data semantics", -// available from http://www.itu.int/rec/T-REC-H.264. -base::Optional<Size> H264SPS::GetCodedSize() const { - // Interlaced frames are twice the height of each field. - const int mb_unit = 16; - int map_unit = frame_mbs_only_flag ? 16 : 32; - - // Verify that the values are not too large before multiplying them. - // TODO(sandersd): These limits could be much smaller. The currently-largest - // specified limit (excluding SVC, multiview, etc., which I didn't bother to - // read) is 543 macroblocks (section A.3.1). - int max_mb_minus1 = std::numeric_limits<int>::max() / mb_unit - 1; - int max_map_units_minus1 = std::numeric_limits<int>::max() / map_unit - 1; - if (pic_width_in_mbs_minus1 > max_mb_minus1 || - pic_height_in_map_units_minus1 > max_map_units_minus1) { - DVLOG(1) << "Coded size is too large."; - return base::nullopt; - } - - return Size(mb_unit * (pic_width_in_mbs_minus1 + 1), - map_unit * (pic_height_in_map_units_minus1 + 1)); -} - -// Also based on section 7.4.2.1.1. -base::Optional<Rect> H264SPS::GetVisibleRect() const { - base::Optional<Size> coded_size = GetCodedSize(); - if (!coded_size) - return base::nullopt; - - if (!frame_cropping_flag) - return Rect(coded_size.value()); - - int crop_unit_x; - int crop_unit_y; - if (chroma_array_type == 0) { - crop_unit_x = 1; - crop_unit_y = frame_mbs_only_flag ? 1 : 2; - } else { - // Section 6.2. - // |chroma_format_idc| may be: - // 1 => 4:2:0 - // 2 => 4:2:2 - // 3 => 4:4:4 - // Everything else has |chroma_array_type| == 0. - int sub_width_c = chroma_format_idc > 2 ? 1 : 2; - int sub_height_c = chroma_format_idc > 1 ? 1 : 2; - crop_unit_x = sub_width_c; - crop_unit_y = sub_height_c * (frame_mbs_only_flag ? 1 : 2); - } - - // Verify that the values are not too large before multiplying. - if (coded_size->width() / crop_unit_x < frame_crop_left_offset || - coded_size->width() / crop_unit_x < frame_crop_right_offset || - coded_size->height() / crop_unit_y < frame_crop_top_offset || - coded_size->height() / crop_unit_y < frame_crop_bottom_offset) { - DVLOG(1) << "Frame cropping exceeds coded size."; - return base::nullopt; - } - int crop_left = crop_unit_x * frame_crop_left_offset; - int crop_right = crop_unit_x * frame_crop_right_offset; - int crop_top = crop_unit_y * frame_crop_top_offset; - int crop_bottom = crop_unit_y * frame_crop_bottom_offset; - - // Verify that the values are sane. Note that some decoders also require that - // crops are smaller than a macroblock and/or that crops must be adjacent to - // at least one corner of the coded frame. - if (coded_size->width() - crop_left <= crop_right || - coded_size->height() - crop_top <= crop_bottom) { - DVLOG(1) << "Frame cropping excludes entire frame."; - return base::nullopt; - } - - return Rect(crop_left, crop_top, - coded_size->width() - crop_left - crop_right, - coded_size->height() - crop_top - crop_bottom); -} - -uint8_t H264SPS::GetIndicatedLevel() const { - // Spec A.3.1 and A.3.2 - // For Baseline, Constrained Baseline and Main profile, the indicated level is - // Level 1b if level_idc is equal to 11 and constraint_set3_flag is true. - if ((profile_idc == H264SPS::kProfileIDCBaseline || - profile_idc == H264SPS::kProfileIDCConstrainedBaseline || - profile_idc == H264SPS::kProfileIDCMain) && - level_idc == 11 && constraint_set3_flag) { - return kLevelIDC1B; // Level 1b - } - - // Otherwise, the level_idc is equal to 9 for Level 1b, and others are equal - // to values of ten times the level numbers. - return base::checked_cast<uint8_t>(level_idc); -} - -bool H264SPS::CheckIndicatedLevelWithinTarget(uint8_t target_level) const { - // See table A-1 in spec. - // Level 1.0 < 1b < 1.1 < 1.2 .... (in numeric order). - uint8_t level = GetIndicatedLevel(); - if (target_level == kLevelIDC1p0) - return level == kLevelIDC1p0; - if (target_level == kLevelIDC1B) - return level == kLevelIDC1p0 || level == kLevelIDC1B; - return level <= target_level; -} - -H264PPS::H264PPS() { - memset(this, 0, sizeof(*this)); -} - -H264SliceHeader::H264SliceHeader() { - memset(this, 0, sizeof(*this)); -} - -H264SEIMessage::H264SEIMessage() { - memset(this, 0, sizeof(*this)); -} - -#define READ_BITS_OR_RETURN(num_bits, out) \ - do { \ - int _out; \ - if (!br_.ReadBits(num_bits, &_out)) { \ - DVLOG(1) \ - << "Error in stream: unexpected EOS while trying to read " #out; \ - return kInvalidStream; \ - } \ - *out = _out; \ - } while (0) - -#define READ_BOOL_OR_RETURN(out) \ - do { \ - int _out; \ - if (!br_.ReadBits(1, &_out)) { \ - DVLOG(1) \ - << "Error in stream: unexpected EOS while trying to read " #out; \ - return kInvalidStream; \ - } \ - *out = _out != 0; \ - } while (0) - -#define READ_UE_OR_RETURN(out) \ - do { \ - if (ReadUE(out) != kOk) { \ - DVLOG(1) << "Error in stream: invalid value while trying to read " #out; \ - return kInvalidStream; \ - } \ - } while (0) - -#define READ_SE_OR_RETURN(out) \ - do { \ - if (ReadSE(out) != kOk) { \ - DVLOG(1) << "Error in stream: invalid value while trying to read " #out; \ - return kInvalidStream; \ - } \ - } while (0) - -#define IN_RANGE_OR_RETURN(val, min, max) \ - do { \ - if ((val) < (min) || (val) > (max)) { \ - DVLOG(1) << "Error in stream: invalid value, expected " #val " to be" \ - << " in range [" << (min) << ":" << (max) << "]" \ - << " found " << (val) << " instead"; \ - return kInvalidStream; \ - } \ - } while (0) - -#define TRUE_OR_RETURN(a) \ - do { \ - if (!(a)) { \ - DVLOG(1) << "Error in stream: invalid value, expected " << #a; \ - return kInvalidStream; \ - } \ - } while (0) - -// ISO 14496 part 10 -// VUI parameters: Table E-1 "Meaning of sample aspect ratio indicator" -static const int kTableSarWidth[] = {0, 1, 12, 10, 16, 40, 24, 20, 32, - 80, 18, 15, 64, 160, 4, 3, 2}; -static const int kTableSarHeight[] = {0, 1, 11, 11, 11, 33, 11, 11, 11, - 33, 11, 11, 33, 99, 3, 2, 1}; -static_assert(base::size(kTableSarWidth) == base::size(kTableSarHeight), - "sar tables must have the same size"); - -H264Parser::H264Parser() { - Reset(); -} - -H264Parser::~H264Parser() = default; - -void H264Parser::Reset() { - stream_ = NULL; - bytes_left_ = 0; - encrypted_ranges_.clear(); - previous_nalu_range_.clear(); -} - -void H264Parser::SetStream(const uint8_t* stream, off_t stream_size) { - std::vector<SubsampleEntry> subsamples; - SetEncryptedStream(stream, stream_size, subsamples); -} - -void H264Parser::SetEncryptedStream( - const uint8_t* stream, - off_t stream_size, - const std::vector<SubsampleEntry>& subsamples) { - DCHECK(stream); - DCHECK_GT(stream_size, 0); - - stream_ = stream; - bytes_left_ = stream_size; - previous_nalu_range_.clear(); - - encrypted_ranges_.clear(); - const uint8_t* start = stream; - const uint8_t* stream_end = stream_ + bytes_left_; - for (size_t i = 0; i < subsamples.size() && start < stream_end; ++i) { - start += subsamples[i].clear_bytes; - - const uint8_t* end = - std::min(start + subsamples[i].cypher_bytes, stream_end); - encrypted_ranges_.Add(start, end); - start = end; - } -} - -const H264PPS* H264Parser::GetPPS(int pps_id) const { - auto it = active_PPSes_.find(pps_id); - if (it == active_PPSes_.end()) { - DVLOG(1) << "Requested a nonexistent PPS id " << pps_id; - return nullptr; - } - - return it->second.get(); -} - -const H264SPS* H264Parser::GetSPS(int sps_id) const { - auto it = active_SPSes_.find(sps_id); - if (it == active_SPSes_.end()) { - DVLOG(1) << "Requested a nonexistent SPS id " << sps_id; - return nullptr; - } - - return it->second.get(); -} - -static inline bool IsStartCode(const uint8_t* data) { - return data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01; -} - -// static -bool H264Parser::FindStartCode(const uint8_t* data, - off_t data_size, - off_t* offset, - off_t* start_code_size) { - DCHECK_GE(data_size, 0); - off_t bytes_left = data_size; - - while (bytes_left >= 3) { - // The start code is "\0\0\1", ones are more unusual than zeroes, so let's - // search for it first. - const uint8_t* tmp = - reinterpret_cast<const uint8_t*>(memchr(data + 2, 1, bytes_left - 2)); - if (!tmp) { - data += bytes_left - 2; - bytes_left = 2; - break; - } - tmp -= 2; - bytes_left -= tmp - data; - data = tmp; - - if (IsStartCode(data)) { - // Found three-byte start code, set pointer at its beginning. - *offset = data_size - bytes_left; - *start_code_size = 3; - - // If there is a zero byte before this start code, - // then it's actually a four-byte start code, so backtrack one byte. - if (*offset > 0 && *(data - 1) == 0x00) { - --(*offset); - ++(*start_code_size); - } - - return true; - } - - ++data; - --bytes_left; - } - - // End of data: offset is pointing to the first byte that was not considered - // as a possible start of a start code. - // Note: there is no security issue when receiving a negative |data_size| - // since in this case, |bytes_left| is equal to |data_size| and thus - // |*offset| is equal to 0 (valid offset). - *offset = data_size - bytes_left; - *start_code_size = 0; - return false; -} - -bool H264Parser::LocateNALU(off_t* nalu_size, off_t* start_code_size) { - // Find the start code of next NALU. - off_t nalu_start_off = 0; - off_t annexb_start_code_size = 0; - - if (!FindStartCodeInClearRanges(stream_, bytes_left_, encrypted_ranges_, - &nalu_start_off, &annexb_start_code_size)) { - DVLOG(4) << "Could not find start code, end of stream?"; - return false; - } - - // Move the stream to the beginning of the NALU (pointing at the start code). - stream_ += nalu_start_off; - bytes_left_ -= nalu_start_off; - - const uint8_t* nalu_data = stream_ + annexb_start_code_size; - off_t max_nalu_data_size = bytes_left_ - annexb_start_code_size; - if (max_nalu_data_size <= 0) { - DVLOG(3) << "End of stream"; - return false; - } - - // Find the start code of next NALU; - // if successful, |nalu_size_without_start_code| is the number of bytes from - // after previous start code to before this one; - // if next start code is not found, it is still a valid NALU since there - // are some bytes left after the first start code: all the remaining bytes - // belong to the current NALU. - off_t next_start_code_size = 0; - off_t nalu_size_without_start_code = 0; - if (!FindStartCodeInClearRanges( - nalu_data, max_nalu_data_size, encrypted_ranges_, - &nalu_size_without_start_code, &next_start_code_size)) { - nalu_size_without_start_code = max_nalu_data_size; - } - *nalu_size = nalu_size_without_start_code + annexb_start_code_size; - *start_code_size = annexb_start_code_size; - return true; -} - -// static -bool H264Parser::FindStartCodeInClearRanges( - const uint8_t* data, - off_t data_size, - const Ranges<const uint8_t*>& encrypted_ranges, - off_t* offset, - off_t* start_code_size) { - if (encrypted_ranges.size() == 0) - return FindStartCode(data, data_size, offset, start_code_size); - - DCHECK_GE(data_size, 0); - const uint8_t* start = data; - do { - off_t bytes_left = data_size - (start - data); - - if (!FindStartCode(start, bytes_left, offset, start_code_size)) - return false; - - // Construct a Ranges object that represents the region occupied - // by the start code and the 1 byte needed to read the NAL unit type. - const uint8_t* start_code = start + *offset; - const uint8_t* start_code_end = start_code + *start_code_size; - Ranges<const uint8_t*> start_code_range; - start_code_range.Add(start_code, start_code_end + 1); - - if (encrypted_ranges.IntersectionWith(start_code_range).size() > 0) { - // The start code is inside an encrypted section so we need to scan - // for another start code. - *start_code_size = 0; - start += std::min(*offset + 1, bytes_left); - } - } while (*start_code_size == 0); - - // Update |*offset| to include the data we skipped over. - *offset += start - data; - return true; -} - -// static -VideoCodecProfile H264Parser::ProfileIDCToVideoCodecProfile(int profile_idc) { - switch (profile_idc) { - case H264SPS::kProfileIDCBaseline: - return H264PROFILE_BASELINE; - case H264SPS::kProfileIDCMain: - return H264PROFILE_MAIN; - case H264SPS::kProfileIDCHigh: - return H264PROFILE_HIGH; - case H264SPS::kProfileIDHigh10: - return H264PROFILE_HIGH10PROFILE; - case H264SPS::kProfileIDHigh422: - return H264PROFILE_HIGH422PROFILE; - case H264SPS::kProfileIDHigh444Predictive: - return H264PROFILE_HIGH444PREDICTIVEPROFILE; - case H264SPS::kProfileIDScalableBaseline: - return H264PROFILE_SCALABLEBASELINE; - case H264SPS::kProfileIDScalableHigh: - return H264PROFILE_SCALABLEHIGH; - case H264SPS::kProfileIDStereoHigh: - return H264PROFILE_STEREOHIGH; - case H264SPS::kProfileIDSMultiviewHigh: - return H264PROFILE_MULTIVIEWHIGH; - } - DVLOG(1) << "unknown video profile: " << profile_idc; - return VIDEO_CODEC_PROFILE_UNKNOWN; -} - -// static -bool H264Parser::ParseNALUs(const uint8_t* stream, - size_t stream_size, - std::vector<H264NALU>* nalus) { - DCHECK(nalus); - H264Parser parser; - parser.SetStream(stream, stream_size); - - while (true) { - H264NALU nalu; - const H264Parser::Result result = parser.AdvanceToNextNALU(&nalu); - if (result == H264Parser::kOk) { - nalus->push_back(nalu); - } else if (result == media::H264Parser::kEOStream) { - return true; - } else { - DLOG(ERROR) << "Unexpected H264 parser result"; - return false; - } - } - NOTREACHED(); - return false; -} - -H264Parser::Result H264Parser::ReadUE(int* val) { - int num_bits = -1; - int bit; - int rest; - - // Count the number of contiguous zero bits. - do { - READ_BITS_OR_RETURN(1, &bit); - num_bits++; - } while (bit == 0); - - if (num_bits > 31) - return kInvalidStream; - - // Calculate exp-Golomb code value of size num_bits. - // Special case for |num_bits| == 31 to avoid integer overflow. The only - // valid representation as an int is 2^31 - 1, so the remaining bits must - // be 0 or else the number is too large. - *val = (1u << num_bits) - 1u; - - if (num_bits == 31) { - READ_BITS_OR_RETURN(num_bits, &rest); - return (rest == 0) ? kOk : kInvalidStream; - } - - if (num_bits > 0) { - READ_BITS_OR_RETURN(num_bits, &rest); - *val += rest; - } - - return kOk; -} - -H264Parser::Result H264Parser::ReadSE(int* val) { - int ue; - Result res; - - // See Chapter 9 in the spec. - res = ReadUE(&ue); - if (res != kOk) - return res; - - if (ue % 2 == 0) - *val = -(ue / 2); - else - *val = ue / 2 + 1; - - return kOk; -} - -H264Parser::Result H264Parser::AdvanceToNextNALU(H264NALU* nalu) { - off_t start_code_size; - off_t nalu_size_with_start_code; - if (!LocateNALU(&nalu_size_with_start_code, &start_code_size)) { - DVLOG(4) << "Could not find next NALU, bytes left in stream: " - << bytes_left_; - stream_ = nullptr; - bytes_left_ = 0; - return kEOStream; - } - - nalu->data = stream_ + start_code_size; - nalu->size = nalu_size_with_start_code - start_code_size; - DVLOG(4) << "NALU found: size=" << nalu_size_with_start_code; - - // Initialize bit reader at the start of found NALU. - if (!br_.Initialize(nalu->data, nalu->size)) { - stream_ = nullptr; - bytes_left_ = 0; - return kEOStream; - } - - // Move parser state to after this NALU, so next time AdvanceToNextNALU - // is called, we will effectively be skipping it; - // other parsing functions will use the position saved - // in bit reader for parsing, so we don't have to remember it here. - stream_ += nalu_size_with_start_code; - bytes_left_ -= nalu_size_with_start_code; - - // Read NALU header, skip the forbidden_zero_bit, but check for it. - int data; - READ_BITS_OR_RETURN(1, &data); - TRUE_OR_RETURN(data == 0); - - READ_BITS_OR_RETURN(2, &nalu->nal_ref_idc); - READ_BITS_OR_RETURN(5, &nalu->nal_unit_type); - - DVLOG(4) << "NALU type: " << static_cast<int>(nalu->nal_unit_type) - << " at: " << reinterpret_cast<const void*>(nalu->data) - << " size: " << nalu->size - << " ref: " << static_cast<int>(nalu->nal_ref_idc); - - previous_nalu_range_.clear(); - previous_nalu_range_.Add(nalu->data, nalu->data + nalu->size); - return kOk; -} - -// Default scaling lists (per spec). -static const int kDefault4x4Intra[kH264ScalingList4x4Length] = { - 6, 13, 13, 20, 20, 20, 28, 28, 28, 28, 32, 32, 32, 37, 37, 42, -}; - -static const int kDefault4x4Inter[kH264ScalingList4x4Length] = { - 10, 14, 14, 20, 20, 20, 24, 24, 24, 24, 27, 27, 27, 30, 30, 34, -}; - -static const int kDefault8x8Intra[kH264ScalingList8x8Length] = { - 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, - 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, - 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, - 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42, -}; - -static const int kDefault8x8Inter[kH264ScalingList8x8Length] = { - 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, - 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, - 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, - 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35, -}; - -static inline void DefaultScalingList4x4( - int i, - int scaling_list4x4[][kH264ScalingList4x4Length]) { - DCHECK_LT(i, 6); - - if (i < 3) - memcpy(scaling_list4x4[i], kDefault4x4Intra, sizeof(kDefault4x4Intra)); - else if (i < 6) - memcpy(scaling_list4x4[i], kDefault4x4Inter, sizeof(kDefault4x4Inter)); -} - -static inline void DefaultScalingList8x8( - int i, - int scaling_list8x8[][kH264ScalingList8x8Length]) { - DCHECK_LT(i, 6); - - if (i % 2 == 0) - memcpy(scaling_list8x8[i], kDefault8x8Intra, sizeof(kDefault8x8Intra)); - else - memcpy(scaling_list8x8[i], kDefault8x8Inter, sizeof(kDefault8x8Inter)); -} - -static void FallbackScalingList4x4( - int i, - const int default_scaling_list_intra[], - const int default_scaling_list_inter[], - int scaling_list4x4[][kH264ScalingList4x4Length]) { - static const int kScalingList4x4ByteSize = - sizeof(scaling_list4x4[0][0]) * kH264ScalingList4x4Length; - - switch (i) { - case 0: - memcpy(scaling_list4x4[i], default_scaling_list_intra, - kScalingList4x4ByteSize); - break; - - case 1: - memcpy(scaling_list4x4[i], scaling_list4x4[0], kScalingList4x4ByteSize); - break; - - case 2: - memcpy(scaling_list4x4[i], scaling_list4x4[1], kScalingList4x4ByteSize); - break; - - case 3: - memcpy(scaling_list4x4[i], default_scaling_list_inter, - kScalingList4x4ByteSize); - break; - - case 4: - memcpy(scaling_list4x4[i], scaling_list4x4[3], kScalingList4x4ByteSize); - break; - - case 5: - memcpy(scaling_list4x4[i], scaling_list4x4[4], kScalingList4x4ByteSize); - break; - - default: - NOTREACHED(); - break; - } -} - -static void FallbackScalingList8x8( - int i, - const int default_scaling_list_intra[], - const int default_scaling_list_inter[], - int scaling_list8x8[][kH264ScalingList8x8Length]) { - static const int kScalingList8x8ByteSize = - sizeof(scaling_list8x8[0][0]) * kH264ScalingList8x8Length; - - switch (i) { - case 0: - memcpy(scaling_list8x8[i], default_scaling_list_intra, - kScalingList8x8ByteSize); - break; - - case 1: - memcpy(scaling_list8x8[i], default_scaling_list_inter, - kScalingList8x8ByteSize); - break; - - case 2: - memcpy(scaling_list8x8[i], scaling_list8x8[0], kScalingList8x8ByteSize); - break; - - case 3: - memcpy(scaling_list8x8[i], scaling_list8x8[1], kScalingList8x8ByteSize); - break; - - case 4: - memcpy(scaling_list8x8[i], scaling_list8x8[2], kScalingList8x8ByteSize); - break; - - case 5: - memcpy(scaling_list8x8[i], scaling_list8x8[3], kScalingList8x8ByteSize); - break; - - default: - NOTREACHED(); - break; - } -} - -H264Parser::Result H264Parser::ParseScalingList(int size, - int* scaling_list, - bool* use_default) { - // See chapter 7.3.2.1.1.1. - int last_scale = 8; - int next_scale = 8; - int delta_scale; - - *use_default = false; - - for (int j = 0; j < size; ++j) { - if (next_scale != 0) { - READ_SE_OR_RETURN(&delta_scale); - IN_RANGE_OR_RETURN(delta_scale, -128, 127); - next_scale = (last_scale + delta_scale + 256) & 0xff; - - if (j == 0 && next_scale == 0) { - *use_default = true; - return kOk; - } - } - - scaling_list[j] = (next_scale == 0) ? last_scale : next_scale; - last_scale = scaling_list[j]; - } - - return kOk; -} - -H264Parser::Result H264Parser::ParseSPSScalingLists(H264SPS* sps) { - // See 7.4.2.1.1. - bool seq_scaling_list_present_flag; - bool use_default; - Result res; - - // Parse scaling_list4x4. - for (int i = 0; i < 6; ++i) { - READ_BOOL_OR_RETURN(&seq_scaling_list_present_flag); - - if (seq_scaling_list_present_flag) { - res = ParseScalingList(base::size(sps->scaling_list4x4[i]), - sps->scaling_list4x4[i], &use_default); - if (res != kOk) - return res; - - if (use_default) - DefaultScalingList4x4(i, sps->scaling_list4x4); - - } else { - FallbackScalingList4x4(i, kDefault4x4Intra, kDefault4x4Inter, - sps->scaling_list4x4); - } - } - - // Parse scaling_list8x8. - for (int i = 0; i < ((sps->chroma_format_idc != 3) ? 2 : 6); ++i) { - READ_BOOL_OR_RETURN(&seq_scaling_list_present_flag); - - if (seq_scaling_list_present_flag) { - res = ParseScalingList(base::size(sps->scaling_list8x8[i]), - sps->scaling_list8x8[i], &use_default); - if (res != kOk) - return res; - - if (use_default) - DefaultScalingList8x8(i, sps->scaling_list8x8); - - } else { - FallbackScalingList8x8(i, kDefault8x8Intra, kDefault8x8Inter, - sps->scaling_list8x8); - } - } - - return kOk; -} - -H264Parser::Result H264Parser::ParsePPSScalingLists(const H264SPS& sps, - H264PPS* pps) { - // See 7.4.2.2. - bool pic_scaling_list_present_flag; - bool use_default; - Result res; - - for (int i = 0; i < 6; ++i) { - READ_BOOL_OR_RETURN(&pic_scaling_list_present_flag); - - if (pic_scaling_list_present_flag) { - res = ParseScalingList(base::size(pps->scaling_list4x4[i]), - pps->scaling_list4x4[i], &use_default); - if (res != kOk) - return res; - - if (use_default) - DefaultScalingList4x4(i, pps->scaling_list4x4); - - } else { - if (!sps.seq_scaling_matrix_present_flag) { - // Table 7-2 fallback rule A in spec. - FallbackScalingList4x4(i, kDefault4x4Intra, kDefault4x4Inter, - pps->scaling_list4x4); - } else { - // Table 7-2 fallback rule B in spec. - FallbackScalingList4x4(i, sps.scaling_list4x4[0], - sps.scaling_list4x4[3], pps->scaling_list4x4); - } - } - } - - if (pps->transform_8x8_mode_flag) { - for (int i = 0; i < ((sps.chroma_format_idc != 3) ? 2 : 6); ++i) { - READ_BOOL_OR_RETURN(&pic_scaling_list_present_flag); - - if (pic_scaling_list_present_flag) { - res = ParseScalingList(base::size(pps->scaling_list8x8[i]), - pps->scaling_list8x8[i], &use_default); - if (res != kOk) - return res; - - if (use_default) - DefaultScalingList8x8(i, pps->scaling_list8x8); - - } else { - if (!sps.seq_scaling_matrix_present_flag) { - // Table 7-2 fallback rule A in spec. - FallbackScalingList8x8(i, kDefault8x8Intra, kDefault8x8Inter, - pps->scaling_list8x8); - } else { - // Table 7-2 fallback rule B in spec. - FallbackScalingList8x8(i, sps.scaling_list8x8[0], - sps.scaling_list8x8[1], pps->scaling_list8x8); - } - } - } - } - return kOk; -} - -H264Parser::Result H264Parser::ParseAndIgnoreHRDParameters( - bool* hrd_parameters_present) { - int data; - READ_BOOL_OR_RETURN(&data); // {nal,vcl}_hrd_parameters_present_flag - if (!data) - return kOk; - - *hrd_parameters_present = true; - - int cpb_cnt_minus1; - READ_UE_OR_RETURN(&cpb_cnt_minus1); - IN_RANGE_OR_RETURN(cpb_cnt_minus1, 0, 31); - READ_BITS_OR_RETURN(8, &data); // bit_rate_scale, cpb_size_scale - for (int i = 0; i <= cpb_cnt_minus1; ++i) { - READ_UE_OR_RETURN(&data); // bit_rate_value_minus1[i] - READ_UE_OR_RETURN(&data); // cpb_size_value_minus1[i] - READ_BOOL_OR_RETURN(&data); // cbr_flag - } - READ_BITS_OR_RETURN(20, &data); // cpb/dpb delays, etc. - - return kOk; -} - -H264Parser::Result H264Parser::ParseVUIParameters(H264SPS* sps) { - bool aspect_ratio_info_present_flag; - READ_BOOL_OR_RETURN(&aspect_ratio_info_present_flag); - if (aspect_ratio_info_present_flag) { - int aspect_ratio_idc; - READ_BITS_OR_RETURN(8, &aspect_ratio_idc); - if (aspect_ratio_idc == H264SPS::kExtendedSar) { - READ_BITS_OR_RETURN(16, &sps->sar_width); - READ_BITS_OR_RETURN(16, &sps->sar_height); - } else { - const int max_aspect_ratio_idc = base::size(kTableSarWidth) - 1; - IN_RANGE_OR_RETURN(aspect_ratio_idc, 0, max_aspect_ratio_idc); - sps->sar_width = kTableSarWidth[aspect_ratio_idc]; - sps->sar_height = kTableSarHeight[aspect_ratio_idc]; - } - } - - int data; - // Read and ignore overscan and video signal type info. - READ_BOOL_OR_RETURN(&data); // overscan_info_present_flag - if (data) - READ_BOOL_OR_RETURN(&data); // overscan_appropriate_flag - - READ_BOOL_OR_RETURN(&sps->video_signal_type_present_flag); - if (sps->video_signal_type_present_flag) { - READ_BITS_OR_RETURN(3, &sps->video_format); - READ_BOOL_OR_RETURN(&sps->video_full_range_flag); - READ_BOOL_OR_RETURN(&sps->colour_description_present_flag); - if (sps->colour_description_present_flag) { - // color description syntax elements - READ_BITS_OR_RETURN(8, &sps->colour_primaries); - READ_BITS_OR_RETURN(8, &sps->transfer_characteristics); - READ_BITS_OR_RETURN(8, &sps->matrix_coefficients); - } - } - - READ_BOOL_OR_RETURN(&data); // chroma_loc_info_present_flag - if (data) { - READ_UE_OR_RETURN(&data); // chroma_sample_loc_type_top_field - READ_UE_OR_RETURN(&data); // chroma_sample_loc_type_bottom_field - } - - // Read and ignore timing info. - READ_BOOL_OR_RETURN(&data); // timing_info_present_flag - if (data) { - READ_BITS_OR_RETURN(16, &data); // num_units_in_tick - READ_BITS_OR_RETURN(16, &data); // num_units_in_tick - READ_BITS_OR_RETURN(16, &data); // time_scale - READ_BITS_OR_RETURN(16, &data); // time_scale - READ_BOOL_OR_RETURN(&data); // fixed_frame_rate_flag - } - - // Read and ignore NAL HRD parameters, if present. - bool hrd_parameters_present = false; - Result res = ParseAndIgnoreHRDParameters(&hrd_parameters_present); - if (res != kOk) - return res; - - // Read and ignore VCL HRD parameters, if present. - res = ParseAndIgnoreHRDParameters(&hrd_parameters_present); - if (res != kOk) - return res; - - if (hrd_parameters_present) // One of NAL or VCL params present is enough. - READ_BOOL_OR_RETURN(&data); // low_delay_hrd_flag - - READ_BOOL_OR_RETURN(&data); // pic_struct_present_flag - READ_BOOL_OR_RETURN(&sps->bitstream_restriction_flag); - if (sps->bitstream_restriction_flag) { - READ_BOOL_OR_RETURN(&data); // motion_vectors_over_pic_boundaries_flag - READ_UE_OR_RETURN(&data); // max_bytes_per_pic_denom - READ_UE_OR_RETURN(&data); // max_bits_per_mb_denom - READ_UE_OR_RETURN(&data); // log2_max_mv_length_horizontal - READ_UE_OR_RETURN(&data); // log2_max_mv_length_vertical - READ_UE_OR_RETURN(&sps->max_num_reorder_frames); - READ_UE_OR_RETURN(&sps->max_dec_frame_buffering); - TRUE_OR_RETURN(sps->max_dec_frame_buffering >= sps->max_num_ref_frames); - IN_RANGE_OR_RETURN(sps->max_num_reorder_frames, 0, - sps->max_dec_frame_buffering); - } - - return kOk; -} - -static void FillDefaultSeqScalingLists(H264SPS* sps) { - for (int i = 0; i < 6; ++i) - for (int j = 0; j < kH264ScalingList4x4Length; ++j) - sps->scaling_list4x4[i][j] = 16; - - for (int i = 0; i < 6; ++i) - for (int j = 0; j < kH264ScalingList8x8Length; ++j) - sps->scaling_list8x8[i][j] = 16; -} - -H264Parser::Result H264Parser::ParseSPS(int* sps_id) { - // See 7.4.2.1. - int data; - Result res; - - *sps_id = -1; - - std::unique_ptr<H264SPS> sps(new H264SPS()); - - READ_BITS_OR_RETURN(8, &sps->profile_idc); - READ_BOOL_OR_RETURN(&sps->constraint_set0_flag); - READ_BOOL_OR_RETURN(&sps->constraint_set1_flag); - READ_BOOL_OR_RETURN(&sps->constraint_set2_flag); - READ_BOOL_OR_RETURN(&sps->constraint_set3_flag); - READ_BOOL_OR_RETURN(&sps->constraint_set4_flag); - READ_BOOL_OR_RETURN(&sps->constraint_set5_flag); - READ_BITS_OR_RETURN(2, &data); // reserved_zero_2bits - READ_BITS_OR_RETURN(8, &sps->level_idc); - READ_UE_OR_RETURN(&sps->seq_parameter_set_id); - TRUE_OR_RETURN(sps->seq_parameter_set_id < 32); - - if (sps->profile_idc == 100 || sps->profile_idc == 110 || - sps->profile_idc == 122 || sps->profile_idc == 244 || - sps->profile_idc == 44 || sps->profile_idc == 83 || - sps->profile_idc == 86 || sps->profile_idc == 118 || - sps->profile_idc == 128) { - READ_UE_OR_RETURN(&sps->chroma_format_idc); - TRUE_OR_RETURN(sps->chroma_format_idc < 4); - - if (sps->chroma_format_idc == 3) - READ_BOOL_OR_RETURN(&sps->separate_colour_plane_flag); - - READ_UE_OR_RETURN(&sps->bit_depth_luma_minus8); - TRUE_OR_RETURN(sps->bit_depth_luma_minus8 < 7); - - READ_UE_OR_RETURN(&sps->bit_depth_chroma_minus8); - TRUE_OR_RETURN(sps->bit_depth_chroma_minus8 < 7); - - READ_BOOL_OR_RETURN(&sps->qpprime_y_zero_transform_bypass_flag); - READ_BOOL_OR_RETURN(&sps->seq_scaling_matrix_present_flag); - - if (sps->seq_scaling_matrix_present_flag) { - DVLOG(4) << "Scaling matrix present"; - res = ParseSPSScalingLists(sps.get()); - if (res != kOk) - return res; - } else { - FillDefaultSeqScalingLists(sps.get()); - } - } else { - sps->chroma_format_idc = 1; - FillDefaultSeqScalingLists(sps.get()); - } - - if (sps->separate_colour_plane_flag) - sps->chroma_array_type = 0; - else - sps->chroma_array_type = sps->chroma_format_idc; - - READ_UE_OR_RETURN(&sps->log2_max_frame_num_minus4); - TRUE_OR_RETURN(sps->log2_max_frame_num_minus4 < 13); - - READ_UE_OR_RETURN(&sps->pic_order_cnt_type); - TRUE_OR_RETURN(sps->pic_order_cnt_type < 3); - - if (sps->pic_order_cnt_type == 0) { - READ_UE_OR_RETURN(&sps->log2_max_pic_order_cnt_lsb_minus4); - TRUE_OR_RETURN(sps->log2_max_pic_order_cnt_lsb_minus4 < 13); - sps->expected_delta_per_pic_order_cnt_cycle = 0; - } else if (sps->pic_order_cnt_type == 1) { - READ_BOOL_OR_RETURN(&sps->delta_pic_order_always_zero_flag); - READ_SE_OR_RETURN(&sps->offset_for_non_ref_pic); - READ_SE_OR_RETURN(&sps->offset_for_top_to_bottom_field); - READ_UE_OR_RETURN(&sps->num_ref_frames_in_pic_order_cnt_cycle); - TRUE_OR_RETURN(sps->num_ref_frames_in_pic_order_cnt_cycle < 255); - - base::CheckedNumeric<int> offset_acc = 0; - for (int i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; ++i) { - READ_SE_OR_RETURN(&sps->offset_for_ref_frame[i]); - offset_acc += sps->offset_for_ref_frame[i]; - } - if (!offset_acc.IsValid()) - return kInvalidStream; - sps->expected_delta_per_pic_order_cnt_cycle = offset_acc.ValueOrDefault(0); - } - - READ_UE_OR_RETURN(&sps->max_num_ref_frames); - READ_BOOL_OR_RETURN(&sps->gaps_in_frame_num_value_allowed_flag); - - READ_UE_OR_RETURN(&sps->pic_width_in_mbs_minus1); - READ_UE_OR_RETURN(&sps->pic_height_in_map_units_minus1); - - READ_BOOL_OR_RETURN(&sps->frame_mbs_only_flag); - if (!sps->frame_mbs_only_flag) - READ_BOOL_OR_RETURN(&sps->mb_adaptive_frame_field_flag); - - READ_BOOL_OR_RETURN(&sps->direct_8x8_inference_flag); - - READ_BOOL_OR_RETURN(&sps->frame_cropping_flag); - if (sps->frame_cropping_flag) { - READ_UE_OR_RETURN(&sps->frame_crop_left_offset); - READ_UE_OR_RETURN(&sps->frame_crop_right_offset); - READ_UE_OR_RETURN(&sps->frame_crop_top_offset); - READ_UE_OR_RETURN(&sps->frame_crop_bottom_offset); - } - - READ_BOOL_OR_RETURN(&sps->vui_parameters_present_flag); - if (sps->vui_parameters_present_flag) { - DVLOG(4) << "VUI parameters present"; - res = ParseVUIParameters(sps.get()); - if (res != kOk) - return res; - } - - // If an SPS with the same id already exists, replace it. - *sps_id = sps->seq_parameter_set_id; - active_SPSes_[*sps_id] = std::move(sps); - - return kOk; -} - -H264Parser::Result H264Parser::ParsePPS(int* pps_id) { - // See 7.4.2.2. - const H264SPS* sps; - Result res; - - *pps_id = -1; - - std::unique_ptr<H264PPS> pps(new H264PPS()); - - READ_UE_OR_RETURN(&pps->pic_parameter_set_id); - READ_UE_OR_RETURN(&pps->seq_parameter_set_id); - TRUE_OR_RETURN(pps->seq_parameter_set_id < 32); - - if (active_SPSes_.find(pps->seq_parameter_set_id) == active_SPSes_.end()) { - DVLOG(1) << "Invalid stream, no SPS id: " << pps->seq_parameter_set_id; - return kInvalidStream; - } - - sps = GetSPS(pps->seq_parameter_set_id); - TRUE_OR_RETURN(sps); - - READ_BOOL_OR_RETURN(&pps->entropy_coding_mode_flag); - READ_BOOL_OR_RETURN(&pps->bottom_field_pic_order_in_frame_present_flag); - - READ_UE_OR_RETURN(&pps->num_slice_groups_minus1); - if (pps->num_slice_groups_minus1 > 1) { - DVLOG(1) << "Slice groups not supported"; - return kUnsupportedStream; - } - - READ_UE_OR_RETURN(&pps->num_ref_idx_l0_default_active_minus1); - TRUE_OR_RETURN(pps->num_ref_idx_l0_default_active_minus1 < 32); - - READ_UE_OR_RETURN(&pps->num_ref_idx_l1_default_active_minus1); - TRUE_OR_RETURN(pps->num_ref_idx_l1_default_active_minus1 < 32); - - READ_BOOL_OR_RETURN(&pps->weighted_pred_flag); - READ_BITS_OR_RETURN(2, &pps->weighted_bipred_idc); - TRUE_OR_RETURN(pps->weighted_bipred_idc < 3); - - READ_SE_OR_RETURN(&pps->pic_init_qp_minus26); - IN_RANGE_OR_RETURN(pps->pic_init_qp_minus26, -26, 25); - - READ_SE_OR_RETURN(&pps->pic_init_qs_minus26); - IN_RANGE_OR_RETURN(pps->pic_init_qs_minus26, -26, 25); - - READ_SE_OR_RETURN(&pps->chroma_qp_index_offset); - IN_RANGE_OR_RETURN(pps->chroma_qp_index_offset, -12, 12); - pps->second_chroma_qp_index_offset = pps->chroma_qp_index_offset; - - READ_BOOL_OR_RETURN(&pps->deblocking_filter_control_present_flag); - READ_BOOL_OR_RETURN(&pps->constrained_intra_pred_flag); - READ_BOOL_OR_RETURN(&pps->redundant_pic_cnt_present_flag); - - if (br_.HasMoreRBSPData()) { - READ_BOOL_OR_RETURN(&pps->transform_8x8_mode_flag); - READ_BOOL_OR_RETURN(&pps->pic_scaling_matrix_present_flag); - - if (pps->pic_scaling_matrix_present_flag) { - DVLOG(4) << "Picture scaling matrix present"; - res = ParsePPSScalingLists(*sps, pps.get()); - if (res != kOk) - return res; - } - - READ_SE_OR_RETURN(&pps->second_chroma_qp_index_offset); - } - - // If a PPS with the same id already exists, replace it. - *pps_id = pps->pic_parameter_set_id; - active_PPSes_[*pps_id] = std::move(pps); - - return kOk; -} - -H264Parser::Result H264Parser::ParseSPSExt(int* sps_id) { - // See 7.4.2.1. - int local_sps_id = -1; - - *sps_id = -1; - - READ_UE_OR_RETURN(&local_sps_id); - TRUE_OR_RETURN(local_sps_id < 32); - - *sps_id = local_sps_id; - return kOk; -} - -H264Parser::Result H264Parser::ParseRefPicListModification( - int num_ref_idx_active_minus1, - H264ModificationOfPicNum* ref_list_mods) { - H264ModificationOfPicNum* pic_num_mod; - - if (num_ref_idx_active_minus1 >= 32) - return kInvalidStream; - - for (int i = 0; i < 32; ++i) { - pic_num_mod = &ref_list_mods[i]; - READ_UE_OR_RETURN(&pic_num_mod->modification_of_pic_nums_idc); - TRUE_OR_RETURN(pic_num_mod->modification_of_pic_nums_idc < 4); - - switch (pic_num_mod->modification_of_pic_nums_idc) { - case 0: - case 1: - READ_UE_OR_RETURN(&pic_num_mod->abs_diff_pic_num_minus1); - break; - - case 2: - READ_UE_OR_RETURN(&pic_num_mod->long_term_pic_num); - break; - - case 3: - // Per spec, list cannot be empty. - if (i == 0) - return kInvalidStream; - return kOk; - - default: - return kInvalidStream; - } - } - - // If we got here, we didn't get loop end marker prematurely, - // so make sure it is there for our client. - int modification_of_pic_nums_idc; - READ_UE_OR_RETURN(&modification_of_pic_nums_idc); - TRUE_OR_RETURN(modification_of_pic_nums_idc == 3); - - return kOk; -} - -H264Parser::Result H264Parser::ParseRefPicListModifications( - H264SliceHeader* shdr) { - Result res; - - if (!shdr->IsISlice() && !shdr->IsSISlice()) { - READ_BOOL_OR_RETURN(&shdr->ref_pic_list_modification_flag_l0); - if (shdr->ref_pic_list_modification_flag_l0) { - res = ParseRefPicListModification(shdr->num_ref_idx_l0_active_minus1, - shdr->ref_list_l0_modifications); - if (res != kOk) - return res; - } - } - - if (shdr->IsBSlice()) { - READ_BOOL_OR_RETURN(&shdr->ref_pic_list_modification_flag_l1); - if (shdr->ref_pic_list_modification_flag_l1) { - res = ParseRefPicListModification(shdr->num_ref_idx_l1_active_minus1, - shdr->ref_list_l1_modifications); - if (res != kOk) - return res; - } - } - - return kOk; -} - -H264Parser::Result H264Parser::ParseWeightingFactors( - int num_ref_idx_active_minus1, - int chroma_array_type, - int luma_log2_weight_denom, - int chroma_log2_weight_denom, - H264WeightingFactors* w_facts) { - int def_luma_weight = 1 << luma_log2_weight_denom; - int def_chroma_weight = 1 << chroma_log2_weight_denom; - - for (int i = 0; i < num_ref_idx_active_minus1 + 1; ++i) { - READ_BOOL_OR_RETURN(&w_facts->luma_weight_flag); - if (w_facts->luma_weight_flag) { - READ_SE_OR_RETURN(&w_facts->luma_weight[i]); - IN_RANGE_OR_RETURN(w_facts->luma_weight[i], -128, 127); - - READ_SE_OR_RETURN(&w_facts->luma_offset[i]); - IN_RANGE_OR_RETURN(w_facts->luma_offset[i], -128, 127); - } else { - w_facts->luma_weight[i] = def_luma_weight; - w_facts->luma_offset[i] = 0; - } - - if (chroma_array_type != 0) { - READ_BOOL_OR_RETURN(&w_facts->chroma_weight_flag); - if (w_facts->chroma_weight_flag) { - for (int j = 0; j < 2; ++j) { - READ_SE_OR_RETURN(&w_facts->chroma_weight[i][j]); - IN_RANGE_OR_RETURN(w_facts->chroma_weight[i][j], -128, 127); - - READ_SE_OR_RETURN(&w_facts->chroma_offset[i][j]); - IN_RANGE_OR_RETURN(w_facts->chroma_offset[i][j], -128, 127); - } - } else { - for (int j = 0; j < 2; ++j) { - w_facts->chroma_weight[i][j] = def_chroma_weight; - w_facts->chroma_offset[i][j] = 0; - } - } - } - } - - return kOk; -} - -H264Parser::Result H264Parser::ParsePredWeightTable(const H264SPS& sps, - H264SliceHeader* shdr) { - READ_UE_OR_RETURN(&shdr->luma_log2_weight_denom); - TRUE_OR_RETURN(shdr->luma_log2_weight_denom < 8); - - if (sps.chroma_array_type != 0) - READ_UE_OR_RETURN(&shdr->chroma_log2_weight_denom); - TRUE_OR_RETURN(shdr->chroma_log2_weight_denom < 8); - - Result res = ParseWeightingFactors( - shdr->num_ref_idx_l0_active_minus1, sps.chroma_array_type, - shdr->luma_log2_weight_denom, shdr->chroma_log2_weight_denom, - &shdr->pred_weight_table_l0); - if (res != kOk) - return res; - - if (shdr->IsBSlice()) { - res = ParseWeightingFactors( - shdr->num_ref_idx_l1_active_minus1, sps.chroma_array_type, - shdr->luma_log2_weight_denom, shdr->chroma_log2_weight_denom, - &shdr->pred_weight_table_l1); - if (res != kOk) - return res; - } - - return kOk; -} - -H264Parser::Result H264Parser::ParseDecRefPicMarking(H264SliceHeader* shdr) { - size_t bits_left_at_start = br_.NumBitsLeft(); - - if (shdr->idr_pic_flag) { - READ_BOOL_OR_RETURN(&shdr->no_output_of_prior_pics_flag); - READ_BOOL_OR_RETURN(&shdr->long_term_reference_flag); - } else { - READ_BOOL_OR_RETURN(&shdr->adaptive_ref_pic_marking_mode_flag); - - H264DecRefPicMarking* marking; - if (shdr->adaptive_ref_pic_marking_mode_flag) { - size_t i; - for (i = 0; i < base::size(shdr->ref_pic_marking); ++i) { - marking = &shdr->ref_pic_marking[i]; - - READ_UE_OR_RETURN(&marking->memory_mgmnt_control_operation); - if (marking->memory_mgmnt_control_operation == 0) - break; - - if (marking->memory_mgmnt_control_operation == 1 || - marking->memory_mgmnt_control_operation == 3) - READ_UE_OR_RETURN(&marking->difference_of_pic_nums_minus1); - - if (marking->memory_mgmnt_control_operation == 2) - READ_UE_OR_RETURN(&marking->long_term_pic_num); - - if (marking->memory_mgmnt_control_operation == 3 || - marking->memory_mgmnt_control_operation == 6) - READ_UE_OR_RETURN(&marking->long_term_frame_idx); - - if (marking->memory_mgmnt_control_operation == 4) - READ_UE_OR_RETURN(&marking->max_long_term_frame_idx_plus1); - - if (marking->memory_mgmnt_control_operation > 6) - return kInvalidStream; - } - - if (i == base::size(shdr->ref_pic_marking)) { - DVLOG(1) << "Ran out of dec ref pic marking fields"; - return kUnsupportedStream; - } - } - } - - shdr->dec_ref_pic_marking_bit_size = bits_left_at_start - br_.NumBitsLeft(); - return kOk; -} - -H264Parser::Result H264Parser::ParseSliceHeader(const H264NALU& nalu, - H264SliceHeader* shdr) { - // See 7.4.3. - const H264SPS* sps; - const H264PPS* pps; - Result res; - - memset(shdr, 0, sizeof(*shdr)); - - shdr->idr_pic_flag = (nalu.nal_unit_type == 5); - shdr->nal_ref_idc = nalu.nal_ref_idc; - shdr->nalu_data = nalu.data; - shdr->nalu_size = nalu.size; - - READ_UE_OR_RETURN(&shdr->first_mb_in_slice); - READ_UE_OR_RETURN(&shdr->slice_type); - TRUE_OR_RETURN(shdr->slice_type < 10); - - READ_UE_OR_RETURN(&shdr->pic_parameter_set_id); - - pps = GetPPS(shdr->pic_parameter_set_id); - TRUE_OR_RETURN(pps); - - sps = GetSPS(pps->seq_parameter_set_id); - TRUE_OR_RETURN(sps); - - if (sps->separate_colour_plane_flag) { - DVLOG(1) << "Interlaced streams not supported"; - return kUnsupportedStream; - } - - READ_BITS_OR_RETURN(sps->log2_max_frame_num_minus4 + 4, &shdr->frame_num); - if (!sps->frame_mbs_only_flag) { - READ_BOOL_OR_RETURN(&shdr->field_pic_flag); - if (shdr->field_pic_flag) { - DVLOG(1) << "Interlaced streams not supported"; - return kUnsupportedStream; - } - } - - if (shdr->idr_pic_flag) - READ_UE_OR_RETURN(&shdr->idr_pic_id); - - size_t bits_left_at_pic_order_cnt_start = br_.NumBitsLeft(); - if (sps->pic_order_cnt_type == 0) { - READ_BITS_OR_RETURN(sps->log2_max_pic_order_cnt_lsb_minus4 + 4, - &shdr->pic_order_cnt_lsb); - if (pps->bottom_field_pic_order_in_frame_present_flag && - !shdr->field_pic_flag) - READ_SE_OR_RETURN(&shdr->delta_pic_order_cnt_bottom); - } - - if (sps->pic_order_cnt_type == 1 && !sps->delta_pic_order_always_zero_flag) { - READ_SE_OR_RETURN(&shdr->delta_pic_order_cnt0); - if (pps->bottom_field_pic_order_in_frame_present_flag && - !shdr->field_pic_flag) - READ_SE_OR_RETURN(&shdr->delta_pic_order_cnt1); - } - - shdr->pic_order_cnt_bit_size = - bits_left_at_pic_order_cnt_start - br_.NumBitsLeft(); - - if (pps->redundant_pic_cnt_present_flag) { - READ_UE_OR_RETURN(&shdr->redundant_pic_cnt); - TRUE_OR_RETURN(shdr->redundant_pic_cnt < 128); - } - - if (shdr->IsBSlice()) - READ_BOOL_OR_RETURN(&shdr->direct_spatial_mv_pred_flag); - - if (shdr->IsPSlice() || shdr->IsSPSlice() || shdr->IsBSlice()) { - READ_BOOL_OR_RETURN(&shdr->num_ref_idx_active_override_flag); - if (shdr->num_ref_idx_active_override_flag) { - READ_UE_OR_RETURN(&shdr->num_ref_idx_l0_active_minus1); - if (shdr->IsBSlice()) - READ_UE_OR_RETURN(&shdr->num_ref_idx_l1_active_minus1); - } else { - shdr->num_ref_idx_l0_active_minus1 = - pps->num_ref_idx_l0_default_active_minus1; - if (shdr->IsBSlice()) { - shdr->num_ref_idx_l1_active_minus1 = - pps->num_ref_idx_l1_default_active_minus1; - } - } - } - if (shdr->field_pic_flag) { - TRUE_OR_RETURN(shdr->num_ref_idx_l0_active_minus1 < 32); - TRUE_OR_RETURN(shdr->num_ref_idx_l1_active_minus1 < 32); - } else { - TRUE_OR_RETURN(shdr->num_ref_idx_l0_active_minus1 < 16); - TRUE_OR_RETURN(shdr->num_ref_idx_l1_active_minus1 < 16); - } - - if (nalu.nal_unit_type == H264NALU::kCodedSliceExtension) { - return kUnsupportedStream; - } else { - res = ParseRefPicListModifications(shdr); - if (res != kOk) - return res; - } - - if ((pps->weighted_pred_flag && (shdr->IsPSlice() || shdr->IsSPSlice())) || - (pps->weighted_bipred_idc == 1 && shdr->IsBSlice())) { - res = ParsePredWeightTable(*sps, shdr); - if (res != kOk) - return res; - } - - if (nalu.nal_ref_idc != 0) { - res = ParseDecRefPicMarking(shdr); - if (res != kOk) - return res; - } - - if (pps->entropy_coding_mode_flag && !shdr->IsISlice() && - !shdr->IsSISlice()) { - READ_UE_OR_RETURN(&shdr->cabac_init_idc); - TRUE_OR_RETURN(shdr->cabac_init_idc < 3); - } - - READ_SE_OR_RETURN(&shdr->slice_qp_delta); - - if (shdr->IsSPSlice() || shdr->IsSISlice()) { - if (shdr->IsSPSlice()) - READ_BOOL_OR_RETURN(&shdr->sp_for_switch_flag); - READ_SE_OR_RETURN(&shdr->slice_qs_delta); - } - - if (pps->deblocking_filter_control_present_flag) { - READ_UE_OR_RETURN(&shdr->disable_deblocking_filter_idc); - TRUE_OR_RETURN(shdr->disable_deblocking_filter_idc < 3); - - if (shdr->disable_deblocking_filter_idc != 1) { - READ_SE_OR_RETURN(&shdr->slice_alpha_c0_offset_div2); - IN_RANGE_OR_RETURN(shdr->slice_alpha_c0_offset_div2, -6, 6); - - READ_SE_OR_RETURN(&shdr->slice_beta_offset_div2); - IN_RANGE_OR_RETURN(shdr->slice_beta_offset_div2, -6, 6); - } - } - - if (pps->num_slice_groups_minus1 > 0) { - DVLOG(1) << "Slice groups not supported"; - return kUnsupportedStream; - } - - size_t epb = br_.NumEmulationPreventionBytesRead(); - shdr->header_bit_size = (shdr->nalu_size - epb) * 8 - br_.NumBitsLeft(); - - return kOk; -} - -H264Parser::Result H264Parser::ParseSEI(H264SEIMessage* sei_msg) { - int byte; - - memset(sei_msg, 0, sizeof(*sei_msg)); - - READ_BITS_OR_RETURN(8, &byte); - while (byte == 0xff) { - sei_msg->type += 255; - READ_BITS_OR_RETURN(8, &byte); - } - sei_msg->type += byte; - - READ_BITS_OR_RETURN(8, &byte); - while (byte == 0xff) { - sei_msg->payload_size += 255; - READ_BITS_OR_RETURN(8, &byte); - } - sei_msg->payload_size += byte; - - DVLOG(4) << "Found SEI message type: " << sei_msg->type - << " payload size: " << sei_msg->payload_size; - - switch (sei_msg->type) { - case H264SEIMessage::kSEIRecoveryPoint: - READ_UE_OR_RETURN(&sei_msg->recovery_point.recovery_frame_cnt); - READ_BOOL_OR_RETURN(&sei_msg->recovery_point.exact_match_flag); - READ_BOOL_OR_RETURN(&sei_msg->recovery_point.broken_link_flag); - READ_BITS_OR_RETURN(2, &sei_msg->recovery_point.changing_slice_group_idc); - break; - - default: - DVLOG(4) << "Unsupported SEI message"; - break; - } - - return kOk; -} - -std::vector<SubsampleEntry> H264Parser::GetCurrentSubsamples() { - DCHECK_EQ(previous_nalu_range_.size(), 1u) - << "This should only be called after a " - "successful call to AdvanceToNextNalu()"; - - auto intersection = encrypted_ranges_.IntersectionWith(previous_nalu_range_); - return EncryptedRangesToSubsampleEntry( - previous_nalu_range_.start(0), previous_nalu_range_.end(0), intersection); -} - -} // namespace media diff --git a/accel/h264_parser.h b/accel/h264_parser.h deleted file mode 100644 index 9db53e7..0000000 --- a/accel/h264_parser.h +++ /dev/null @@ -1,563 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of an H264 Annex-B video stream parser. -// Note: ported from Chromium commit head: 600904374759 -// Note: GetColorSpace() is not ported. - -#ifndef H264_PARSER_H_ -#define H264_PARSER_H_ - -#include <stddef.h> -#include <stdint.h> -#include <sys/types.h> - -#include <map> -#include <memory> -#include <vector> - -#include "base/macros.h" -#include "base/optional.h" -#include "h264_bit_reader.h" -#include "ranges.h" -#include "rect.h" -#include "size.h" -#include "subsample_entry.h" -#include "video_codecs.h" - -namespace media { - -struct SubsampleEntry; - -// For explanations of each struct and its members, see H.264 specification -// at http://www.itu.int/rec/T-REC-H.264. -struct H264NALU { - H264NALU(); - - enum Type { - kUnspecified = 0, - kNonIDRSlice = 1, - kSliceDataA = 2, - kSliceDataB = 3, - kSliceDataC = 4, - kIDRSlice = 5, - kSEIMessage = 6, - kSPS = 7, - kPPS = 8, - kAUD = 9, - kEOSeq = 10, - kEOStream = 11, - kFiller = 12, - kSPSExt = 13, - kReserved14 = 14, - kReserved15 = 15, - kReserved16 = 16, - kReserved17 = 17, - kReserved18 = 18, - kCodedSliceAux = 19, - kCodedSliceExtension = 20, - }; - - // After (without) start code; we don't own the underlying memory - // and a shallow copy should be made when copying this struct. - const uint8_t* data; - off_t size; // From after start code to start code of next NALU (or EOS). - - int nal_ref_idc; - int nal_unit_type; -}; - -enum { - kH264ScalingList4x4Length = 16, - kH264ScalingList8x8Length = 64, -}; - -struct H264SPS { - H264SPS(); - - enum H264ProfileIDC { - kProfileIDCBaseline = 66, - kProfileIDCConstrainedBaseline = kProfileIDCBaseline, - kProfileIDCMain = 77, - kProfileIDScalableBaseline = 83, - kProfileIDScalableHigh = 86, - kProfileIDCHigh = 100, - kProfileIDHigh10 = 110, - kProfileIDSMultiviewHigh = 118, - kProfileIDHigh422 = 122, - kProfileIDStereoHigh = 128, - kProfileIDHigh444Predictive = 244, - }; - - enum H264LevelIDC : uint8_t { - kLevelIDC1p0 = 10, - kLevelIDC1B = 9, - kLevelIDC1p1 = 11, - kLevelIDC1p2 = 12, - kLevelIDC1p3 = 13, - kLevelIDC2p0 = 20, - kLevelIDC2p1 = 21, - kLevelIDC2p2 = 22, - kLevelIDC3p0 = 30, - kLevelIDC3p1 = 31, - kLevelIDC3p2 = 32, - kLevelIDC4p0 = 40, - kLevelIDC4p1 = 41, - kLevelIDC4p2 = 42, - kLevelIDC5p0 = 50, - kLevelIDC5p1 = 51, - kLevelIDC5p2 = 52, - kLevelIDC6p0 = 60, - kLevelIDC6p1 = 61, - kLevelIDC6p2 = 62, - }; - - enum AspectRatioIdc { - kExtendedSar = 255, - }; - - enum { - // Constants for HRD parameters (spec ch. E.2.2). - kBitRateScaleConstantTerm = 6, // Equation E-37. - kCPBSizeScaleConstantTerm = 4, // Equation E-38. - kDefaultInitialCPBRemovalDelayLength = 24, - kDefaultDPBOutputDelayLength = 24, - kDefaultTimeOffsetLength = 24, - }; - - int profile_idc; - bool constraint_set0_flag; - bool constraint_set1_flag; - bool constraint_set2_flag; - bool constraint_set3_flag; - bool constraint_set4_flag; - bool constraint_set5_flag; - int level_idc; - int seq_parameter_set_id; - - int chroma_format_idc; - bool separate_colour_plane_flag; - int bit_depth_luma_minus8; - int bit_depth_chroma_minus8; - bool qpprime_y_zero_transform_bypass_flag; - - bool seq_scaling_matrix_present_flag; - int scaling_list4x4[6][kH264ScalingList4x4Length]; - int scaling_list8x8[6][kH264ScalingList8x8Length]; - - int log2_max_frame_num_minus4; - int pic_order_cnt_type; - int log2_max_pic_order_cnt_lsb_minus4; - bool delta_pic_order_always_zero_flag; - int offset_for_non_ref_pic; - int offset_for_top_to_bottom_field; - int num_ref_frames_in_pic_order_cnt_cycle; - int expected_delta_per_pic_order_cnt_cycle; // calculated - int offset_for_ref_frame[255]; - int max_num_ref_frames; - bool gaps_in_frame_num_value_allowed_flag; - int pic_width_in_mbs_minus1; - int pic_height_in_map_units_minus1; - bool frame_mbs_only_flag; - bool mb_adaptive_frame_field_flag; - bool direct_8x8_inference_flag; - bool frame_cropping_flag; - int frame_crop_left_offset; - int frame_crop_right_offset; - int frame_crop_top_offset; - int frame_crop_bottom_offset; - - bool vui_parameters_present_flag; - int sar_width; // Set to 0 when not specified. - int sar_height; // Set to 0 when not specified. - bool bitstream_restriction_flag; - int max_num_reorder_frames; - int max_dec_frame_buffering; - bool timing_info_present_flag; - int num_units_in_tick; - int time_scale; - bool fixed_frame_rate_flag; - - bool video_signal_type_present_flag; - int video_format; - bool video_full_range_flag; - bool colour_description_present_flag; - int colour_primaries; - int transfer_characteristics; - int matrix_coefficients; - - // TODO(posciak): actually parse these instead of ParseAndIgnoreHRDParameters. - bool nal_hrd_parameters_present_flag; - int cpb_cnt_minus1; - int bit_rate_scale; - int cpb_size_scale; - int bit_rate_value_minus1[32]; - int cpb_size_value_minus1[32]; - bool cbr_flag[32]; - int initial_cpb_removal_delay_length_minus_1; - int cpb_removal_delay_length_minus1; - int dpb_output_delay_length_minus1; - int time_offset_length; - - bool low_delay_hrd_flag; - - int chroma_array_type; - - // Get corresponding SPS |level_idc| and |constraint_set3_flag| value from - // requested |profile| and |level| (see Spec A.3.1). - static void GetLevelConfigFromProfileLevel(VideoCodecProfile profile, - uint8_t level, - int* level_idc, - bool* constraint_set3_flag); - - // Helpers to compute frequently-used values. These methods return - // base::nullopt if they encounter integer overflow. They do not verify that - // the results are in-spec for the given profile or level. - base::Optional<Size> GetCodedSize() const; - base::Optional<Rect> GetVisibleRect() const; - - // Helper to compute indicated level from parsed SPS data. The value of - // indicated level would be included in H264LevelIDC enum representing the - // level as in name. - uint8_t GetIndicatedLevel() const; - // Helper to check if indicated level is lower than or equal to - // |target_level|. - bool CheckIndicatedLevelWithinTarget(uint8_t target_level) const; -}; - -struct H264PPS { - H264PPS(); - - int pic_parameter_set_id; - int seq_parameter_set_id; - bool entropy_coding_mode_flag; - bool bottom_field_pic_order_in_frame_present_flag; - int num_slice_groups_minus1; - // TODO(posciak): Slice groups not implemented, could be added at some point. - int num_ref_idx_l0_default_active_minus1; - int num_ref_idx_l1_default_active_minus1; - bool weighted_pred_flag; - int weighted_bipred_idc; - int pic_init_qp_minus26; - int pic_init_qs_minus26; - int chroma_qp_index_offset; - bool deblocking_filter_control_present_flag; - bool constrained_intra_pred_flag; - bool redundant_pic_cnt_present_flag; - bool transform_8x8_mode_flag; - - bool pic_scaling_matrix_present_flag; - int scaling_list4x4[6][kH264ScalingList4x4Length]; - int scaling_list8x8[6][kH264ScalingList8x8Length]; - - int second_chroma_qp_index_offset; -}; - -struct H264ModificationOfPicNum { - int modification_of_pic_nums_idc; - union { - int abs_diff_pic_num_minus1; - int long_term_pic_num; - }; -}; - -struct H264WeightingFactors { - bool luma_weight_flag; - bool chroma_weight_flag; - int luma_weight[32]; - int luma_offset[32]; - int chroma_weight[32][2]; - int chroma_offset[32][2]; -}; - -struct H264DecRefPicMarking { - int memory_mgmnt_control_operation; - int difference_of_pic_nums_minus1; - int long_term_pic_num; - int long_term_frame_idx; - int max_long_term_frame_idx_plus1; -}; - -struct H264SliceHeader { - H264SliceHeader(); - - enum { kRefListSize = 32, kRefListModSize = kRefListSize }; - - enum Type { - kPSlice = 0, - kBSlice = 1, - kISlice = 2, - kSPSlice = 3, - kSISlice = 4, - }; - - bool IsPSlice() const; - bool IsBSlice() const; - bool IsISlice() const; - bool IsSPSlice() const; - bool IsSISlice() const; - - bool idr_pic_flag; // from NAL header - int nal_ref_idc; // from NAL header - const uint8_t* nalu_data; // from NAL header - off_t nalu_size; // from NAL header - off_t header_bit_size; // calculated - - int first_mb_in_slice; - int slice_type; - int pic_parameter_set_id; - int colour_plane_id; // TODO(posciak): use this! http://crbug.com/139878 - int frame_num; - bool field_pic_flag; - bool bottom_field_flag; - int idr_pic_id; - int pic_order_cnt_lsb; - int delta_pic_order_cnt_bottom; - int delta_pic_order_cnt0; - int delta_pic_order_cnt1; - int redundant_pic_cnt; - bool direct_spatial_mv_pred_flag; - - bool num_ref_idx_active_override_flag; - int num_ref_idx_l0_active_minus1; - int num_ref_idx_l1_active_minus1; - bool ref_pic_list_modification_flag_l0; - bool ref_pic_list_modification_flag_l1; - H264ModificationOfPicNum ref_list_l0_modifications[kRefListModSize]; - H264ModificationOfPicNum ref_list_l1_modifications[kRefListModSize]; - - int luma_log2_weight_denom; - int chroma_log2_weight_denom; - - bool luma_weight_l0_flag; - bool chroma_weight_l0_flag; - H264WeightingFactors pred_weight_table_l0; - - bool luma_weight_l1_flag; - bool chroma_weight_l1_flag; - H264WeightingFactors pred_weight_table_l1; - - bool no_output_of_prior_pics_flag; - bool long_term_reference_flag; - - bool adaptive_ref_pic_marking_mode_flag; - H264DecRefPicMarking ref_pic_marking[kRefListSize]; - - int cabac_init_idc; - int slice_qp_delta; - bool sp_for_switch_flag; - int slice_qs_delta; - int disable_deblocking_filter_idc; - int slice_alpha_c0_offset_div2; - int slice_beta_offset_div2; - - // Calculated. - // Size in bits of dec_ref_pic_marking() syntax element. - size_t dec_ref_pic_marking_bit_size; - size_t pic_order_cnt_bit_size; -}; - -struct H264SEIRecoveryPoint { - int recovery_frame_cnt; - bool exact_match_flag; - bool broken_link_flag; - int changing_slice_group_idc; -}; - -struct H264SEIMessage { - H264SEIMessage(); - - enum Type { - kSEIRecoveryPoint = 6, - }; - - int type; - int payload_size; - union { - // Placeholder; in future more supported types will contribute to more - // union members here. - H264SEIRecoveryPoint recovery_point; - }; -}; - -// Class to parse an Annex-B H.264 stream, -// as specified in chapters 7 and Annex B of the H.264 spec. -class H264Parser { - public: - enum Result { - kOk, - kInvalidStream, // error in stream - kUnsupportedStream, // stream not supported by the parser - kEOStream, // end of stream - }; - - // Find offset from start of data to next NALU start code - // and size of found start code (3 or 4 bytes). - // If no start code is found, offset is pointing to the first unprocessed byte - // (i.e. the first byte that was not considered as a possible start of a start - // code) and |*start_code_size| is set to 0. - // Preconditions: - // - |data_size| >= 0 - // Postconditions: - // - |*offset| is between 0 and |data_size| included. - // It is strictly less than |data_size| if |data_size| > 0. - // - |*start_code_size| is either 0, 3 or 4. - static bool FindStartCode(const uint8_t* data, - off_t data_size, - off_t* offset, - off_t* start_code_size); - - // Wrapper for FindStartCode() that skips over start codes that - // may appear inside of |encrypted_ranges_|. - // Returns true if a start code was found. Otherwise returns false. - static bool FindStartCodeInClearRanges(const uint8_t* data, - off_t data_size, - const Ranges<const uint8_t*>& ranges, - off_t* offset, - off_t* start_code_size); - - static VideoCodecProfile ProfileIDCToVideoCodecProfile(int profile_idc); - - // Parses the input stream and returns all the NALUs through |nalus|. Returns - // false if the stream is invalid. - static bool ParseNALUs(const uint8_t* stream, - size_t stream_size, - std::vector<H264NALU>* nalus); - - H264Parser(); - ~H264Parser(); - - void Reset(); - // Set current stream pointer to |stream| of |stream_size| in bytes, - // |stream| owned by caller. - // |subsamples| contains information about what parts of |stream| are - // encrypted. - void SetStream(const uint8_t* stream, off_t stream_size); - void SetEncryptedStream(const uint8_t* stream, - off_t stream_size, - const std::vector<SubsampleEntry>& subsamples); - - // Read the stream to find the next NALU, identify it and return - // that information in |*nalu|. This advances the stream to the beginning - // of this NALU, but not past it, so subsequent calls to NALU-specific - // parsing functions (ParseSPS, etc.) will parse this NALU. - // If the caller wishes to skip the current NALU, it can call this function - // again, instead of any NALU-type specific parse functions below. - Result AdvanceToNextNALU(H264NALU* nalu); - - // NALU-specific parsing functions. - // These should be called after AdvanceToNextNALU(). - - // SPSes and PPSes are owned by the parser class and the memory for their - // structures is managed here, not by the caller, as they are reused - // across NALUs. - // - // Parse an SPS/PPS NALU and save their data in the parser, returning id - // of the parsed structure in |*pps_id|/|*sps_id|. - // To get a pointer to a given SPS/PPS structure, use GetSPS()/GetPPS(), - // passing the returned |*sps_id|/|*pps_id| as parameter. - // TODO(posciak,fischman): consider replacing returning Result from Parse*() - // methods with a scoped_ptr and adding an AtEOS() function to check for EOS - // if Parse*() return NULL. - Result ParseSPS(int* sps_id); - Result ParsePPS(int* pps_id); - - // Parses the SPS ID from the SPSExt, but otherwise does nothing. - Result ParseSPSExt(int* sps_id); - - // Return a pointer to SPS/PPS with given |sps_id|/|pps_id| or NULL if not - // present. - const H264SPS* GetSPS(int sps_id) const; - const H264PPS* GetPPS(int pps_id) const; - - // Slice headers and SEI messages are not used across NALUs by the parser - // and can be discarded after current NALU, so the parser does not store - // them, nor does it manage their memory. - // The caller has to provide and manage it instead. - - // Parse a slice header, returning it in |*shdr|. |*nalu| must be set to - // the NALU returned from AdvanceToNextNALU() and corresponding to |*shdr|. - Result ParseSliceHeader(const H264NALU& nalu, H264SliceHeader* shdr); - - // Parse a SEI message, returning it in |*sei_msg|, provided and managed - // by the caller. - Result ParseSEI(H264SEIMessage* sei_msg); - - // The return value of this method changes for every successful call to - // AdvanceToNextNALU(). - // This returns the subsample information for the last NALU that was output - // from AdvanceToNextNALU(). - std::vector<SubsampleEntry> GetCurrentSubsamples(); - - private: - // Move the stream pointer to the beginning of the next NALU, - // i.e. pointing at the next start code. - // Return true if a NALU has been found. - // If a NALU is found: - // - its size in bytes is returned in |*nalu_size| and includes - // the start code as well as the trailing zero bits. - // - the size in bytes of the start code is returned in |*start_code_size|. - bool LocateNALU(off_t* nalu_size, off_t* start_code_size); - - // Exp-Golomb code parsing as specified in chapter 9.1 of the spec. - // Read one unsigned exp-Golomb code from the stream and return in |*val|. - Result ReadUE(int* val); - - // Read one signed exp-Golomb code from the stream and return in |*val|. - Result ReadSE(int* val); - - // Parse scaling lists (see spec). - Result ParseScalingList(int size, int* scaling_list, bool* use_default); - Result ParseSPSScalingLists(H264SPS* sps); - Result ParsePPSScalingLists(const H264SPS& sps, H264PPS* pps); - - // Parse optional VUI parameters in SPS (see spec). - Result ParseVUIParameters(H264SPS* sps); - // Set |hrd_parameters_present| to true only if they are present. - Result ParseAndIgnoreHRDParameters(bool* hrd_parameters_present); - - // Parse reference picture lists' modifications (see spec). - Result ParseRefPicListModifications(H264SliceHeader* shdr); - Result ParseRefPicListModification(int num_ref_idx_active_minus1, - H264ModificationOfPicNum* ref_list_mods); - - // Parse prediction weight table (see spec). - Result ParsePredWeightTable(const H264SPS& sps, H264SliceHeader* shdr); - - // Parse weighting factors (see spec). - Result ParseWeightingFactors(int num_ref_idx_active_minus1, - int chroma_array_type, - int luma_log2_weight_denom, - int chroma_log2_weight_denom, - H264WeightingFactors* w_facts); - - // Parse decoded reference picture marking information (see spec). - Result ParseDecRefPicMarking(H264SliceHeader* shdr); - - // Pointer to the current NALU in the stream. - const uint8_t* stream_; - - // Bytes left in the stream after the current NALU. - off_t bytes_left_; - - H264BitReader br_; - - // PPSes and SPSes stored for future reference. - std::map<int, std::unique_ptr<H264SPS>> active_SPSes_; - std::map<int, std::unique_ptr<H264PPS>> active_PPSes_; - - // Ranges of encrypted bytes in the buffer passed to - // SetEncryptedStream(). - Ranges<const uint8_t*> encrypted_ranges_; - - // This contains the range of the previous NALU found in - // AdvanceToNextNalu(). Holds exactly one range. - Ranges<const uint8_t*> previous_nalu_range_; - - DISALLOW_COPY_AND_ASSIGN(H264Parser); -}; - -} // namespace media - -#endif // H264_PARSER_H_ diff --git a/accel/macros.h b/accel/macros.h deleted file mode 100644 index 22c3ad6..0000000 --- a/accel/macros.h +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MACROS_H_ -#define MACROS_H_ - -#include "base/logging.h" - -#define DVLOGF(level) DVLOG(level) << __func__ << "(): " -#define VLOGF(level) VLOG(level) << __func__ << "(): " -#define VPLOGF(level) VPLOG(level) << __func__ << "(): " - -#endif // MACROS_H_ diff --git a/accel/media_limits.h b/accel/media_limits.h deleted file mode 100644 index 97f3ab3..0000000 --- a/accel/media_limits.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: aa8dd1475415 -// Note: Changed name from "limits.h" to "media_limits.h" to prevent the -// ambiguity with C library <limits.h> - -// Contains limit definition constants for the media subsystem. - -#ifndef MEDIA_LIMITS_H_ -#define MEDIA_LIMITS_H_ - -namespace media { - -namespace limits { - -enum { - // Maximum possible dimension (width or height) for any video. - kMaxDimension = (1 << 15) - 1, // 32767 - - // Maximum possible canvas size (width multiplied by height) for any video. - kMaxCanvas = (1 << (14 * 2)), // 16384 x 16384 - - // Total number of video frames which are populating in the pipeline. - kMaxVideoFrames = 4, - - // The following limits are used by AudioParameters::IsValid(). - // - // A few notes on sample rates of common formats: - // - AAC files are limited to 96 kHz. - // - MP3 files are limited to 48 kHz. - // - Vorbis used to be limited to 96 kHz, but no longer has that - // restriction. - // - Most PC audio hardware is limited to 192 kHz, some specialized DAC - // devices will use 384 kHz though. - kMaxSampleRate = 384000, - kMinSampleRate = 3000, - kMaxChannels = 32, - kMaxBytesPerSample = 4, - kMaxBitsPerSample = kMaxBytesPerSample * 8, - kMaxSamplesPerPacket = kMaxSampleRate, - kMaxPacketSizeInBytes = - kMaxBytesPerSample * kMaxChannels * kMaxSamplesPerPacket, - - // This limit is used by ParamTraits<VideoCaptureParams>. - kMaxFramesPerSecond = 1000, - - // The minimum elapsed amount of time (in seconds) for a playback to be - // considered as having active engagement. - kMinimumElapsedWatchTimeSecs = 7, - - // Maximum lengths for various EME API parameters. These are checks to - // prevent unnecessarily large parameters from being passed around, and the - // lengths are somewhat arbitrary as the EME spec doesn't specify any limits. - kMinCertificateLength = 128, - kMaxCertificateLength = 16 * 1024, - kMaxSessionIdLength = 512, - kMinKeyIdLength = 1, - kMaxKeyIdLength = 512, - kMaxKeyIds = 128, - kMaxInitDataLength = 64 * 1024, // 64 KB - kMaxSessionResponseLength = 64 * 1024, // 64 KB - kMaxKeySystemLength = 256, - -// Minimum and maximum buffer sizes for certain audio platforms. -#if defined(OS_MACOSX) - kMinAudioBufferSize = 128, - kMaxAudioBufferSize = 4096, -#elif defined(USE_CRAS) - // Though CRAS has different per-board defaults, allow explicitly requesting - // this buffer size on any board. - kMinAudioBufferSize = 256, - kMaxAudioBufferSize = 8192, -#endif - - // Maximum buffer size supported by Web Audio. - kMaxWebAudioBufferSize = 8192, - - // Bounds for the number of threads used for software video decoding. - kMinVideoDecodeThreads = 2, - kMaxVideoDecodeThreads = - 16, // Matches ffmpeg's MAX_AUTO_THREADS. Higher values can result in - // immediate out of memory errors for high resolution content. See - // https://crbug.com/893984 -}; - -} // namespace limits - -} // namespace media - -#endif // MEDIA_LIMITS_H_ diff --git a/accel/native_pixmap_handle.cc b/accel/native_pixmap_handle.cc deleted file mode 100644 index 050a683..0000000 --- a/accel/native_pixmap_handle.cc +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: a9d98e6 - -#include "native_pixmap_handle.h" - -namespace media { - -NativePixmapPlane::NativePixmapPlane() - : stride(0), offset(0), size(0), modifier(0) {} - -NativePixmapPlane::NativePixmapPlane(int stride, - int offset, - uint64_t size, - uint64_t modifier) - : stride(stride), offset(offset), size(size), modifier(modifier) {} - -NativePixmapPlane::NativePixmapPlane(const NativePixmapPlane& other) = default; - -NativePixmapPlane::~NativePixmapPlane() {} - -NativePixmapHandle::NativePixmapHandle() {} -NativePixmapHandle::NativePixmapHandle(const NativePixmapHandle& other) = - default; - -NativePixmapHandle::~NativePixmapHandle() {} - -} // namespace media diff --git a/accel/native_pixmap_handle.h b/accel/native_pixmap_handle.h deleted file mode 100644 index 62e2294..0000000 --- a/accel/native_pixmap_handle.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: a9d98e6 - -#ifndef NATIVE_PIXMAP_HANDLE_H_ -#define NATIVE_PIXMAP_HANDLE_H_ - -#include <vector> - -#include "base/file_descriptor_posix.h" - -namespace media { - -// NativePixmapPlane is used to carry the plane related information for GBM -// buffer. More fields can be added if they are plane specific. -struct NativePixmapPlane { - // This is the same value as DRM_FORMAT_MOD_INVALID, which is not a valid - // modifier. We use this to indicate that layout information - // (tiling/compression) if any will be communicated out of band. - static constexpr uint64_t kNoModifier = 0x00ffffffffffffffULL; - - NativePixmapPlane(); - NativePixmapPlane(int stride, - int offset, - uint64_t size, - uint64_t modifier = kNoModifier); - NativePixmapPlane(const NativePixmapPlane& other); - ~NativePixmapPlane(); - - // The strides and offsets in bytes to be used when accessing the buffers via - // a memory mapping. One per plane per entry. - int stride; - int offset; - // Size in bytes of the plane. - // This is necessary to map the buffers. - uint64_t size; - // The modifier is retrieved from GBM library and passed to EGL driver. - // Generally it's platform specific, and we don't need to modify it in - // Chromium code. Also one per plane per entry. - uint64_t modifier; -}; - -struct NativePixmapHandle { - NativePixmapHandle(); - NativePixmapHandle(const NativePixmapHandle& other); - - ~NativePixmapHandle(); - - // File descriptors for the underlying memory objects (usually dmabufs). - std::vector<base::FileDescriptor> fds; - std::vector<NativePixmapPlane> planes; -}; - -} // namespace media - -#endif // NATIVE_PIXMAP_HANDLE_H_ diff --git a/accel/picture.cc b/accel/picture.cc deleted file mode 100644 index 8933bc5..0000000 --- a/accel/picture.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "picture.h" - -namespace media { - -PictureBuffer::PictureBuffer(int32_t id, const Size& size) - : id_(id), size_(size) {} - -PictureBuffer::PictureBuffer(int32_t id, - const Size& size, - VideoPixelFormat pixel_format) - : id_(id), - size_(size), - pixel_format_(pixel_format) {} - -PictureBuffer::PictureBuffer(const PictureBuffer& other) = default; - -PictureBuffer::~PictureBuffer() = default; - -Picture::Picture(int32_t picture_buffer_id, - int32_t bitstream_buffer_id, - const Rect& visible_rect, - bool allow_overlay) - : picture_buffer_id_(picture_buffer_id), - bitstream_buffer_id_(bitstream_buffer_id), - visible_rect_(visible_rect), - allow_overlay_(allow_overlay) {} - -Picture::Picture(const Picture& other) = default; - -Picture::~Picture() = default; - -} // namespace media diff --git a/accel/picture.h b/accel/picture.h deleted file mode 100644 index e07b677..0000000 --- a/accel/picture.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: d264e47 - -#ifndef PICTURE_H_ -#define PICTURE_H_ - -#include <stdint.h> - -#include <vector> - -#include "rect.h" -#include "size.h" -#include "video_pixel_format.h" - -namespace media { - -// A picture buffer that has size and pixel format information. -class PictureBuffer { - public: - PictureBuffer(int32_t id, const Size& size); - PictureBuffer(int32_t id, - const Size& size, - VideoPixelFormat pixel_format); - PictureBuffer(const PictureBuffer& other); - ~PictureBuffer(); - - // Returns the client-specified id of the buffer. - int32_t id() const { return id_; } - - // Returns the size of the buffer. - Size size() const { return size_; } - - void set_size(const Size& size) { size_ = size; } - - VideoPixelFormat pixel_format() const { return pixel_format_; } - - private: - int32_t id_; - Size size_; - VideoPixelFormat pixel_format_ = PIXEL_FORMAT_UNKNOWN; -}; - -// A decoded picture frame. -class Picture { - public: - Picture(int32_t picture_buffer_id, - int32_t bitstream_buffer_id, - const Rect& visible_rect, - bool allow_overlay); - Picture(const Picture&); - ~Picture(); - - // Returns the id of the picture buffer where this picture is contained. - int32_t picture_buffer_id() const { return picture_buffer_id_; } - - // Returns the id of the bitstream buffer from which this frame was decoded. - int32_t bitstream_buffer_id() const { return bitstream_buffer_id_; } - - void set_bitstream_buffer_id(int32_t bitstream_buffer_id) { - bitstream_buffer_id_ = bitstream_buffer_id; - } - - // Returns the visible rectangle of the picture. Its size may be smaller - // than the size of the PictureBuffer, as it is the only visible part of the - // Picture contained in the PictureBuffer. - Rect visible_rect() const { return visible_rect_; } - - bool allow_overlay() const { return allow_overlay_; } - - private: - int32_t picture_buffer_id_; - int32_t bitstream_buffer_id_; - Rect visible_rect_; - bool allow_overlay_; -}; - -} // namespace media - -#endif // PICTURE_H_ diff --git a/accel/ranges.cc b/accel/ranges.cc deleted file mode 100644 index 4394011..0000000 --- a/accel/ranges.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: a4f94d3 - -#include "ranges.h" - -namespace media { - -template<> -void Ranges<base::TimeDelta>::DCheckLT(const base::TimeDelta& lhs, - const base::TimeDelta& rhs) const { - DCHECK(lhs < rhs) << lhs.ToInternalValue() << " < " << rhs.ToInternalValue(); -} - -} // namespace media diff --git a/accel/ranges.h b/accel/ranges.h deleted file mode 100644 index 6a76ae4..0000000 --- a/accel/ranges.h +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 1323b9c - -#ifndef RANGES_H_ -#define RANGES_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <algorithm> -#include <ostream> -#include <vector> - -#include "base/logging.h" -#include "base/time/time.h" - -namespace media { - -// Ranges allows holding an ordered list of ranges of [start,end) intervals. -// The canonical example use-case is holding the list of ranges of buffered -// bytes or times in a <video> tag. -template <class T> // Endpoint type; typically a base::TimeDelta or an int64_t. -class Ranges { - public: - // Allow copy & assign. - - // Add (start,end) to this object, coallescing overlaps as appropriate. - // Returns the number of stored ranges, post coallescing. - size_t Add(T start, T end); - - // Return the number of disjoint ranges. - size_t size() const; - - // Return the "i"'th range's start & end (0-based). - T start(size_t i) const; - T end(size_t i) const; - - // Clear all ranges. - void clear(); - - // Computes the intersection between this range and |other|. - Ranges<T> IntersectionWith(const Ranges<T>& other) const; - - private: - // Wrapper around DCHECK_LT allowing comparisons of operator<<'able T's. - void DCheckLT(const T& lhs, const T& rhs) const; - - // Disjoint, in increasing order of start. - std::vector<std::pair<T, T> > ranges_; -}; - -////////////////////////////////////////////////////////////////////// -// EVERYTHING BELOW HERE IS IMPLEMENTATION DETAIL!! -////////////////////////////////////////////////////////////////////// - -template<class T> -size_t Ranges<T>::Add(T start, T end) { - if (start == end) // Nothing to be done with empty ranges. - return ranges_.size(); - - DCheckLT(start, end); - size_t i; - // Walk along the array of ranges until |start| is no longer larger than the - // current interval's end. - for (i = 0; i < ranges_.size() && ranges_[i].second < start; ++i) { - // Empty body - } - - // Now we know |start| belongs in the i'th slot. - // If i is the end of the range, append new range and done. - if (i == ranges_.size()) { - ranges_.push_back(std::make_pair(start, end)); - return ranges_.size(); - } - - // If |end| is less than i->first, then [start,end) is a new (non-overlapping) - // i'th entry pushing everyone else back, and done. - if (end < ranges_[i].first) { - ranges_.insert(ranges_.begin() + i, std::make_pair(start, end)); - return ranges_.size(); - } - - // Easy cases done. Getting here means there is overlap between [start,end) - // and the existing ranges. - - // Now: start <= i->second && i->first <= end - if (start < ranges_[i].first) - ranges_[i].first = start; - if (ranges_[i].second < end) - ranges_[i].second = end; - - // Now: [start,end) is contained in the i'th range, and we'd be done, except - // for the fact that the newly-extended i'th range might now overlap - // subsequent ranges. Merge until discontinuities appear. Note that there's - // no need to test/merge previous ranges, since needing that would mean the - // original loop went too far. - while ((i + 1) < ranges_.size() && - ranges_[i + 1].first <= ranges_[i].second) { - ranges_[i].second = std::max(ranges_[i].second, ranges_[i + 1].second); - ranges_.erase(ranges_.begin() + i + 1); - } - - return ranges_.size(); -} - -template<> -void Ranges<base::TimeDelta>::DCheckLT(const base::TimeDelta& lhs, - const base::TimeDelta& rhs) const; - -template<class T> -void Ranges<T>::DCheckLT(const T& lhs, const T& rhs) const { - DCHECK_LT(lhs, rhs); -} - -template<class T> -size_t Ranges<T>::size() const { - return ranges_.size(); -} - -template<class T> -T Ranges<T>::start(size_t i) const { - return ranges_[i].first; -} - -template<class T> -T Ranges<T>::end(size_t i) const { - return ranges_[i].second; -} - -template<class T> -void Ranges<T>::clear() { - ranges_.clear(); -} - -template<class T> -Ranges<T> Ranges<T>::IntersectionWith(const Ranges<T>& other) const { - Ranges<T> result; - - size_t i = 0; - size_t j = 0; - - while (i < size() && j < other.size()) { - T max_start = std::max(start(i), other.start(j)); - T min_end = std::min(end(i), other.end(j)); - - // Add an intersection range to the result if the ranges overlap. - if (max_start < min_end) - result.Add(max_start, min_end); - - if (end(i) < other.end(j)) - ++i; - else - ++j; - } - - return result; -} - -} // namespace media - -#endif // RANGES_H_ diff --git a/accel/rect.h b/accel/rect.h deleted file mode 100644 index b98522d..0000000 --- a/accel/rect.h +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 8a796386c11a -// Note: only necessary functions are ported from gfx::Rect - -// Defines a simple integer rectangle class. The containment semantics -// are array-like; that is, the coordinate (x, y) is considered to be -// contained by the rectangle, but the coordinate (x + width, y) is not. -// The class will happily let you create malformed rectangles (that is, -// rectangles with negative width and/or height), but there will be assertions -// in the operations (such as Contains()) to complain in this case. - -#ifndef RECT_H_ -#define RECT_H_ - -#include <string> - -#include "base/strings/stringprintf.h" -#include "size.h" - -namespace media { - -// Helper struct for rect to replace gfx::Rect usage from original code. -// Only partial functions of gfx::Rect is implemented here. -class Rect { - public: - constexpr Rect() = default; - constexpr Rect(int width, int height) : size_(width, height) {} - constexpr Rect(int x, int y, int width, int height) - : x_(x), - y_(y), - size_(GetClampedValue(x, width), GetClampedValue(y, height)) {} - constexpr explicit Rect(const Size& size) : size_(size) {} - - constexpr int x() const { return x_; } - // Sets the X position while preserving the width. - void set_x(int x) { - x_ = x; - size_.set_width(GetClampedValue(x, width())); - } - - constexpr int y() const { return y_; } - // Sets the Y position while preserving the height. - void set_y(int y) { - y_ = y; - size_.set_height(GetClampedValue(y, height())); - } - - constexpr int width() const { return size_.width(); } - void set_width(int width) { size_.set_width(GetClampedValue(x(), width)); } - - constexpr int height() const { return size_.height(); } - void set_height(int height) { - size_.set_height(GetClampedValue(y(), height)); - } - - constexpr const Size& size() const { return size_; } - void set_size(const Size& size) { - set_width(size.width()); - set_height(size.height()); - } - - constexpr int right() const { return x() + width(); } - constexpr int bottom() const { return y() + height(); } - - void SetRect(int x, int y, int width, int height) { - set_x(x); - set_y(y); - // Ensure that width and height remain valid. - set_width(width); - set_height(height); - } - - // Returns true if the area of the rectangle is zero. - bool IsEmpty() const { return size_.IsEmpty(); } - - // Returns true if this rectangle contains the specified rectangle. - bool Contains(const Rect& rect) const { - return (rect.x() >= x() && rect.right() <= right() && rect.y() >= y() && - rect.bottom() <= bottom()); - } - - // Computes the intersection of this rectangle with the given rectangle. - void Intersect(const Rect& rect) { - if (IsEmpty() || rect.IsEmpty()) { - SetRect(0, 0, 0, 0); // Throws away empty position. - return; - } - - int left = std::max(x(), rect.x()); - int top = std::max(y(), rect.y()); - int new_right = std::min(right(), rect.right()); - int new_bottom = std::min(bottom(), rect.bottom()); - - if (left >= new_right || top >= new_bottom) { - SetRect(0, 0, 0, 0); // Throws away empty position. - return; - } - - SetRect(left, top, new_right - left, new_bottom - top); - } - - std::string ToString() const { - return base::StringPrintf("(%d,%d) %s", - x_, y_, size().ToString().c_str()); - } - - private: - int x_ = 0; - int y_ = 0; - Size size_; - - // Returns true iff a+b would overflow max int. - static constexpr bool AddWouldOverflow(int a, int b) { - // In this function, GCC tries to make optimizations that would only work if - // max - a wouldn't overflow but it isn't smart enough to notice that a > 0. - // So cast everything to unsigned to avoid this. As it is guaranteed that - // max - a and b are both already positive, the cast is a noop. - // - // This is intended to be: a > 0 && max - a < b - return a > 0 && b > 0 && - static_cast<unsigned>(std::numeric_limits<int>::max() - a) < - static_cast<unsigned>(b); - } - - // Clamp the size to avoid integer overflow in bottom() and right(). - // This returns the width given an origin and a width. - // TODO(enne): this should probably use base::ClampAdd, but that - // function is not a constexpr. - static constexpr int GetClampedValue(int origin, int size) { - return AddWouldOverflow(origin, size) - ? std::numeric_limits<int>::max() - origin - : size; - } -}; - -inline bool operator==(const Rect& lhs, const Rect& rhs) { - return lhs.x() == rhs.x() && lhs.y() == rhs.y() && lhs.size() == rhs.size(); -} - -inline bool operator!=(const Rect& lhs, const Rect& rhs) { - return !(lhs == rhs); -} - -} // namespace media - -#endif // RECT_H_ diff --git a/accel/shared_memory_region.cc b/accel/shared_memory_region.cc deleted file mode 100644 index 775a5f2..0000000 --- a/accel/shared_memory_region.cc +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 60f9667 - -#include "base/sys_info.h" -#include "shared_memory_region.h" - -namespace media { - -SharedMemoryRegion::SharedMemoryRegion(const base::SharedMemoryHandle& handle, - off_t offset, - size_t size, - bool read_only) - : shm_(handle, read_only), - offset_(offset), - size_(size), - alignment_size_(offset % base::SysInfo::VMAllocationGranularity()) { - DCHECK_GE(offset_, 0) << "Invalid offset: " << offset_; -} - -SharedMemoryRegion::SharedMemoryRegion(const BitstreamBuffer& bitstream_buffer, - bool read_only) - : SharedMemoryRegion(bitstream_buffer.handle(), - bitstream_buffer.offset(), - bitstream_buffer.size(), - read_only) {} - -bool SharedMemoryRegion::Map() { - if (offset_ < 0) { - DVLOG(1) << "Invalid offset: " << offset_; - return false; - } - return shm_.MapAt(offset_ - alignment_size_, size_ + alignment_size_); -} - -void* SharedMemoryRegion::memory() { - int8_t* addr = reinterpret_cast<int8_t*>(shm_.memory()); - return addr ? addr + alignment_size_ : nullptr; -} - -} // namespace media diff --git a/accel/shared_memory_region.h b/accel/shared_memory_region.h deleted file mode 100644 index 3c5d4b3..0000000 --- a/accel/shared_memory_region.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 60f9667 - -#ifndef SHARED_MEMORY_REGION_H_ -#define SHARED_MEMORY_REGION_H_ - -#include "base/memory/shared_memory.h" -#include "bitstream_buffer.h" - -namespace media { - -// Helper class to access a region of a SharedMemory. Different from -// SharedMemory, in which the |offset| of function MapAt() must be aligned to -// the value of |SysInfo::VMAllocationGranularity()|, the |offset| of a -// SharedMemoryRegion needs not to be aligned, this class hides the details -// and returns the mapped address of the given offset. -class SharedMemoryRegion { - public: - // Creates a SharedMemoryRegion. - // The mapped memory region begins at |offset| bytes from the start of the - // shared memory and the length is |size|. It will take the ownership of - // the |handle| and release the resource when being destroyed. Different - // from SharedMemory, the |offset| needs not to be aligned to the value of - // |SysInfo::VMAllocationGranularity()|. - SharedMemoryRegion(const base::SharedMemoryHandle& handle, - off_t offset, - size_t size, - bool read_only); - - // Creates a SharedMemoryRegion from the given |bistream_buffer|. - SharedMemoryRegion(const BitstreamBuffer& bitstream_buffer, bool read_only); - - // Maps the shared memory into the caller's address space. - // Return true on success, false otherwise. - bool Map(); - - // Gets a pointer to the mapped region if it has been mapped via Map(). - // Returns |nullptr| if it is not mapped. The returned pointer points - // to the memory at the offset previously passed to the constructor. - void* memory(); - - size_t size() const { return size_; } - - private: - base::SharedMemory shm_; - off_t offset_; - size_t size_; - size_t alignment_size_; - - DISALLOW_COPY_AND_ASSIGN(SharedMemoryRegion); -}; - -} // namespace media - -#endif // SHARED_MEMORY_REGION_H_ diff --git a/accel/size.h b/accel/size.h deleted file mode 100644 index 265dc55..0000000 --- a/accel/size.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 4db7af61f923 -// Note: only necessary functions are ported from gfx::Size - -#ifndef SIZE_H_ -#define SIZE_H_ - -#include <string> - -#include "base/numerics/safe_math.h" -#include "base/strings/stringprintf.h" - -namespace media { - -// Helper struct for size to replace gfx::size usage from original code. -// Only partial functions of gfx::size is implemented here. -struct Size { - public: - constexpr Size() : width_(0), height_(0) {} - constexpr Size(int width, int height) - : width_(std::max(0, width)), height_(std::max(0, height)) {} - - Size& operator=(const Size& ps) { - set_width(ps.width()); - set_height(ps.height()); - return *this; - } - - constexpr int width() const { return width_; } - constexpr int height() const { return height_; } - - void set_width(int width) { width_ = std::max(0, width); } - void set_height(int height) { height_ = std::max(0, height); } - - // This call will CHECK if the area of this size would overflow int. - int GetArea() const { return GetCheckedArea().ValueOrDie(); } - - // Returns a checked numeric representation of the area. - base::CheckedNumeric<int> GetCheckedArea() const { - base::CheckedNumeric<int> checked_area = width(); - checked_area *= height(); - return checked_area; - } - - void SetSize(int width, int height) { - set_width(width); - set_height(height); - } - - bool IsEmpty() const { return !width() || !height(); } - - std::string ToString() const { - return base::StringPrintf("%dx%d", width(), height()); - } - - private: - int width_; - int height_; -}; - -inline bool operator==(const Size& lhs, const Size& rhs) { - return lhs.width() == rhs.width() && lhs.height() == rhs.height(); -} - -inline bool operator!=(const Size& lhs, const Size& rhs) { - return !(lhs == rhs); -} - -} // namespace media - -#endif // SIZE_H_ diff --git a/accel/subsample_entry.h b/accel/subsample_entry.h deleted file mode 100644 index 1e0bfad..0000000 --- a/accel/subsample_entry.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 7014d6d - -#ifndef SUBSAMPLE_ENTRY_H_ -#define SUBSAMPLE_ENTRY_H_ - -#include <stdint.h> - -namespace media { - -// The Common Encryption spec provides for subsample encryption, where portions -// of a sample are set in cleartext. A SubsampleEntry specifies the number of -// clear and encrypted bytes in each subsample. For decryption, all of the -// encrypted bytes in a sample should be considered a single logical stream, -// regardless of how they are divided into subsamples, and the clear bytes -// should not be considered as part of decryption. This is logically equivalent -// to concatenating all 'cypher_bytes' portions of subsamples, decrypting that -// result, and then copying each byte from the decrypted block over the -// position of the corresponding encrypted byte. -struct SubsampleEntry { - SubsampleEntry() : clear_bytes(0), cypher_bytes(0) {} - SubsampleEntry(uint32_t clear_bytes, uint32_t cypher_bytes) - : clear_bytes(clear_bytes), cypher_bytes(cypher_bytes) {} - uint32_t clear_bytes; - uint32_t cypher_bytes; -}; - -} // namespace media - -#endif // SUBSAMPLE_ENTRY_H_ diff --git a/accel/v4l2_device.cc b/accel/v4l2_device.cc deleted file mode 100644 index 5c258ab..0000000 --- a/accel/v4l2_device.cc +++ /dev/null @@ -1,1858 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2f13d62f0c0d -// Note: Added some missing defines that are only defined in newer kernel -// versions (e.g. V4L2_PIX_FMT_VP8_FRAME) - -#include "v4l2_device.h" - -#include <fcntl.h> -#include <linux/media.h> -#include <linux/videodev2.h> -#include <poll.h> -#include <string.h> -#include <sys/ioctl.h> -#include <sys/mman.h> - -#include <algorithm> -#include <set> -#include <sstream> - -#include "base/bind.h" -#include "base/logging.h" -#include "base/numerics/safe_conversions.h" -#include "base/posix/eintr_wrapper.h" - -#include "color_plane_layout.h" -#include "generic_v4l2_device.h" -#include "macros.h" -#include "video_pixel_format.h" - -// VP8 parsed frames -#ifndef V4L2_PIX_FMT_VP8_FRAME -#define V4L2_PIX_FMT_VP8_FRAME v4l2_fourcc('V', 'P', '8', 'F') -#endif - -// VP9 parsed frames -#ifndef V4L2_PIX_FMT_VP9_FRAME -#define V4L2_PIX_FMT_VP9_FRAME v4l2_fourcc('V', 'P', '9', 'F') -#endif - -// H264 parsed slices -#ifndef V4L2_PIX_FMT_H264_SLICE -#define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') -#endif - -#define REQUEST_DEVICE "/dev/media-dec0" - -namespace media { - -V4L2ExtCtrl::V4L2ExtCtrl(uint32_t id) { - memset(&ctrl, 0, sizeof(ctrl)); - ctrl.id = id; -} - -V4L2ExtCtrl::V4L2ExtCtrl(uint32_t id, int32_t val) : V4L2ExtCtrl(id) { - ctrl.value = val; -} - -// Class used to store the state of a buffer that should persist between -// reference creations. This includes: -// * Result of initial VIDIOC_QUERYBUF ioctl, -// * Plane mappings. -// -// Also provides helper functions. -class V4L2Buffer { - public: - static std::unique_ptr<V4L2Buffer> Create(scoped_refptr<V4L2Device> device, - enum v4l2_buf_type type, - enum v4l2_memory memory, - const struct v4l2_format& format, - size_t buffer_id); - ~V4L2Buffer(); - - void* GetPlaneMapping(const size_t plane); - size_t GetMemoryUsage() const; - const struct v4l2_buffer& v4l2_buffer() const { return v4l2_buffer_; } - - private: - V4L2Buffer(scoped_refptr<V4L2Device> device, - enum v4l2_buf_type type, - enum v4l2_memory memory, - const struct v4l2_format& format, - size_t buffer_id); - bool Query(); - - scoped_refptr<V4L2Device> device_; - std::vector<void*> plane_mappings_; - - // V4L2 data as queried by QUERYBUF. - struct v4l2_buffer v4l2_buffer_; - // WARNING: do not change this to a vector or something smaller than - // VIDEO_MAX_PLANES, otherwise the Tegra libv4l2 will write data beyond - // the number of allocated planes, resulting in memory corruption. - struct v4l2_plane v4l2_planes_[VIDEO_MAX_PLANES]; - - struct v4l2_format format_ __attribute__((unused)); - scoped_refptr<VideoFrame> video_frame_; - - DISALLOW_COPY_AND_ASSIGN(V4L2Buffer); -}; - -std::unique_ptr<V4L2Buffer> V4L2Buffer::Create(scoped_refptr<V4L2Device> device, - enum v4l2_buf_type type, - enum v4l2_memory memory, - const struct v4l2_format& format, - size_t buffer_id) { - // Not using std::make_unique because constructor is private. - std::unique_ptr<V4L2Buffer> buffer( - new V4L2Buffer(device, type, memory, format, buffer_id)); - - if (!buffer->Query()) - return nullptr; - - return buffer; -} - -V4L2Buffer::V4L2Buffer(scoped_refptr<V4L2Device> device, - enum v4l2_buf_type type, - enum v4l2_memory memory, - const struct v4l2_format& format, - size_t buffer_id) - : device_(device), format_(format) { - DCHECK(V4L2_TYPE_IS_MULTIPLANAR(type)); - DCHECK_LE(format.fmt.pix_mp.num_planes, base::size(v4l2_planes_)); - - memset(v4l2_planes_, 0, sizeof(v4l2_planes_)); - memset(&v4l2_buffer_, 0, sizeof(v4l2_buffer_)); - v4l2_buffer_.m.planes = v4l2_planes_; - // Just in case we got more planes than we want. - v4l2_buffer_.length = - std::min(static_cast<size_t>(format.fmt.pix_mp.num_planes), - base::size(v4l2_planes_)); - v4l2_buffer_.index = buffer_id; - v4l2_buffer_.type = type; - v4l2_buffer_.memory = memory; - plane_mappings_.resize(v4l2_buffer_.length); -} - -V4L2Buffer::~V4L2Buffer() { - if (v4l2_buffer_.memory == V4L2_MEMORY_MMAP) { - for (size_t i = 0; i < plane_mappings_.size(); i++) - if (plane_mappings_[i] != nullptr) - device_->Munmap(plane_mappings_[i], v4l2_buffer_.m.planes[i].length); - } -} - -bool V4L2Buffer::Query() { - int ret = device_->Ioctl(VIDIOC_QUERYBUF, &v4l2_buffer_); - if (ret) { - VPLOGF(1) << "VIDIOC_QUERYBUF failed: "; - return false; - } - - DCHECK(plane_mappings_.size() == v4l2_buffer_.length); - - return true; -} - -void* V4L2Buffer::GetPlaneMapping(const size_t plane) { - if (plane >= plane_mappings_.size()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return nullptr; - } - - void* p = plane_mappings_[plane]; - if (p) - return p; - - // Do this check here to avoid repeating it after a buffer has been - // successfully mapped (we know we are of MMAP type by then). - if (v4l2_buffer_.memory != V4L2_MEMORY_MMAP) { - VLOGF(1) << "Cannot create mapping on non-MMAP buffer"; - return nullptr; - } - - p = device_->Mmap(NULL, v4l2_buffer_.m.planes[plane].length, - PROT_READ | PROT_WRITE, MAP_SHARED, - v4l2_buffer_.m.planes[plane].m.mem_offset); - if (p == MAP_FAILED) { - VPLOGF(1) << "mmap() failed: "; - return nullptr; - } - - plane_mappings_[plane] = p; - return p; -} - -size_t V4L2Buffer::GetMemoryUsage() const { - size_t usage = 0; - for (size_t i = 0; i < v4l2_buffer_.length; i++) { - usage += v4l2_buffer_.m.planes[i].length; - } - return usage; -} - -// A thread-safe pool of buffer indexes, allowing buffers to be obtained and -// returned from different threads. All the methods of this class are -// thread-safe. Users should keep a scoped_refptr to instances of this class -// in order to ensure the list remains alive as long as they need it. -class V4L2BuffersList : public base::RefCountedThreadSafe<V4L2BuffersList> { - public: - V4L2BuffersList() = default; - // Return a buffer to this list. Also can be called to set the initial pool - // of buffers. - // Note that it is illegal to return the same buffer twice. - void ReturnBuffer(size_t buffer_id); - // Get any of the buffers in the list. There is no order guarantee whatsoever. - base::Optional<size_t> GetFreeBuffer(); - // Get the buffer with specified index. - base::Optional<size_t> GetFreeBuffer(size_t requested_buffer_id); - // Number of buffers currently in this list. - size_t size() const; - - private: - friend class base::RefCountedThreadSafe<V4L2BuffersList>; - ~V4L2BuffersList() = default; - - mutable base::Lock lock_; - std::set<size_t> free_buffers_ GUARDED_BY(lock_); - DISALLOW_COPY_AND_ASSIGN(V4L2BuffersList); -}; - -void V4L2BuffersList::ReturnBuffer(size_t buffer_id) { - base::AutoLock auto_lock(lock_); - - auto inserted = free_buffers_.emplace(buffer_id); - DCHECK(inserted.second); -} - -base::Optional<size_t> V4L2BuffersList::GetFreeBuffer() { - base::AutoLock auto_lock(lock_); - - auto iter = free_buffers_.begin(); - if (iter == free_buffers_.end()) { - DVLOGF(4) << "No free buffer available!"; - return base::nullopt; - } - - size_t buffer_id = *iter; - free_buffers_.erase(iter); - - return buffer_id; -} - -base::Optional<size_t> V4L2BuffersList::GetFreeBuffer( - size_t requested_buffer_id) { - base::AutoLock auto_lock(lock_); - - return (free_buffers_.erase(requested_buffer_id) > 0) - ? base::make_optional(requested_buffer_id) - : base::nullopt; -} - -size_t V4L2BuffersList::size() const { - base::AutoLock auto_lock(lock_); - - return free_buffers_.size(); -} - -// Module-private class that let users query/write V4L2 buffer information. -// It also makes some private V4L2Queue methods available to this module only. -class V4L2BufferRefBase { - public: - V4L2BufferRefBase(const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue); - ~V4L2BufferRefBase(); - - bool QueueBuffer(); - void* GetPlaneMapping(const size_t plane); - - // Checks that the number of passed FDs is adequate for the current format - // and buffer configuration. Only useful for DMABUF buffers. - bool CheckNumFDsForFormat(const size_t num_fds) const; - - // Data from the buffer, that users can query and/or write. - struct v4l2_buffer v4l2_buffer_; - // WARNING: do not change this to a vector or something smaller than - // VIDEO_MAX_PLANES, otherwise the Tegra libv4l2 will write data beyond - // the number of allocated planes, resulting in memory corruption. - struct v4l2_plane v4l2_planes_[VIDEO_MAX_PLANES]; - - private: - size_t BufferId() const { return v4l2_buffer_.index; } - - friend class V4L2WritableBufferRef; - // A weak pointer to the queue this buffer belongs to. Will remain valid as - // long as the underlying V4L2 buffer is valid too. - // This can only be accessed from the sequence protected by sequence_checker_. - // Thread-safe methods (like ~V4L2BufferRefBase) must *never* access this. - base::WeakPtr<V4L2Queue> queue_; - // Where to return this buffer if it goes out of scope without being queued. - scoped_refptr<V4L2BuffersList> return_to_; - bool queued = false; - - SEQUENCE_CHECKER(sequence_checker_); - DISALLOW_COPY_AND_ASSIGN(V4L2BufferRefBase); -}; - -V4L2BufferRefBase::V4L2BufferRefBase(const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue) - : queue_(std::move(queue)), return_to_(queue_->free_buffers_) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(V4L2_TYPE_IS_MULTIPLANAR(v4l2_buffer.type)); - DCHECK_LE(v4l2_buffer.length, base::size(v4l2_planes_)); - DCHECK(return_to_); - - memcpy(&v4l2_buffer_, &v4l2_buffer, sizeof(v4l2_buffer_)); - memcpy(v4l2_planes_, v4l2_buffer.m.planes, - sizeof(struct v4l2_plane) * v4l2_buffer.length); - v4l2_buffer_.m.planes = v4l2_planes_; -} - -V4L2BufferRefBase::~V4L2BufferRefBase() { - // We are the last reference and are only accessing the thread-safe - // return_to_, so we are safe to call from any sequence. - // If we have been queued, then the queue is our owner so we don't need to - // return to the free buffers list. - if (!queued) - return_to_->ReturnBuffer(BufferId()); -} - -bool V4L2BufferRefBase::QueueBuffer() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!queue_) - return false; - - queued = queue_->QueueBuffer(&v4l2_buffer_); - - return queued; -} - -void* V4L2BufferRefBase::GetPlaneMapping(const size_t plane) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!queue_) - return nullptr; - - return queue_->buffers_[BufferId()]->GetPlaneMapping(plane); -} - -bool V4L2BufferRefBase::CheckNumFDsForFormat(const size_t num_fds) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!queue_) - return false; - - // We have not used SetFormat(), assume this is ok. - // Hopefully we standardize SetFormat() in the future. - if (!queue_->current_format_) - return true; - - const size_t required_fds = queue_->current_format_->fmt.pix_mp.num_planes; - // Sanity check. - DCHECK_EQ(v4l2_buffer_.length, required_fds); - if (num_fds < required_fds) { - VLOGF(1) << "Insufficient number of FDs given for the current format. " - << num_fds << " provided, " << required_fds << " required."; - return false; - } - - const auto* planes = v4l2_buffer_.m.planes; - for (size_t i = v4l2_buffer_.length - 1; i >= num_fds; --i) { - // Assume that an fd is a duplicate of a previous plane's fd if offset != 0. - // Otherwise, if offset == 0, return error as it is likely pointing to - // a new plane. - if (planes[i].data_offset == 0) { - VLOGF(1) << "Additional dmabuf fds point to a new buffer."; - return false; - } - } - - return true; -} - -V4L2WritableBufferRef::V4L2WritableBufferRef( - const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue) - : buffer_data_( - std::make_unique<V4L2BufferRefBase>(v4l2_buffer, std::move(queue))) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); -} - -V4L2WritableBufferRef::V4L2WritableBufferRef(V4L2WritableBufferRef&& other) - : buffer_data_(std::move(other.buffer_data_)) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK_CALLED_ON_VALID_SEQUENCE(other.sequence_checker_); -} - -V4L2WritableBufferRef::~V4L2WritableBufferRef() { - // Only valid references should be sequence-checked - if (buffer_data_) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - } -} - -V4L2WritableBufferRef& V4L2WritableBufferRef::operator=( - V4L2WritableBufferRef&& other) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK_CALLED_ON_VALID_SEQUENCE(other.sequence_checker_); - - if (this == &other) - return *this; - - buffer_data_ = std::move(other.buffer_data_); - - return *this; -} - -enum v4l2_memory V4L2WritableBufferRef::Memory() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return static_cast<enum v4l2_memory>(buffer_data_->v4l2_buffer_.memory); -} - -bool V4L2WritableBufferRef::DoQueue(V4L2RequestRef* /*request_ref*/) && { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - bool queued = buffer_data_->QueueBuffer(); - - // Clear our own reference. - buffer_data_.reset(); - - return queued; -} - -bool V4L2WritableBufferRef::QueueMMap(V4L2RequestRef* request_ref) && { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - // Move ourselves so our data gets freed no matter when we return - V4L2WritableBufferRef self(std::move(*this)); - - if (self.Memory() != V4L2_MEMORY_MMAP) { - VLOGF(1) << "Called on invalid buffer type!"; - return false; - } - - return std::move(self).DoQueue(request_ref); -} - -bool V4L2WritableBufferRef::QueueUserPtr(const std::vector<void*>& ptrs, - V4L2RequestRef* request_ref) && { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - // Move ourselves so our data gets freed no matter when we return - V4L2WritableBufferRef self(std::move(*this)); - - if (self.Memory() != V4L2_MEMORY_USERPTR) { - VLOGF(1) << "Called on invalid buffer type!"; - return false; - } - - if (ptrs.size() != self.PlanesCount()) { - VLOGF(1) << "Provided " << ptrs.size() << " pointers while we require " - << self.buffer_data_->v4l2_buffer_.length << "."; - return false; - } - - for (size_t i = 0; i < ptrs.size(); i++) - self.buffer_data_->v4l2_buffer_.m.planes[i].m.userptr = - reinterpret_cast<unsigned long>(ptrs[i]); - - return std::move(self).DoQueue(request_ref); -} - -bool V4L2WritableBufferRef::QueueDMABuf(const std::vector<base::ScopedFD>& scoped_fds, - V4L2RequestRef* request_ref) && { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::vector<int> fds; - fds.reserve(scoped_fds.size()); - for (const base::ScopedFD& scoped_fd : scoped_fds) - fds.push_back(scoped_fd.get()); - - return std::move(*this).QueueDMABuf(fds, request_ref); -} - -bool V4L2WritableBufferRef::QueueDMABuf(const std::vector<int>& fds, - V4L2RequestRef* request_ref) && { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - // Move ourselves so our data gets freed no matter when we return - V4L2WritableBufferRef self(std::move(*this)); - - if (self.Memory() != V4L2_MEMORY_DMABUF) { - VLOGF(1) << "Called on invalid buffer type!"; - return false; - } - - if (!self.buffer_data_->CheckNumFDsForFormat(fds.size())) - return false; - - size_t num_planes = self.PlanesCount(); - for (size_t i = 0; i < num_planes; i++) - self.buffer_data_->v4l2_buffer_.m.planes[i].m.fd = fds[i]; - - return std::move(self).DoQueue(request_ref); -} - -size_t V4L2WritableBufferRef::PlanesCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.length; -} - -size_t V4L2WritableBufferRef::GetPlaneSize(const size_t plane) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return 0; - } - - return buffer_data_->v4l2_buffer_.m.planes[plane].length; -} - -void V4L2WritableBufferRef::SetPlaneSize(const size_t plane, - const size_t size) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - enum v4l2_memory memory = Memory(); - if (memory == V4L2_MEMORY_MMAP) { - DCHECK_EQ(buffer_data_->v4l2_buffer_.m.planes[plane].length, size); - return; - } - DCHECK(memory == V4L2_MEMORY_USERPTR || memory == V4L2_MEMORY_DMABUF); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return; - } - - buffer_data_->v4l2_buffer_.m.planes[plane].length = size; -} - -void* V4L2WritableBufferRef::GetPlaneMapping(const size_t plane) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->GetPlaneMapping(plane); -} - -void V4L2WritableBufferRef::SetTimeStamp(const struct timeval& timestamp) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - buffer_data_->v4l2_buffer_.timestamp = timestamp; -} - -const struct timeval& V4L2WritableBufferRef::GetTimeStamp() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.timestamp; -} - -void V4L2WritableBufferRef::SetPlaneBytesUsed(const size_t plane, - const size_t bytes_used) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return; - } - - if (bytes_used > GetPlaneSize(plane)) { - VLOGF(1) << "Set bytes used " << bytes_used << " larger than plane size " - << GetPlaneSize(plane) << "."; - return; - } - - buffer_data_->v4l2_buffer_.m.planes[plane].bytesused = bytes_used; -} - -size_t V4L2WritableBufferRef::GetPlaneBytesUsed(const size_t plane) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return 0; - } - - return buffer_data_->v4l2_buffer_.m.planes[plane].bytesused; -} - -void V4L2WritableBufferRef::SetPlaneDataOffset(const size_t plane, - const size_t data_offset) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return; - } - - buffer_data_->v4l2_buffer_.m.planes[plane].data_offset = data_offset; -} - -size_t V4L2WritableBufferRef::BufferId() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.index; -} - -V4L2ReadableBuffer::V4L2ReadableBuffer(const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue) - : buffer_data_( - std::make_unique<V4L2BufferRefBase>(v4l2_buffer, std::move(queue))) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); -} - -V4L2ReadableBuffer::~V4L2ReadableBuffer() { - // This method is thread-safe. Since we are the destructor, we are guaranteed - // to be called from the only remaining reference to us. Also, we are just - // calling the destructor of buffer_data_, which is also thread-safe. - DCHECK(buffer_data_); -} - -bool V4L2ReadableBuffer::IsLast() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.flags & V4L2_BUF_FLAG_LAST; -} - -bool V4L2ReadableBuffer::IsKeyframe() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.flags & V4L2_BUF_FLAG_KEYFRAME; -} - -struct timeval V4L2ReadableBuffer::GetTimeStamp() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.timestamp; -} - -size_t V4L2ReadableBuffer::PlanesCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.length; -} - -const void* V4L2ReadableBuffer::GetPlaneMapping(const size_t plane) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->GetPlaneMapping(plane); -} - -size_t V4L2ReadableBuffer::GetPlaneBytesUsed(const size_t plane) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return 0; - } - - return buffer_data_->v4l2_planes_[plane].bytesused; -} - -size_t V4L2ReadableBuffer::GetPlaneDataOffset(const size_t plane) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - if (plane >= PlanesCount()) { - VLOGF(1) << "Invalid plane " << plane << " requested."; - return 0; - } - - return buffer_data_->v4l2_planes_[plane].data_offset; -} - -size_t V4L2ReadableBuffer::BufferId() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(buffer_data_); - - return buffer_data_->v4l2_buffer_.index; -} - -// This class is used to expose buffer reference classes constructors to -// this module. This is to ensure that nobody else can create buffer references. -class V4L2BufferRefFactory { - public: - static V4L2WritableBufferRef CreateWritableRef( - const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue) { - return V4L2WritableBufferRef(v4l2_buffer, std::move(queue)); - } - - static V4L2ReadableBufferRef CreateReadableRef( - const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue) { - return new V4L2ReadableBuffer(v4l2_buffer, std::move(queue)); - } -}; - -// Helper macros that print the queue type with logs. -#define VPQLOGF(level) \ - VPLOGF(level) << "(" << V4L2Device::V4L2BufferTypeToString(type_) << ") " -#define VQLOGF(level) \ - VLOGF(level) << "(" << V4L2Device::V4L2BufferTypeToString(type_) << ") " -#define DVQLOGF(level) \ - DVLOGF(level) << "(" << V4L2Device::V4L2BufferTypeToString(type_) << ") " - -V4L2Queue::V4L2Queue(scoped_refptr<V4L2Device> dev, - enum v4l2_buf_type type, - base::OnceClosure destroy_cb) - : type_(type), - device_(dev), - destroy_cb_(std::move(destroy_cb)), - weak_this_factory_(this) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); -} - -V4L2Queue::~V4L2Queue() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (is_streaming_) { - VQLOGF(1) << "Queue is still streaming, trying to stop it..."; - Streamoff(); - } - - DCHECK(queued_buffers_.empty()); - DCHECK(!free_buffers_); - - if (!buffers_.empty()) { - VQLOGF(1) << "Buffers are still allocated, trying to deallocate them..."; - DeallocateBuffers(); - } - - std::move(destroy_cb_).Run(); -} - -base::Optional<struct v4l2_format> V4L2Queue::SetFormat(uint32_t fourcc, - const Size& size, - size_t buffer_size) { - struct v4l2_format format; - memset(&format, 0, sizeof(format)); - format.type = type_; - format.fmt.pix_mp.pixelformat = fourcc; - format.fmt.pix_mp.width = size.width(); - format.fmt.pix_mp.height = size.height(); - format.fmt.pix_mp.num_planes = V4L2Device::GetNumPlanesOfV4L2PixFmt(fourcc); - format.fmt.pix_mp.plane_fmt[0].sizeimage = buffer_size; - if (device_->Ioctl(VIDIOC_S_FMT, &format) != 0 || - format.fmt.pix_mp.pixelformat != fourcc) { - VPQLOGF(2) << "Failed to set format on queue " << type_ - << ". format_fourcc=0x" << std::hex << fourcc; - return base::nullopt; - } - - current_format_ = format; - return current_format_; -} - -size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(!free_buffers_); - DCHECK_EQ(queued_buffers_.size(), 0u); - - if (IsStreaming()) { - VQLOGF(1) << "Cannot allocate buffers while streaming."; - return 0; - } - - if (buffers_.size() != 0) { - VQLOGF(1) - << "Cannot allocate new buffers while others are still allocated."; - return 0; - } - - if (count == 0) { - VQLOGF(1) << "Attempting to allocate 0 buffers."; - return 0; - } - - // First query the number of planes in the buffers we are about to request. - // This should not be required, but Tegra's VIDIOC_QUERYBUF will fail on - // output buffers if the number of specified planes does not exactly match the - // format. - struct v4l2_format format = {.type = type_}; - int ret = device_->Ioctl(VIDIOC_G_FMT, &format); - if (ret) { - VPQLOGF(1) << "VIDIOC_G_FMT failed"; - return 0; - } - planes_count_ = format.fmt.pix_mp.num_planes; - DCHECK_LE(planes_count_, static_cast<size_t>(VIDEO_MAX_PLANES)); - - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = count; - reqbufs.type = type_; - reqbufs.memory = memory; - DVQLOGF(3) << "Requesting " << count << " buffers."; - - ret = device_->Ioctl(VIDIOC_REQBUFS, &reqbufs); - if (ret) { - VPQLOGF(1) << "VIDIOC_REQBUFS failed"; - return 0; - } - DVQLOGF(3) << "queue " << type_ << ": got " << reqbufs.count << " buffers."; - - memory_ = memory; - - free_buffers_ = new V4L2BuffersList(); - - // Now query all buffer information. - for (size_t i = 0; i < reqbufs.count; i++) { - auto buffer = V4L2Buffer::Create(device_, type_, memory_, format, i); - - if (!buffer) { - DeallocateBuffers(); - - return 0; - } - - buffers_.emplace_back(std::move(buffer)); - free_buffers_->ReturnBuffer(i); - } - - DCHECK(free_buffers_); - DCHECK_EQ(free_buffers_->size(), buffers_.size()); - DCHECK_EQ(queued_buffers_.size(), 0u); - - return buffers_.size(); -} - -bool V4L2Queue::DeallocateBuffers() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (IsStreaming()) { - VQLOGF(1) << "Cannot deallocate buffers while streaming."; - return false; - } - - if (buffers_.size() == 0) - return true; - - weak_this_factory_.InvalidateWeakPtrs(); - buffers_.clear(); - free_buffers_ = nullptr; - - // Free all buffers. - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = 0; - reqbufs.type = type_; - reqbufs.memory = memory_; - - int ret = device_->Ioctl(VIDIOC_REQBUFS, &reqbufs); - if (ret) { - VPQLOGF(1) << "VIDIOC_REQBUFS failed"; - return false; - } - - DCHECK(!free_buffers_); - DCHECK_EQ(queued_buffers_.size(), 0u); - - return true; -} - -size_t V4L2Queue::GetMemoryUsage() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - size_t usage = 0; - for (const auto& buf : buffers_) { - usage += buf->GetMemoryUsage(); - } - return usage; -} - -v4l2_memory V4L2Queue::GetMemoryType() const { - return memory_; -} - -base::Optional<V4L2WritableBufferRef> V4L2Queue::GetFreeBuffer() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // No buffers allocated at the moment? - if (!free_buffers_) - return base::nullopt; - - auto buffer_id = free_buffers_->GetFreeBuffer(); - if (!buffer_id.has_value()) - return base::nullopt; - - return V4L2BufferRefFactory::CreateWritableRef( - buffers_[buffer_id.value()]->v4l2_buffer(), - weak_this_factory_.GetWeakPtr()); -} - -base::Optional<V4L2WritableBufferRef> V4L2Queue::GetFreeBuffer( - size_t requested_buffer_id) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // No buffers allocated at the moment? - if (!free_buffers_) - return base::nullopt; - - auto buffer_id = free_buffers_->GetFreeBuffer(requested_buffer_id); - if (!buffer_id.has_value()) - return base::nullopt; - - return V4L2BufferRefFactory::CreateWritableRef( - buffers_[buffer_id.value()]->v4l2_buffer(), - weak_this_factory_.GetWeakPtr()); -} - -bool V4L2Queue::QueueBuffer(struct v4l2_buffer* v4l2_buffer) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - int ret = device_->Ioctl(VIDIOC_QBUF, v4l2_buffer); - if (ret) { - VPQLOGF(1) << "VIDIOC_QBUF failed"; - return false; - } - - auto inserted = queued_buffers_.emplace(v4l2_buffer->index); - DCHECK_EQ(inserted.second, true); - - device_->SchedulePoll(); - - return true; -} - -std::pair<bool, V4L2ReadableBufferRef> V4L2Queue::DequeueBuffer() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // No need to dequeue if no buffers queued. - if (QueuedBuffersCount() == 0) - return std::make_pair(true, nullptr); - - if (!IsStreaming()) { - VQLOGF(1) << "Attempting to dequeue a buffer while not streaming."; - return std::make_pair(true, nullptr); - } - - struct v4l2_buffer v4l2_buffer; - memset(&v4l2_buffer, 0, sizeof(v4l2_buffer)); - // WARNING: do not change this to a vector or something smaller than - // VIDEO_MAX_PLANES, otherwise the Tegra libv4l2 will write data beyond - // the number of allocated planes, resulting in memory corruption. - struct v4l2_plane planes[VIDEO_MAX_PLANES]; - memset(planes, 0, sizeof(planes)); - v4l2_buffer.type = type_; - v4l2_buffer.memory = memory_; - v4l2_buffer.m.planes = planes; - v4l2_buffer.length = planes_count_; - int ret = device_->Ioctl(VIDIOC_DQBUF, &v4l2_buffer); - if (ret) { - // TODO(acourbot): we should not have to check for EPIPE as codec clients - // should not call this method after the last buffer is dequeued. - switch (errno) { - case EAGAIN: - case EPIPE: - // This is not an error so we'll need to continue polling but won't - // provide a buffer. - device_->SchedulePoll(); - return std::make_pair(true, nullptr); - default: - VPQLOGF(1) << "VIDIOC_DQBUF failed"; - return std::make_pair(false, nullptr); - } - } - - auto it = queued_buffers_.find(v4l2_buffer.index); - DCHECK(it != queued_buffers_.end()); - queued_buffers_.erase(*it); - - if (QueuedBuffersCount() > 0) - device_->SchedulePoll(); - - DCHECK(free_buffers_); - return std::make_pair(true, - V4L2BufferRefFactory::CreateReadableRef( - v4l2_buffer, weak_this_factory_.GetWeakPtr())); -} - -bool V4L2Queue::IsStreaming() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - return is_streaming_; -} - -bool V4L2Queue::Streamon() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (is_streaming_) - return true; - - int arg = static_cast<int>(type_); - int ret = device_->Ioctl(VIDIOC_STREAMON, &arg); - if (ret) { - VPQLOGF(1) << "VIDIOC_STREAMON failed"; - return false; - } - - is_streaming_ = true; - - return true; -} - -bool V4L2Queue::Streamoff() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // We do not check the value of IsStreaming(), because we may have queued - // buffers to the queue and wish to get them back - in such as case, we may - // need to do a VIDIOC_STREAMOFF on a stopped queue. - - int arg = static_cast<int>(type_); - int ret = device_->Ioctl(VIDIOC_STREAMOFF, &arg); - if (ret) { - VPQLOGF(1) << "VIDIOC_STREAMOFF failed"; - return false; - } - - for (const auto& buffer_id : queued_buffers_) { - DCHECK(free_buffers_); - free_buffers_->ReturnBuffer(buffer_id); - } - - queued_buffers_.clear(); - - is_streaming_ = false; - - return true; -} - -size_t V4L2Queue::AllocatedBuffersCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - return buffers_.size(); -} - -size_t V4L2Queue::FreeBuffersCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - return free_buffers_ ? free_buffers_->size() : 0; -} - -size_t V4L2Queue::QueuedBuffersCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - return queued_buffers_.size(); -} - -#undef VDQLOGF -#undef VPQLOGF -#undef VQLOGF - -bool V4L2Queue::SupportsRequests() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - return supports_requests_; -} - -// This class is used to expose V4L2Queue's constructor to this module. This is -// to ensure that nobody else can create instances of it. -class V4L2QueueFactory { - public: - static scoped_refptr<V4L2Queue> CreateQueue(scoped_refptr<V4L2Device> dev, - enum v4l2_buf_type type, - base::OnceClosure destroy_cb) { - return new V4L2Queue(std::move(dev), type, std::move(destroy_cb)); - } -}; - -V4L2Device::V4L2Device() { - DETACH_FROM_SEQUENCE(client_sequence_checker_); -} - -V4L2Device::~V4L2Device() {} - -scoped_refptr<V4L2Queue> V4L2Device::GetQueue(enum v4l2_buf_type type) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - switch (type) { - // Supported queue types. - case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: - break; - default: - VLOGF(1) << "Unsupported V4L2 queue type: " << type; - return nullptr; - } - - // TODO(acourbot): we should instead query the device for available queues, - // and allocate them accordingly. This will do for now though. - auto it = queues_.find(type); - if (it != queues_.end()) - return scoped_refptr<V4L2Queue>(it->second); - - scoped_refptr<V4L2Queue> queue = V4L2QueueFactory::CreateQueue( - this, type, base::BindOnce(&V4L2Device::OnQueueDestroyed, this, type)); - - queues_[type] = queue.get(); - return queue; -} - -void V4L2Device::OnQueueDestroyed(v4l2_buf_type buf_type) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - auto it = queues_.find(buf_type); - DCHECK(it != queues_.end()); - queues_.erase(it); -} - -// static -scoped_refptr<V4L2Device> V4L2Device::Create() { - DVLOGF(3); - - scoped_refptr<V4L2Device> device; - - device = new GenericV4L2Device(); - if (device->Initialize()) - return device; - - VLOGF(1) << "Failed to create a V4L2Device"; - return nullptr; -} - -// static -uint32_t V4L2Device::VideoCodecProfileToV4L2PixFmt(VideoCodecProfile profile, - bool slice_based) { - if (profile >= H264PROFILE_MIN && profile <= H264PROFILE_MAX) { - if (slice_based) - return V4L2_PIX_FMT_H264_SLICE; - else - return V4L2_PIX_FMT_H264; - } else if (profile >= VP8PROFILE_MIN && profile <= VP8PROFILE_MAX) { - if (slice_based) - return V4L2_PIX_FMT_VP8_FRAME; - else - return V4L2_PIX_FMT_VP8; - } else if (profile >= VP9PROFILE_MIN && profile <= VP9PROFILE_MAX) { - if (slice_based) - return V4L2_PIX_FMT_VP9_FRAME; - else - return V4L2_PIX_FMT_VP9; - } else { - LOG(ERROR) << "Unknown profile: " << GetProfileName(profile); - return 0; - } -} - -// static -VideoCodecProfile V4L2Device::V4L2ProfileToVideoCodecProfile(VideoCodec codec, - uint32_t profile) { - switch (codec) { - case kCodecH264: - switch (profile) { - case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE: - case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: - return H264PROFILE_BASELINE; - case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN: - return H264PROFILE_MAIN; - case V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED: - return H264PROFILE_EXTENDED; - case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH: - return H264PROFILE_HIGH; - } - break; - case kCodecVP8: - switch (profile) { - case V4L2_MPEG_VIDEO_VP8_PROFILE_0: - case V4L2_MPEG_VIDEO_VP8_PROFILE_1: - case V4L2_MPEG_VIDEO_VP8_PROFILE_2: - case V4L2_MPEG_VIDEO_VP8_PROFILE_3: - return VP8PROFILE_ANY; - } - break; - case kCodecVP9: - switch (profile) { - case V4L2_MPEG_VIDEO_VP9_PROFILE_0: - return VP9PROFILE_PROFILE0; - case V4L2_MPEG_VIDEO_VP9_PROFILE_1: - return VP9PROFILE_PROFILE1; - case V4L2_MPEG_VIDEO_VP9_PROFILE_2: - return VP9PROFILE_PROFILE2; - case V4L2_MPEG_VIDEO_VP9_PROFILE_3: - return VP9PROFILE_PROFILE3; - } - break; - default: - VLOGF(2) << "Unknown codec: " << codec; - } - VLOGF(2) << "Unknown profile: " << profile; - return VIDEO_CODEC_PROFILE_UNKNOWN; -} - -std::vector<VideoCodecProfile> V4L2Device::V4L2PixFmtToVideoCodecProfiles( - uint32_t pix_fmt, - bool is_encoder) { - auto get_supported_profiles = [this]( - VideoCodec codec, - std::vector<VideoCodecProfile>* profiles) { - uint32_t query_id = 0; - switch (codec) { - case kCodecH264: - query_id = V4L2_CID_MPEG_VIDEO_H264_PROFILE; - break; - case kCodecVP8: - query_id = V4L2_CID_MPEG_VIDEO_VP8_PROFILE; - break; - case kCodecVP9: - query_id = V4L2_CID_MPEG_VIDEO_VP9_PROFILE; - break; - default: - return false; - } - - v4l2_queryctrl query_ctrl = {}; - query_ctrl.id = query_id; - if (Ioctl(VIDIOC_QUERYCTRL, &query_ctrl) != 0) { - return false; - } - v4l2_querymenu query_menu = {}; - query_menu.id = query_ctrl.id; - for (query_menu.index = query_ctrl.minimum; - static_cast<int>(query_menu.index) <= query_ctrl.maximum; - query_menu.index++) { - if (Ioctl(VIDIOC_QUERYMENU, &query_menu) == 0) { - const VideoCodecProfile profile = - V4L2Device::V4L2ProfileToVideoCodecProfile(codec, query_menu.index); - if (profile != VIDEO_CODEC_PROFILE_UNKNOWN) - profiles->push_back(profile); - } - } - return true; - }; - - std::vector<VideoCodecProfile> profiles; - switch (pix_fmt) { - case V4L2_PIX_FMT_H264: - case V4L2_PIX_FMT_H264_SLICE: - if (!get_supported_profiles(kCodecH264, &profiles)) { - DLOG(WARNING) << "Driver doesn't support QUERY H264 profiles, " - << "use default values, Base, Main, High"; - profiles = { - H264PROFILE_BASELINE, - H264PROFILE_MAIN, - H264PROFILE_HIGH, - }; - } - break; - case V4L2_PIX_FMT_VP8: - case V4L2_PIX_FMT_VP8_FRAME: - profiles = {VP8PROFILE_ANY}; - break; - case V4L2_PIX_FMT_VP9: - case V4L2_PIX_FMT_VP9_FRAME: - if (!get_supported_profiles(kCodecVP9, &profiles)) { - DLOG(WARNING) << "Driver doesn't support QUERY VP9 profiles, " - << "use default values, Profile0"; - profiles = {VP9PROFILE_PROFILE0}; - } - break; - default: - VLOGF(1) << "Unhandled pixelformat " << FourccToString(pix_fmt); - return {}; - } - - // Erase duplicated profiles. - std::sort(profiles.begin(), profiles.end()); - profiles.erase(std::unique(profiles.begin(), profiles.end()), profiles.end()); - return profiles; -} - -// static -int32_t V4L2Device::VideoCodecProfileToV4L2H264Profile( - VideoCodecProfile profile) { - switch (profile) { - case H264PROFILE_BASELINE: - return V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE; - case H264PROFILE_MAIN: - return V4L2_MPEG_VIDEO_H264_PROFILE_MAIN; - case H264PROFILE_EXTENDED: - return V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED; - case H264PROFILE_HIGH: - return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH; - case H264PROFILE_HIGH10PROFILE: - return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_10; - case H264PROFILE_HIGH422PROFILE: - return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_422; - case H264PROFILE_HIGH444PREDICTIVEPROFILE: - return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE; - case H264PROFILE_SCALABLEBASELINE: - return V4L2_MPEG_VIDEO_H264_PROFILE_SCALABLE_BASELINE; - case H264PROFILE_SCALABLEHIGH: - return V4L2_MPEG_VIDEO_H264_PROFILE_SCALABLE_HIGH; - case H264PROFILE_STEREOHIGH: - return V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH; - case H264PROFILE_MULTIVIEWHIGH: - return V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH; - default: - DVLOGF(1) << "Add more cases as needed"; - return -1; - } -} - -// static -int32_t V4L2Device::H264LevelIdcToV4L2H264Level(uint8_t level_idc) { - switch (level_idc) { - case 10: - return V4L2_MPEG_VIDEO_H264_LEVEL_1_0; - case 9: - return V4L2_MPEG_VIDEO_H264_LEVEL_1B; - case 11: - return V4L2_MPEG_VIDEO_H264_LEVEL_1_1; - case 12: - return V4L2_MPEG_VIDEO_H264_LEVEL_1_2; - case 13: - return V4L2_MPEG_VIDEO_H264_LEVEL_1_3; - case 20: - return V4L2_MPEG_VIDEO_H264_LEVEL_2_0; - case 21: - return V4L2_MPEG_VIDEO_H264_LEVEL_2_1; - case 22: - return V4L2_MPEG_VIDEO_H264_LEVEL_2_2; - case 30: - return V4L2_MPEG_VIDEO_H264_LEVEL_3_0; - case 31: - return V4L2_MPEG_VIDEO_H264_LEVEL_3_1; - case 32: - return V4L2_MPEG_VIDEO_H264_LEVEL_3_2; - case 40: - return V4L2_MPEG_VIDEO_H264_LEVEL_4_0; - case 41: - return V4L2_MPEG_VIDEO_H264_LEVEL_4_1; - case 42: - return V4L2_MPEG_VIDEO_H264_LEVEL_4_2; - case 50: - return V4L2_MPEG_VIDEO_H264_LEVEL_5_0; - case 51: - return V4L2_MPEG_VIDEO_H264_LEVEL_5_1; - default: - DVLOGF(1) << "Unrecognized level_idc: " << static_cast<int>(level_idc); - return -1; - } -} - -// static -Size V4L2Device::AllocatedSizeFromV4L2Format(const struct v4l2_format& format) { - Size coded_size; - Size visible_size; - VideoPixelFormat frame_format = PIXEL_FORMAT_UNKNOWN; - size_t bytesperline = 0; - // Total bytes in the frame. - size_t sizeimage = 0; - - if (V4L2_TYPE_IS_MULTIPLANAR(format.type)) { - DCHECK_GT(format.fmt.pix_mp.num_planes, 0); - bytesperline = - base::checked_cast<int>(format.fmt.pix_mp.plane_fmt[0].bytesperline); - for (size_t i = 0; i < format.fmt.pix_mp.num_planes; ++i) { - sizeimage += - base::checked_cast<int>(format.fmt.pix_mp.plane_fmt[i].sizeimage); - } - visible_size.SetSize(base::checked_cast<int>(format.fmt.pix_mp.width), - base::checked_cast<int>(format.fmt.pix_mp.height)); - const uint32_t pix_fmt = format.fmt.pix_mp.pixelformat; - const auto frame_fourcc = Fourcc::FromV4L2PixFmt(pix_fmt); - if (!frame_fourcc) { - VLOGF(1) << "Unsupported format " << FourccToString(pix_fmt); - return coded_size; - } - frame_format = frame_fourcc->ToVideoPixelFormat(); - } else { - bytesperline = base::checked_cast<int>(format.fmt.pix.bytesperline); - sizeimage = base::checked_cast<int>(format.fmt.pix.sizeimage); - visible_size.SetSize(base::checked_cast<int>(format.fmt.pix.width), - base::checked_cast<int>(format.fmt.pix.height)); - const uint32_t fourcc = format.fmt.pix.pixelformat; - const auto frame_fourcc = Fourcc::FromV4L2PixFmt(fourcc); - if (!frame_fourcc) { - VLOGF(1) << "Unsupported format " << FourccToString(fourcc); - return coded_size; - } - frame_format = frame_fourcc ? frame_fourcc->ToVideoPixelFormat() - : PIXEL_FORMAT_UNKNOWN; - } - - // V4L2 does not provide per-plane bytesperline (bpl) when different - // components are sharing one physical plane buffer. In this case, it only - // provides bpl for the first component in the plane. So we can't depend on it - // for calculating height, because bpl may vary within one physical plane - // buffer. For example, YUV420 contains 3 components in one physical plane, - // with Y at 8 bits per pixel, and Cb/Cr at 4 bits per pixel per component, - // but we only get 8 pits per pixel from bytesperline in physical plane 0. - // So we need to get total frame bpp from elsewhere to calculate coded height. - - // We need bits per pixel for one component only to calculate - // coded_width from bytesperline. - int plane_horiz_bits_per_pixel = - VideoFrame::PlaneHorizontalBitsPerPixel(frame_format, 0); - - // Adding up bpp for each component will give us total bpp for all components. - int total_bpp = 0; - for (size_t i = 0; i < VideoFrame::NumPlanes(frame_format); ++i) - total_bpp += VideoFrame::PlaneBitsPerPixel(frame_format, i); - - if (sizeimage == 0 || bytesperline == 0 || plane_horiz_bits_per_pixel == 0 || - total_bpp == 0 || (bytesperline * 8) % plane_horiz_bits_per_pixel != 0) { - VLOGF(1) << "Invalid format provided"; - return coded_size; - } - - // Coded width can be calculated by taking the first component's bytesperline, - // which in V4L2 always applies to the first component in physical plane - // buffer. - int coded_width = bytesperline * 8 / plane_horiz_bits_per_pixel; - // Sizeimage is coded_width * coded_height * total_bpp. - int coded_height = sizeimage * 8 / coded_width / total_bpp; - - coded_size.SetSize(coded_width, coded_height); - DVLOGF(3) << "coded_size=" << coded_size.ToString(); - - // Sanity checks. Calculated coded size has to contain given visible size - // and fulfill buffer byte size requirements. - DCHECK(Rect(coded_size).Contains(Rect(visible_size))); - DCHECK_LE(sizeimage, VideoFrame::AllocationSize(frame_format, coded_size)); - - return coded_size; -} - -// static -const char* V4L2Device::V4L2MemoryToString(const v4l2_memory memory) { - switch (memory) { - case V4L2_MEMORY_MMAP: - return "V4L2_MEMORY_MMAP"; - case V4L2_MEMORY_USERPTR: - return "V4L2_MEMORY_USERPTR"; - case V4L2_MEMORY_DMABUF: - return "V4L2_MEMORY_DMABUF"; - case V4L2_MEMORY_OVERLAY: - return "V4L2_MEMORY_OVERLAY"; - default: - return "UNKNOWN"; - } -} - -// static -const char* V4L2Device::V4L2BufferTypeToString( - const enum v4l2_buf_type buf_type) { - switch (buf_type) { - case V4L2_BUF_TYPE_VIDEO_OUTPUT: - return "OUTPUT"; - case V4L2_BUF_TYPE_VIDEO_CAPTURE: - return "CAPTURE"; - case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - return "OUTPUT_MPLANE"; - case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: - return "CAPTURE_MPLANE"; - default: - return "UNKNOWN"; - } -} - -// static -std::string V4L2Device::V4L2FormatToString(const struct v4l2_format& format) { - std::ostringstream s; - s << "v4l2_format type: " << format.type; - if (format.type == V4L2_BUF_TYPE_VIDEO_CAPTURE || - format.type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { - // single-planar - const struct v4l2_pix_format& pix = format.fmt.pix; - s << ", width_height: " << Size(pix.width, pix.height).ToString() - << ", pixelformat: " << FourccToString(pix.pixelformat) - << ", field: " << pix.field << ", bytesperline: " << pix.bytesperline - << ", sizeimage: " << pix.sizeimage; - } else if (V4L2_TYPE_IS_MULTIPLANAR(format.type)) { - const struct v4l2_pix_format_mplane& pix_mp = format.fmt.pix_mp; - // As long as num_planes's type is uint8_t, ostringstream treats it as a - // char instead of an integer, which is not what we want. Casting - // pix_mp.num_planes unsigned int solves the issue. - s << ", width_height: " << Size(pix_mp.width, pix_mp.height).ToString() - << ", pixelformat: " << FourccToString(pix_mp.pixelformat) - << ", field: " << pix_mp.field - << ", num_planes: " << static_cast<unsigned int>(pix_mp.num_planes); - for (size_t i = 0; i < pix_mp.num_planes; ++i) { - const struct v4l2_plane_pix_format& plane_fmt = pix_mp.plane_fmt[i]; - s << ", plane_fmt[" << i << "].sizeimage: " << plane_fmt.sizeimage - << ", plane_fmt[" << i << "].bytesperline: " << plane_fmt.bytesperline; - } - } else { - s << " unsupported yet."; - } - return s.str(); -} - -// static -std::string V4L2Device::V4L2BufferToString(const struct v4l2_buffer& buffer) { - std::ostringstream s; - s << "v4l2_buffer type: " << buffer.type << ", memory: " << buffer.memory - << ", index: " << buffer.index << " bytesused: " << buffer.bytesused - << ", length: " << buffer.length; - if (buffer.type == V4L2_BUF_TYPE_VIDEO_CAPTURE || - buffer.type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { - // single-planar - if (buffer.memory == V4L2_MEMORY_MMAP) { - s << ", m.offset: " << buffer.m.offset; - } else if (buffer.memory == V4L2_MEMORY_USERPTR) { - s << ", m.userptr: " << buffer.m.userptr; - } else if (buffer.memory == V4L2_MEMORY_DMABUF) { - s << ", m.fd: " << buffer.m.fd; - } - } else if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) { - for (size_t i = 0; i < buffer.length; ++i) { - const struct v4l2_plane& plane = buffer.m.planes[i]; - s << ", m.planes[" << i << "](bytesused: " << plane.bytesused - << ", length: " << plane.length - << ", data_offset: " << plane.data_offset; - if (buffer.memory == V4L2_MEMORY_MMAP) { - s << ", m.mem_offset: " << plane.m.mem_offset; - } else if (buffer.memory == V4L2_MEMORY_USERPTR) { - s << ", m.userptr: " << plane.m.userptr; - } else if (buffer.memory == V4L2_MEMORY_DMABUF) { - s << ", m.fd: " << plane.m.fd; - } - s << ")"; - } - } else { - s << " unsupported yet."; - } - return s.str(); -} - -// static -base::Optional<VideoFrameLayout> V4L2Device::V4L2FormatToVideoFrameLayout( - const struct v4l2_format& format) { - if (!V4L2_TYPE_IS_MULTIPLANAR(format.type)) { - VLOGF(1) << "v4l2_buf_type is not multiplanar: " << std::hex << "0x" - << format.type; - return base::nullopt; - } - const v4l2_pix_format_mplane& pix_mp = format.fmt.pix_mp; - const uint32_t& pix_fmt = pix_mp.pixelformat; - const auto video_fourcc = Fourcc::FromV4L2PixFmt(pix_fmt); - if (!video_fourcc) { - VLOGF(1) << "Failed to convert pixel format to VideoPixelFormat: " - << FourccToString(pix_fmt); - return base::nullopt; - } - const VideoPixelFormat video_format = video_fourcc->ToVideoPixelFormat(); - const size_t num_buffers = pix_mp.num_planes; - const size_t num_color_planes = VideoFrame::NumPlanes(video_format); - if (num_color_planes == 0) { - VLOGF(1) << "Unsupported video format for NumPlanes(): " - << VideoPixelFormatToString(video_format); - return base::nullopt; - } - if (num_buffers > num_color_planes) { - VLOGF(1) << "pix_mp.num_planes: " << num_buffers - << " should not be larger than NumPlanes(" - << VideoPixelFormatToString(video_format) - << "): " << num_color_planes; - return base::nullopt; - } - // Reserve capacity in advance to prevent unnecessary vector reallocation. - std::vector<ColorPlaneLayout> planes; - planes.reserve(num_color_planes); - for (size_t i = 0; i < num_buffers; ++i) { - const v4l2_plane_pix_format& plane_format = pix_mp.plane_fmt[i]; - planes.emplace_back(static_cast<int32_t>(plane_format.bytesperline), 0u, - plane_format.sizeimage); - } - // For the case that #color planes > #buffers, it fills stride of color - // plane which does not map to buffer. - // Right now only some pixel formats are supported: NV12, YUV420, YVU420. - if (num_color_planes > num_buffers) { - const int32_t y_stride = planes[0].stride; - // Note that y_stride is from v4l2 bytesperline and its type is uint32_t. - // It is safe to cast to size_t. - const size_t y_stride_abs = static_cast<size_t>(y_stride); - switch (pix_fmt) { - case V4L2_PIX_FMT_NV12: - // The stride of UV is the same as Y in NV12. - // The height is half of Y plane. - planes.emplace_back(y_stride, y_stride_abs * pix_mp.height, - y_stride_abs * pix_mp.height / 2); - DCHECK_EQ(2u, planes.size()); - break; - case V4L2_PIX_FMT_YUV420: - case V4L2_PIX_FMT_YVU420: { - // The spec claims that two Cx rows (including padding) is exactly as - // long as one Y row (including padding). So stride of Y must be even - // number. - if (y_stride % 2 != 0 || pix_mp.height % 2 != 0) { - VLOGF(1) << "Plane-Y stride and height should be even; stride: " - << y_stride << ", height: " << pix_mp.height; - return base::nullopt; - } - const int32_t half_stride = y_stride / 2; - const size_t plane_0_area = y_stride_abs * pix_mp.height; - const size_t plane_1_area = plane_0_area / 4; - planes.emplace_back(half_stride, plane_0_area, plane_1_area); - planes.emplace_back(half_stride, plane_0_area + plane_1_area, - plane_1_area); - DCHECK_EQ(3u, planes.size()); - break; - } - default: - VLOGF(1) << "Cannot derive stride for each plane for pixel format " - << FourccToString(pix_fmt); - return base::nullopt; - } - } - - // Some V4L2 devices expect buffers to be page-aligned. We cannot detect - // such devices individually, so set this as a video frame layout property. - constexpr size_t buffer_alignment = 0x1000; - if (num_buffers == 1) { - return VideoFrameLayout::CreateWithPlanes( - video_format, Size(pix_mp.width, pix_mp.height), std::move(planes), - buffer_alignment); - } else { - return VideoFrameLayout::CreateMultiPlanar( - video_format, Size(pix_mp.width, pix_mp.height), std::move(planes), - buffer_alignment); - } -} - -// static -size_t V4L2Device::GetNumPlanesOfV4L2PixFmt(uint32_t pix_fmt) { - base::Optional<Fourcc> fourcc = Fourcc::FromV4L2PixFmt(pix_fmt); - if (fourcc && fourcc->IsMultiPlanar()) { - return VideoFrame::NumPlanes(fourcc->ToVideoPixelFormat()); - } - return 1u; -} - -void V4L2Device::GetSupportedResolution(uint32_t pixelformat, - Size* min_resolution, - Size* max_resolution) { - max_resolution->SetSize(0, 0); - min_resolution->SetSize(0, 0); - v4l2_frmsizeenum frame_size; - memset(&frame_size, 0, sizeof(frame_size)); - frame_size.pixel_format = pixelformat; - for (; Ioctl(VIDIOC_ENUM_FRAMESIZES, &frame_size) == 0; ++frame_size.index) { - if (frame_size.type == V4L2_FRMSIZE_TYPE_DISCRETE) { - if (frame_size.discrete.width >= - base::checked_cast<uint32_t>(max_resolution->width()) && - frame_size.discrete.height >= - base::checked_cast<uint32_t>(max_resolution->height())) { - max_resolution->SetSize(frame_size.discrete.width, - frame_size.discrete.height); - } - if (min_resolution->IsEmpty() || - (frame_size.discrete.width <= - base::checked_cast<uint32_t>(min_resolution->width()) && - frame_size.discrete.height <= - base::checked_cast<uint32_t>(min_resolution->height()))) { - min_resolution->SetSize(frame_size.discrete.width, - frame_size.discrete.height); - } - } else if (frame_size.type == V4L2_FRMSIZE_TYPE_STEPWISE || - frame_size.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) { - max_resolution->SetSize(frame_size.stepwise.max_width, - frame_size.stepwise.max_height); - min_resolution->SetSize(frame_size.stepwise.min_width, - frame_size.stepwise.min_height); - break; - } - } - if (max_resolution->IsEmpty()) { - max_resolution->SetSize(1920, 1088); - VLOGF(1) << "GetSupportedResolution failed to get maximum resolution for " - << "fourcc " << FourccToString(pixelformat) << ", fall back to " - << max_resolution->ToString(); - } - if (min_resolution->IsEmpty()) { - min_resolution->SetSize(16, 16); - VLOGF(1) << "GetSupportedResolution failed to get minimum resolution for " - << "fourcc " << FourccToString(pixelformat) << ", fall back to " - << min_resolution->ToString(); - } -} - -std::vector<uint32_t> V4L2Device::EnumerateSupportedPixelformats( - v4l2_buf_type buf_type) { - std::vector<uint32_t> pixelformats; - - v4l2_fmtdesc fmtdesc; - memset(&fmtdesc, 0, sizeof(fmtdesc)); - fmtdesc.type = buf_type; - - for (; Ioctl(VIDIOC_ENUM_FMT, &fmtdesc) == 0; ++fmtdesc.index) { - DVLOGF(3) << "Found " << fmtdesc.description << std::hex << " (0x" - << fmtdesc.pixelformat << ")"; - pixelformats.push_back(fmtdesc.pixelformat); - } - - return pixelformats; -} - -VideoDecodeAccelerator::SupportedProfiles -V4L2Device::EnumerateSupportedDecodeProfiles(const size_t num_formats, - const uint32_t pixelformats[]) { - VideoDecodeAccelerator::SupportedProfiles profiles; - - const auto& supported_pixelformats = - EnumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); - - for (uint32_t pixelformat : supported_pixelformats) { - if (std::find(pixelformats, pixelformats + num_formats, pixelformat) == - pixelformats + num_formats) - continue; - - VideoDecodeAccelerator::SupportedProfile profile; - GetSupportedResolution(pixelformat, &profile.min_resolution, - &profile.max_resolution); - - const auto video_codec_profiles = - V4L2PixFmtToVideoCodecProfiles(pixelformat, false); - - for (const auto& video_codec_profile : video_codec_profiles) { - profile.profile = video_codec_profile; - profiles.push_back(profile); - - DVLOGF(3) << "Found decoder profile " << GetProfileName(profile.profile) - << ", resolutions: " << profile.min_resolution.ToString() << " " - << profile.max_resolution.ToString(); - } - } - - return profiles; -} - -VideoEncodeAccelerator::SupportedProfiles -V4L2Device::EnumerateSupportedEncodeProfiles() { - VideoEncodeAccelerator::SupportedProfiles profiles; - - const auto& supported_pixelformats = - EnumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); - - for (const auto& pixelformat : supported_pixelformats) { - VideoEncodeAccelerator::SupportedProfile profile; - profile.max_framerate_numerator = 30; - profile.max_framerate_denominator = 1; - Size min_resolution; - GetSupportedResolution(pixelformat, &min_resolution, - &profile.max_resolution); - - const auto video_codec_profiles = - V4L2PixFmtToVideoCodecProfiles(pixelformat, true); - - for (const auto& video_codec_profile : video_codec_profiles) { - profile.profile = video_codec_profile; - profiles.push_back(profile); - - DVLOGF(3) << "Found encoder profile " << GetProfileName(profile.profile) - << ", max resolution: " << profile.max_resolution.ToString(); - } - } - - return profiles; -} - -bool V4L2Device::StartPolling(V4L2DevicePoller::EventCallback event_callback, - base::RepeatingClosure error_callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - if (!device_poller_) { - device_poller_ = - std::make_unique<V4L2DevicePoller>(this, "V4L2DeviceThreadPoller"); - } - - bool ret = device_poller_->StartPolling(std::move(event_callback), - std::move(error_callback)); - - if (!ret) - device_poller_ = nullptr; - - return ret; -} - -bool V4L2Device::StopPolling() { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - return !device_poller_ || device_poller_->StopPolling(); -} - -void V4L2Device::SchedulePoll() { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - if (!device_poller_ || !device_poller_->IsPolling()) - return; - - device_poller_->SchedulePoll(); -} - -bool V4L2Device::IsCtrlExposed(uint32_t ctrl_id) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - struct v4l2_queryctrl query_ctrl; - memset(&query_ctrl, 0, sizeof(query_ctrl)); - query_ctrl.id = ctrl_id; - - return Ioctl(VIDIOC_QUERYCTRL, &query_ctrl) == 0; -} - -bool V4L2Device::SetExtCtrls(uint32_t ctrl_class, - std::vector<V4L2ExtCtrl> ctrls) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - if (ctrls.empty()) - return true; - - struct v4l2_ext_controls ext_ctrls; - memset(&ext_ctrls, 0, sizeof(ext_ctrls)); - ext_ctrls.ctrl_class = ctrl_class; - ext_ctrls.count = ctrls.size(); - ext_ctrls.controls = &ctrls[0].ctrl; - return Ioctl(VIDIOC_S_EXT_CTRLS, &ext_ctrls) == 0; -} - -bool V4L2Device::IsCommandSupported(uint32_t command_id) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - struct v4l2_encoder_cmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.cmd = command_id; - - return Ioctl(VIDIOC_TRY_ENCODER_CMD, &cmd) == 0; -} - -bool V4L2Device::HasCapabilities(uint32_t capabilities) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - struct v4l2_capability caps; - memset(&caps, 0, sizeof(caps)); - if (Ioctl(VIDIOC_QUERYCAP, &caps) != 0) { - LOG(ERROR) << "Failed to query capabilities"; - return false; - } - - return (caps.capabilities & capabilities) == capabilities; -} - -} // namespace media diff --git a/accel/v4l2_device.h b/accel/v4l2_device.h deleted file mode 100644 index f00a604..0000000 --- a/accel/v4l2_device.h +++ /dev/null @@ -1,587 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file defines the V4L2Device interface which is used by the -// V4L2DecodeAccelerator class to delegate/pass the device specific -// handling of any of the functionalities. -// Note: ported from Chromium commit head: 2f13d62f0c0d -// Note: the complete v4l2 device code is ported from Chromium, but some parts -// have been removed: -// - All V4L2 request functionality has been removed, as it required a newer -// kernel version. -// - void SetConfigStore() has been removed as it depends on a newer kernel -// version. -// - QueueDMABuf() from native pixmap planes has been removed, as -// NativePixmapPlane have not been ported. -// - GetVideoFrame() is removed as it depends on some helper functions that have -// not been ported. -// - GL-related functionality has been removed: canCreateEGLImageFrom(), -// CreateEGLImage(), CreateGLImage() and GetTextureTarget() -// - V4L2PixFmtToDrmFormat() has been removed, as DRM is not supported yet. - -#ifndef V4L2_DEVICE_H_ -#define V4L2_DEVICE_H_ - -#include <linux/videodev2.h> -#include <stddef.h> -#include <stdint.h> - -#include <queue> -#include <vector> - -#include "base/containers/flat_map.h" -#include "base/files/scoped_file.h" -#include "base/memory/ref_counted.h" - -#include "fourcc.h" -#include "size.h" -#include "v4l2_device_poller.h" -#include "video_codecs.h" -#include "video_decode_accelerator.h" -#include "video_encode_accelerator.h" -#include "video_frame.h" -#include "video_frame_layout.h" -#include "video_pixel_format.h" - -// TODO(mojahsu): remove this once V4L2 headers are updated. -#ifndef V4L2_PIX_FMT_JPEG_RAW -#define V4L2_PIX_FMT_JPEG_RAW v4l2_fourcc('J', 'P', 'G', 'R') -#endif -#ifndef V4L2_CID_JPEG_LUMA_QUANTIZATION -#define V4L2_CID_JPEG_LUMA_QUANTIZATION (V4L2_CID_JPEG_CLASS_BASE + 5) -#endif -#ifndef V4L2_CID_JPEG_CHROMA_QUANTIZATION -#define V4L2_CID_JPEG_CHROMA_QUANTIZATION (V4L2_CID_JPEG_CLASS_BASE + 6) -#endif - -// TODO(b/132589320): remove this once V4L2 header is updated. -#ifndef V4L2_PIX_FMT_MM21 -// MTK 8-bit block mode, two non-contiguous planes. -#define V4L2_PIX_FMT_MM21 v4l2_fourcc('M', 'M', '2', '1') -#endif - -namespace media { - -class V4L2Queue; -class V4L2BufferRefBase; -class V4L2BuffersList; -class V4L2DecodeSurface; - -// Dummy V4L2RequestRef. The full request queue functionality could not be -// ported as it requires a newer kernel header. -class V4L2RequestRef {}; - -// Wrapper for the 'v4l2_ext_control' structure. -struct V4L2ExtCtrl { - V4L2ExtCtrl(uint32_t id); - V4L2ExtCtrl(uint32_t id, int32_t val); - struct v4l2_ext_control ctrl; -}; - -// A unique reference to a buffer for clients to prepare and submit. -// -// Clients can prepare a buffer for queuing using the methods of this class, and -// then either queue it using the Queue() method corresponding to the memory -// type of the buffer, or drop the reference to make the buffer available again. -class V4L2WritableBufferRef { - public: - V4L2WritableBufferRef(V4L2WritableBufferRef&& other); - V4L2WritableBufferRef() = delete; - V4L2WritableBufferRef& operator=(V4L2WritableBufferRef&& other); - - // Return the memory type of the buffer. Useful to e.g. decide which Queue() - // method to use. - enum v4l2_memory Memory() const; - - // Queue a MMAP buffer. - // When requests are supported, a |request_ref| can be passed along this - // the buffer to be submitted. - // If successful, true is returned and the reference to the buffer is dropped - // so this reference becomes invalid. - // In case of error, false is returned and the buffer is returned to the free - // list. - bool QueueMMap(V4L2RequestRef* request_ref = nullptr) &&; - // Queue a USERPTR buffer, assigning |ptrs| as pointer for each plane. - // The size of |ptrs| must be equal to the number of planes of this buffer. - // When requests are supported, a |request_ref| can be passed along this - // the buffer to be submitted. - // If successful, true is returned and the reference to the buffer is dropped - // so this reference becomes invalid. - // In case of error, false is returned and the buffer is returned to the free - // list. - bool QueueUserPtr(const std::vector<void*>& ptrs, - V4L2RequestRef* request_ref = nullptr) &&; - // Queue a DMABUF buffer, assigning |fds| as file descriptors for each plane. - // It is allowed the number of |fds| might be greater than the number of - // planes of this buffer. It happens when the v4l2 pixel format is single - // planar. The fd of the first plane is only used in that case. - // When requests are supported, a |request_ref| can be passed along this - // the buffer to be submitted. - // If successful, true is returned and the reference to the buffer is dropped - // so this reference becomes invalid. - // In case of error, false is returned and the buffer is returned to the free - // list. - bool QueueDMABuf(const std::vector<base::ScopedFD>& scoped_fds, - V4L2RequestRef* request_ref = nullptr) &&; - // Queue a DMABUF buffer, assigning |fds| as file descriptors for each plane. - // It is allowed the number of |fds| might be greater than the number of - // planes of this buffer. It happens when the v4l2 pixel format is single - // planar. The fd of the first plane is only used in that case. - // When requests are supported, a |request_ref| can be passed along this - // the buffer to be submitted. - // If successful, true is returned and the reference to the buffer is dropped - // so this reference becomes invalid. - // In case of error, false is returned and the buffer is returned to the free - // list. - bool QueueDMABuf(const std::vector<int>& fds, - V4L2RequestRef* request_ref = nullptr) &&; - - // Returns the number of planes in this buffer. - size_t PlanesCount() const; - // Returns the size of the requested |plane|, in bytes. - size_t GetPlaneSize(const size_t plane) const; - // Set the size of the requested |plane|, in bytes. It is only valid for - // USERPTR and DMABUF buffers. When using MMAP buffer, this method triggers a - // DCHECK and is a no-op for release builds. - void SetPlaneSize(const size_t plane, const size_t size); - // This method can only be used with MMAP buffers. - // It will return a pointer to the data of the |plane|th plane. - // In case of error (invalid plane index or mapping failed), a nullptr is - // returned. - void* GetPlaneMapping(const size_t plane); - // Set the timestamp field for this buffer. - void SetTimeStamp(const struct timeval& timestamp); - // Return the previously-set timestamp field for this buffer. - const struct timeval& GetTimeStamp() const; - // Set the number of bytes used for |plane|. - void SetPlaneBytesUsed(const size_t plane, const size_t bytes_used); - // Returns the previously-set number of bytes used for |plane|. - size_t GetPlaneBytesUsed(const size_t plane) const; - // Set the data offset for |plane|, in bytes. - void SetPlaneDataOffset(const size_t plane, const size_t data_offset); - - // Return the V4L2 buffer ID of the underlying buffer. - // TODO(acourbot) This is used for legacy clients but should be ultimately - // removed. See crbug/879971 - size_t BufferId() const; - - ~V4L2WritableBufferRef(); - - private: - // Do the actual queue operation once the v4l2_buffer structure is properly - // filled. - // When requests are supported, a |request_ref| can be passed along this - // the buffer to be submitted. - bool DoQueue(V4L2RequestRef* request_ref) &&; - - V4L2WritableBufferRef(const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue); - friend class V4L2BufferRefFactory; - - std::unique_ptr<V4L2BufferRefBase> buffer_data_; - - SEQUENCE_CHECKER(sequence_checker_); - DISALLOW_COPY_AND_ASSIGN(V4L2WritableBufferRef); -}; - -// A reference to a read-only, dequeued buffer. -// -// Clients use this class to query the buffer state and content, and are -// guaranteed that the buffer will not be reused until all references are -// destroyed. -// All methods of this class must be called from the same sequence, but -// instances of V4L2ReadableBuffer objects can be destroyed from any sequence. -// They can even outlive the V4L2 buffers they originate from. This flexibility -// is required because V4L2ReadableBufferRefs can be embedded into VideoFrames, -// which are then passed to other threads and not necessarily destroyed before -// the V4L2Queue buffers are freed. -class V4L2ReadableBuffer - : public base::RefCountedThreadSafe<V4L2ReadableBuffer> { - public: - // Returns whether the V4L2_BUF_FLAG_LAST flag is set for this buffer. - bool IsLast() const; - // Returns whether the V4L2_BUF_FLAG_KEYFRAME flag is set for this buffer. - bool IsKeyframe() const; - // Return the timestamp set by the driver on this buffer. - struct timeval GetTimeStamp() const; - // Returns the number of planes in this buffer. - size_t PlanesCount() const; - // Returns the number of bytes used for |plane|. - size_t GetPlaneBytesUsed(size_t plane) const; - // Returns the data offset for |plane|. - size_t GetPlaneDataOffset(size_t plane) const; - // This method can only be used with MMAP buffers. - // It will return a pointer to the data of the |plane|th plane. - // In case of error (invalid plane index or mapping failed), a nullptr is - // returned. - const void* GetPlaneMapping(const size_t plane) const; - - // Return the V4L2 buffer ID of the underlying buffer. - // TODO(acourbot) This is used for legacy clients but should be ultimately - // removed. See crbug/879971 - size_t BufferId() const; - - private: - friend class V4L2BufferRefFactory; - friend class base::RefCountedThreadSafe<V4L2ReadableBuffer>; - - ~V4L2ReadableBuffer(); - - V4L2ReadableBuffer(const struct v4l2_buffer& v4l2_buffer, - base::WeakPtr<V4L2Queue> queue); - - std::unique_ptr<V4L2BufferRefBase> buffer_data_; - - SEQUENCE_CHECKER(sequence_checker_); - DISALLOW_COPY_AND_ASSIGN(V4L2ReadableBuffer); -}; - -// Shortcut for naming consistency. -using V4L2ReadableBufferRef = scoped_refptr<V4L2ReadableBuffer>; - -class V4L2Device; -class V4L2Buffer; - -// Interface representing a specific queue of a |V4L2Device|. It provides free -// and queued buffer management that is commonly required by clients. -// -// Buffers managed by this class undergo the following cycle: -// 1) Allocated buffers are put into a free buffers pool, indicating that they -// are used neither by the client nor the hardware. -// 2) The client obtains a unique, writable reference to one of the free -// buffers in order to set its content and other parameters. -// 3) The client then queues the buffer obtained in 2), which invalidates its -// reference. The buffer is now prepared to be processed by the hardware. -// 4) Once the hardware is done with the buffer, it is ready to be dequeued by -// the client. The client obtains a read-only, counted reference to the -// buffer and can read its content and metadata, as well as making other -// references to it. The buffer will not be reused until all the references -// are dropped. Once this happens, the buffer goes back to the free list -// described in 1). -class V4L2Queue : public base::RefCountedThreadSafe<V4L2Queue> { - public: - // Set |fourcc| as the current format on this queue. |size| corresponds to the - // desired buffer's dimensions (i.e. width and height members of - // v4l2_pix_format_mplane (if not applicable, pass Size()). - // |buffer_size| is the desired size in bytes of the buffer for single-planar - // formats (i.e. sizeimage of the first plane). It can be set to 0 if not - // relevant for the desired format. - // If the format could be set, then the |v4l2_format| reflecting the actual - // format is returned. It is guaranteed to feature the specified |fourcc|, - // but any other parameter (including |size| and |buffer_size| may have been - // adjusted by the driver, so the caller must check their values. - base::Optional<struct v4l2_format> SetFormat(uint32_t fourcc, - const Size& size, - size_t buffer_size) - WARN_UNUSED_RESULT; - - // Allocate |count| buffers for the current format of this queue, with a - // specific |memory| allocation, and returns the number of buffers allocated - // or zero if an error occurred, or if references to any previously allocated - // buffers are still held by any clients. - // - // The number of allocated buffers may be larger than the number requested, so - // callers must always check the return value. - // - // Calling this method while buffers are still allocated results in an error. - size_t AllocateBuffers(size_t count, - enum v4l2_memory memory) WARN_UNUSED_RESULT; - - // Deallocate all buffers previously allocated by |AllocateBuffers|. Any - // references to buffers previously allocated held by the client must be - // released, or this call will fail. - bool DeallocateBuffers(); - - // Returns the memory usage of v4l2 buffers owned by this V4L2Queue which are - // mapped in user space memory. - size_t GetMemoryUsage() const; - - // Returns |memory_|, memory type of last buffers allocated by this V4L2Queue. - v4l2_memory GetMemoryType() const; - - // Return a reference to a free buffer for the caller to prepare and submit, - // or nullopt if no buffer is currently free. - // - // If the caller discards the returned reference, the underlying buffer is - // made available to clients again. - base::Optional<V4L2WritableBufferRef> GetFreeBuffer(); - base::Optional<V4L2WritableBufferRef> GetFreeBuffer( - size_t requested_buffer_id); - - // Attempt to dequeue a buffer, and return a reference to it if one was - // available. - // - // The first element of the returned pair will be false if an error occurred, - // in which case the second element will be nullptr. If no error occurred, - // then the first element will be true and the second element will contain a - // reference to the dequeued buffer if one was available, or nullptr - // otherwise. - // Dequeued buffers will not be reused by the driver until all references to - // them are dropped. - std::pair<bool, V4L2ReadableBufferRef> DequeueBuffer(); - - // Returns true if this queue is currently streaming. - bool IsStreaming() const; - // If not currently streaming, starts streaming. Returns true if we started - // streaming, or were already streaming, or false if we were not streaming - // and an error occurred when attempting to start the stream. On failure, any - // previously-queued buffers will be dequeued without processing and made - // available to the client, while any buffers held by the client will remain - // unchanged and their ownership will remain with the client. - bool Streamon(); - // If currently streaming, stops streaming. Also make all queued buffers - // available to the client again regardless of the streaming state. - // If an error occurred while attempting to stop streaming, then false is - // returned and queued buffers are left untouched since the V4L2 queue may - // still be using them. - bool Streamoff(); - - // Returns the number of buffers currently allocated for this queue. - size_t AllocatedBuffersCount() const; - // Returns the number of currently free buffers on this queue. - size_t FreeBuffersCount() const; - // Returns the number of buffers currently queued on this queue. - size_t QueuedBuffersCount() const; - - // Returns true if requests are supported by this queue. - bool SupportsRequests(); - - private: - ~V4L2Queue(); - - // Called when clients request a buffer to be queued. - bool QueueBuffer(struct v4l2_buffer* v4l2_buffer); - - const enum v4l2_buf_type type_; - enum v4l2_memory memory_ = V4L2_MEMORY_MMAP; - bool is_streaming_ = false; - // Set to true if the queue supports requests. - bool supports_requests_ = false; - size_t planes_count_ = 0; - // Current format as set by SetFormat. - base::Optional<struct v4l2_format> current_format_; - - std::vector<std::unique_ptr<V4L2Buffer>> buffers_; - - // Buffers that are available for client to get and submit. - // Buffers in this list are not referenced by anyone else than ourselves. - scoped_refptr<V4L2BuffersList> free_buffers_; - // Buffers that have been queued by the client, and not dequeued yet. - std::set<size_t> queued_buffers_; - - scoped_refptr<V4L2Device> device_; - // Callback to call in this queue's destructor. - base::OnceClosure destroy_cb_; - - V4L2Queue(scoped_refptr<V4L2Device> dev, - enum v4l2_buf_type type, - base::OnceClosure destroy_cb); - friend class V4L2QueueFactory; - friend class V4L2BufferRefBase; - friend class base::RefCountedThreadSafe<V4L2Queue>; - - SEQUENCE_CHECKER(sequence_checker_); - - base::WeakPtrFactory<V4L2Queue> weak_this_factory_; - - DISALLOW_COPY_AND_ASSIGN(V4L2Queue); -}; - -class V4L2Device : public base::RefCountedThreadSafe<V4L2Device> { - public: - // Utility format conversion functions - // If there is no corresponding single- or multi-planar format, returns 0. - static uint32_t VideoCodecProfileToV4L2PixFmt(VideoCodecProfile profile, - bool slice_based); - static VideoCodecProfile V4L2ProfileToVideoCodecProfile(VideoCodec codec, - uint32_t profile); - std::vector<VideoCodecProfile> V4L2PixFmtToVideoCodecProfiles( - uint32_t pix_fmt, - bool is_encoder); - // Calculates the largest plane's allocation size requested by a V4L2 device. - static Size AllocatedSizeFromV4L2Format(const struct v4l2_format& format); - - // Convert required H264 profile and level to V4L2 enums. - static int32_t VideoCodecProfileToV4L2H264Profile(VideoCodecProfile profile); - static int32_t H264LevelIdcToV4L2H264Level(uint8_t level_idc); - - // Converts v4l2_memory to a string. - static const char* V4L2MemoryToString(const v4l2_memory memory); - - // Returns the printable name of a v4l2_buf_type. - static const char* V4L2BufferTypeToString(const enum v4l2_buf_type buf_type); - - // Composes human readable string of v4l2_format. - static std::string V4L2FormatToString(const struct v4l2_format& format); - - // Composes human readable string of v4l2_buffer. - static std::string V4L2BufferToString(const struct v4l2_buffer& buffer); - - // Composes VideoFrameLayout based on v4l2_format. - // If error occurs, it returns base::nullopt. - static base::Optional<VideoFrameLayout> V4L2FormatToVideoFrameLayout( - const struct v4l2_format& format); - - // Returns number of planes of |pix_fmt|. - static size_t GetNumPlanesOfV4L2PixFmt(uint32_t pix_fmt); - - enum class Type { - kDecoder, - kEncoder, - kImageProcessor, - kJpegDecoder, - kJpegEncoder, - }; - - // Create and initialize an appropriate V4L2Device instance for the current - // platform, or return nullptr if not available. - static scoped_refptr<V4L2Device> Create(); - - // Open a V4L2 device of |type| for use with |v4l2_pixfmt|. - // Return true on success. - // The device will be closed in the destructor. - virtual bool Open(Type type, uint32_t v4l2_pixfmt) = 0; - - // Returns the V4L2Queue corresponding to the requested |type|, or nullptr - // if the requested queue type is not supported. - scoped_refptr<V4L2Queue> GetQueue(enum v4l2_buf_type type); - - // Parameters and return value are the same as for the standard ioctl() system - // call. - virtual int Ioctl(int request, void* arg) = 0; - - // This method sleeps until either: - // - SetDevicePollInterrupt() is called (on another thread), - // - |poll_device| is true, and there is new data to be read from the device, - // or an event from the device has arrived; in the latter case - // |*event_pending| will be set to true. - // Returns false on error, true otherwise. - // This method should be called from a separate thread. - virtual bool Poll(bool poll_device, bool* event_pending) = 0; - - // These methods are used to interrupt the thread sleeping on Poll() and force - // it to return regardless of device state, which is usually when the client - // is no longer interested in what happens with the device (on cleanup, - // client state change, etc.). When SetDevicePollInterrupt() is called, Poll() - // will return immediately, and any subsequent calls to it will also do so - // until ClearDevicePollInterrupt() is called. - virtual bool SetDevicePollInterrupt() = 0; - virtual bool ClearDevicePollInterrupt() = 0; - - // Wrappers for standard mmap/munmap system calls. - virtual void* Mmap(void* addr, - unsigned int len, - int prot, - int flags, - unsigned int offset) = 0; - virtual void Munmap(void* addr, unsigned int len) = 0; - - // Return a vector of dmabuf file descriptors, exported for V4L2 buffer with - // |index|, assuming the buffer contains |num_planes| V4L2 planes and is of - // |type|. Return an empty vector on failure. - // The caller is responsible for closing the file descriptors after use. - virtual std::vector<base::ScopedFD> GetDmabufsForV4L2Buffer( - int index, - size_t num_planes, - enum v4l2_buf_type type) = 0; - - // Returns the preferred V4L2 input formats for |type| or empty if none. - virtual std::vector<uint32_t> PreferredInputFormat(Type type) = 0; - - // NOTE: The below methods to query capabilities have a side effect of - // closing the previously-open device, if any, and should not be called after - // Open(). - // TODO(posciak): fix this. - - // Get minimum and maximum resolution for fourcc |pixelformat| and store to - // |min_resolution| and |max_resolution|. - void GetSupportedResolution(uint32_t pixelformat, - Size* min_resolution, - Size* max_resolution); - - std::vector<uint32_t> EnumerateSupportedPixelformats(v4l2_buf_type buf_type); - - // Return V4L2 pixelformats supported by the available image processor - // devices for |buf_type|. - virtual std::vector<uint32_t> GetSupportedImageProcessorPixelformats( - v4l2_buf_type buf_type) = 0; - - // Return supported profiles for decoder, including only profiles for given - // fourcc |pixelformats|. - virtual VideoDecodeAccelerator::SupportedProfiles GetSupportedDecodeProfiles( - const size_t num_formats, - const uint32_t pixelformats[]) = 0; - - // Return supported profiles for encoder. - virtual VideoEncodeAccelerator::SupportedProfiles - GetSupportedEncodeProfiles() = 0; - - // Return true if image processing is supported, false otherwise. - virtual bool IsImageProcessingSupported() = 0; - - // Return true if JPEG codec is supported, false otherwise. - virtual bool IsJpegDecodingSupported() = 0; - virtual bool IsJpegEncodingSupported() = 0; - - // Start polling on this V4L2Device. |event_callback| will be posted to - // the caller's sequence if a buffer is ready to be dequeued and/or a V4L2 - // event has been posted. |error_callback| will be posted to the client's - // sequence if a polling error has occurred. - bool StartPolling(V4L2DevicePoller::EventCallback event_callback, - base::RepeatingClosure error_callback); - // Stop polling this V4L2Device if polling was active. No new events will - // be posted after this method has returned. - bool StopPolling(); - // Schedule a polling event if polling is enabled. This method is intended - // to be called from V4L2Queue, clients should not need to call it directly. - void SchedulePoll(); - - // Check whether the V4L2 control with specified |ctrl_id| is supported. - bool IsCtrlExposed(uint32_t ctrl_id); - // Set the specified list of |ctrls| for the specified |ctrl_class|, returns - // whether the operation succeeded. - bool SetExtCtrls(uint32_t ctrl_class, std::vector<V4L2ExtCtrl> ctrls); - - // Check whether the V4L2 command with specified |command_id| is supported. - bool IsCommandSupported(uint32_t command_id); - // Check whether the V4L2 device has the specified |capabilities|. - bool HasCapabilities(uint32_t capabilities); - - protected: - friend class base::RefCountedThreadSafe<V4L2Device>; - V4L2Device(); - virtual ~V4L2Device(); - - VideoDecodeAccelerator::SupportedProfiles EnumerateSupportedDecodeProfiles( - const size_t num_formats, - const uint32_t pixelformats[]); - - VideoEncodeAccelerator::SupportedProfiles EnumerateSupportedEncodeProfiles(); - - private: - // Perform platform-specific initialization of the device instance. - // Return true on success, false on error or if the particular implementation - // is not available. - virtual bool Initialize() = 0; - - // Associates a v4l2_buf_type to its queue. - base::flat_map<enum v4l2_buf_type, V4L2Queue*> queues_; - - // Callback that is called upon a queue's destruction, to cleanup its pointer - // in queues_. - void OnQueueDestroyed(v4l2_buf_type buf_type); - - // Used if EnablePolling() is called to signal the user that an event - // happened or a buffer is ready to be dequeued. - std::unique_ptr<V4L2DevicePoller> device_poller_; - - // Indicates whether the request queue creation has been tried once. - bool requests_queue_creation_called_ = false; - - SEQUENCE_CHECKER(client_sequence_checker_); -}; - -} // namespace media - -#endif // V4L2_DEVICE_H_ diff --git a/accel/v4l2_device_poller.cc b/accel/v4l2_device_poller.cc deleted file mode 100644 index c5eb820..0000000 --- a/accel/v4l2_device_poller.cc +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 22d34680c8ac - -#include "v4l2_device_poller.h" - -#include <string> - -#include "base/bind.h" -#include "base/threading/sequenced_task_runner_handle.h" -#include "base/threading/thread_checker.h" - -#include "macros.h" -#include "v4l2_device.h" - -namespace media { - -V4L2DevicePoller::V4L2DevicePoller(V4L2Device* const device, - const std::string& thread_name) - : device_(device), - poll_thread_(std::move(thread_name)), - trigger_poll_(base::WaitableEvent::ResetPolicy::AUTOMATIC, - base::WaitableEvent::InitialState::NOT_SIGNALED), - stop_polling_(false) { - DETACH_FROM_SEQUENCE(client_sequence_checker_); -} - -V4L2DevicePoller::~V4L2DevicePoller() { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - StopPolling(); -} - -bool V4L2DevicePoller::StartPolling(EventCallback event_callback, - base::RepeatingClosure error_callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - if (IsPolling()) - return true; - - DVLOGF(4) << "Starting polling"; - - client_task_runner_ = base::SequencedTaskRunnerHandle::Get(); - error_callback_ = error_callback; - - if (!poll_thread_.Start()) { - VLOGF(1) << "Failed to start device poll thread"; - return false; - } - - event_callback_ = std::move(event_callback); - - stop_polling_.store(false); - poll_thread_.task_runner()->PostTask( - FROM_HERE, base::BindOnce(&V4L2DevicePoller::DevicePollTask, - base::Unretained(this))); - - DVLOGF(3) << "Polling thread started"; - - SchedulePoll(); - - return true; -} - -bool V4L2DevicePoller::StopPolling() { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - if (!IsPolling()) - return true; - - DVLOGF(4) << "Stopping polling"; - - stop_polling_.store(true); - - trigger_poll_.Signal(); - - if (!device_->SetDevicePollInterrupt()) { - VLOGF(1) << "Failed to interrupt device poll."; - return false; - } - - DVLOGF(3) << "Stop device poll thread"; - poll_thread_.Stop(); - - if (!device_->ClearDevicePollInterrupt()) { - VLOGF(1) << "Failed to clear interrupting device poll."; - return false; - } - - DVLOGF(4) << "Polling thread stopped"; - - return true; -} - -bool V4L2DevicePoller::IsPolling() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - return poll_thread_.IsRunning(); -} - -void V4L2DevicePoller::SchedulePoll() { - DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); - - // A call to DevicePollTask() will be posted when we actually start polling. - if (!IsPolling()) - return; - - DVLOGF(4) << "Scheduling poll"; - - trigger_poll_.Signal(); -} - -void V4L2DevicePoller::DevicePollTask() { - DCHECK(poll_thread_.task_runner()->RunsTasksInCurrentSequence()); - - while (true) { - DVLOGF(4) << "Waiting for poll to be scheduled."; - trigger_poll_.Wait(); - - if (stop_polling_) { - DVLOGF(4) << "Poll stopped, exiting."; - break; - } - - bool event_pending = false; - DVLOGF(4) << "Polling device."; - if (!device_->Poll(true, &event_pending)) { - VLOGF(1) << "An error occurred while polling, calling error callback"; - client_task_runner_->PostTask(FROM_HERE, error_callback_); - return; - } - - DVLOGF(4) << "Poll returned, calling event callback."; - client_task_runner_->PostTask(FROM_HERE, - base::Bind(event_callback_, event_pending)); - } -} - -} // namespace media diff --git a/accel/v4l2_device_poller.h b/accel/v4l2_device_poller.h deleted file mode 100644 index aac3d8c..0000000 --- a/accel/v4l2_device_poller.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: f65c38dcdac2 - -#ifndef V4L2_V4L2_DEVICE_POLLER_H_ -#define V4L2_V4L2_DEVICE_POLLER_H_ - -#include <atomic> - -#include "base/callback_forward.h" -#include "base/sequence_checker.h" -#include "base/sequenced_task_runner.h" -#include "base/synchronization/waitable_event.h" -#include "base/threading/thread.h" - -namespace media { - -class V4L2Device; - -// Allows a client to poll() on a given V4L2Device and be signaled when -// a buffer is ready to be dequeued or a V4L2 event has been received. Polling -// is done on a dedicated thread, and notifications are delivered in the form of -// a callback to the listener's sequence. -// -// All the methods of this class (with the exception of the constructor) must be -// called from the same sequence. -// -// Note that the service callback may also be called when no particular event -// occurred due to the way poll() works. It is the responsibility of the caller -// to call SchedulePoll() again if there may still be pending events. -class V4L2DevicePoller { - public: - // Callback to be called when buffer ready/V4L2 event has potentially been - // polled. |event| is set if a V4L2 event has been detected. - using EventCallback = base::RepeatingCallback<void(bool event)>; - - // Create a poller for |device|, using a thread named |thread_name|. - // Notification won't start until |StartPolling()| is called. - V4L2DevicePoller(V4L2Device* const device, const std::string& thread_name); - ~V4L2DevicePoller(); - - // Starts polling. |event_callback| will be posted on the caller's sequence - // every time an event occurs. The client is then responsible for consuming - // all pending events in that callback. If new events may still happen after - // the callback has run, the client must call |SchedulePoll()| again in order - // to be notified for them. - // - // If an error occurs during polling, |error_callback| will be posted on the - // caller's sequence. - bool StartPolling(EventCallback event_callback, - base::RepeatingClosure error_callback); - // Stop polling and stop the thread. The poller won't post any new event to - // the caller's sequence after this method has returned. - bool StopPolling(); - // Returns true if currently polling, false otherwise. - bool IsPolling() const; - // Attempts polling the V4L2 device. This method should be called whenever - // doing something that may trigger an event of interest (buffer dequeue or - // V4L2 event), for instance queueing a buffer. In the absence of a pending - // event, poll() will return immediately and the service callback will be - // posted to the caller's sequence. The client is then responsible for calling - // this method again when it is interested in receiving events. - void SchedulePoll(); - - private: - // Perform a poll() on |device_| and post either |service_task_| or - // |error_callback_| on the client's sequence when poll() returns. - void DevicePollTask(); - - // V4L2 device we are polling. - V4L2Device* const device_; - // Thread on which polling is done. - base::Thread poll_thread_; - // Callback to post to the client's sequence when an event occurs. - EventCallback event_callback_; - // Closure to post to the client's sequence when an error occurs. - base::RepeatingClosure error_callback_; - // Client sequence's task runner, where closures are posted. - scoped_refptr<base::SequencedTaskRunner> client_task_runner_; - - // Since poll() returns immediately if no buffers have been queued, we cannot - // rely on it to pause the polling thread until an event occurs. Instead, - // the polling thread will wait on this WaitableEvent (signaled by - // |SchedulePoll| before calling poll(), so we only call it when we are - // actually waiting for an event. - base::WaitableEvent trigger_poll_; - // Set to true when we wish to stop polling, instructing the poller thread - // to break its loop. - std::atomic_bool stop_polling_; - - SEQUENCE_CHECKER(client_sequence_checker_); -}; - -} // namespace media - -#endif // V4L2_V4L2_DEVICE_POLLER_H_ diff --git a/accel/v4l2_video_decode_accelerator.cc b/accel/v4l2_video_decode_accelerator.cc deleted file mode 100644 index 487a01c..0000000 --- a/accel/v4l2_video_decode_accelerator.cc +++ /dev/null @@ -1,1922 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 91175b1 -// Note: image processor is not ported. - -#include "v4l2_video_decode_accelerator.h" - -#include <dlfcn.h> -#include <errno.h> -#include <fcntl.h> -#include <linux/videodev2.h> -#include <poll.h> -#include <string.h> -#include <sys/eventfd.h> -#include <sys/ioctl.h> -#include <sys/mman.h> - -#include <numeric> - -#include "base/bind.h" -#include "base/command_line.h" -#include "base/memory/ptr_util.h" -#include "base/message_loop/message_loop.h" -#include "base/numerics/safe_conversions.h" -#include "base/posix/eintr_wrapper.h" -#include "base/single_thread_task_runner.h" -#include "base/threading/thread_task_runner_handle.h" -#include "build/build_config.h" - -#include "generic_v4l2_device.h" -#include "macros.h" -#include "rect.h" -#include "shared_memory_region.h" - -#define NOTIFY_ERROR(x) \ - do { \ - VLOGF(1) << "Setting error state:" << x; \ - SetErrorState(x); \ - } while (0) - -#define IOCTL_OR_ERROR_RETURN_VALUE(type, arg, value, type_str) \ - do { \ - if (device_->Ioctl(type, arg) != 0) { \ - VPLOGF(1) << "ioctl() failed: " << type_str; \ - NOTIFY_ERROR(PLATFORM_FAILURE); \ - return value; \ - } \ - } while (0) - -#define IOCTL_OR_ERROR_RETURN(type, arg) \ - IOCTL_OR_ERROR_RETURN_VALUE(type, arg, ((void)0), #type) - -#define IOCTL_OR_ERROR_RETURN_FALSE(type, arg) \ - IOCTL_OR_ERROR_RETURN_VALUE(type, arg, false, #type) - -#define IOCTL_OR_LOG_ERROR(type, arg) \ - do { \ - if (device_->Ioctl(type, arg) != 0) \ - VPLOGF(1) << "ioctl() failed: " << #type; \ - } while (0) - -namespace media { - -namespace { -// Copied from older version of V4L2 device. -VideoPixelFormat V4L2PixFmtToVideoPixelFormat(uint32_t pix_fmt) { - switch (pix_fmt) { - case V4L2_PIX_FMT_NV12: - case V4L2_PIX_FMT_NV12M: - return PIXEL_FORMAT_NV12; - - case V4L2_PIX_FMT_YUV420: - case V4L2_PIX_FMT_YUV420M: - return PIXEL_FORMAT_I420; - - case V4L2_PIX_FMT_YVU420: - return PIXEL_FORMAT_YV12; - - case V4L2_PIX_FMT_YUV422M: - return PIXEL_FORMAT_I422; - - case V4L2_PIX_FMT_RGB32: - return PIXEL_FORMAT_ARGB; - - default: - DVLOGF(1) << "Add more cases as needed"; - return PIXEL_FORMAT_UNKNOWN; - } -} -} // namespace - -// static -const uint32_t V4L2VideoDecodeAccelerator::supported_input_fourccs_[] = { - V4L2_PIX_FMT_H264, V4L2_PIX_FMT_VP8, V4L2_PIX_FMT_VP9, -}; - -struct V4L2VideoDecodeAccelerator::BitstreamBufferRef { - BitstreamBufferRef( - base::WeakPtr<Client>& client, - scoped_refptr<base::SingleThreadTaskRunner>& client_task_runner, - BitstreamBuffer buffer, - int32_t input_id); - ~BitstreamBufferRef(); - const base::WeakPtr<Client> client; - const scoped_refptr<base::SingleThreadTaskRunner> client_task_runner; - base::ScopedFD dmabuf_fd; - const size_t offset; - const size_t size; - const int32_t input_id; -}; - -V4L2VideoDecodeAccelerator::BitstreamBufferRef::BitstreamBufferRef( - base::WeakPtr<Client>& client, - scoped_refptr<base::SingleThreadTaskRunner>& client_task_runner, - BitstreamBuffer buffer, - int32_t input_id) - : client(client), - client_task_runner(client_task_runner), - offset(buffer.offset()), - size(buffer.size()), - input_id(input_id) { - base::SharedMemoryHandle handle = buffer.handle(); - // NOTE: BitstreamBuffer and SharedMemoryHandle don't own file descriptor. - // There is no need of duplicating the file descriptor here. - // |handle| is invalid only if flush is dummy. - DCHECK(handle.IsValid() || input_id == kFlushBufferId); - if (handle.IsValid()) - dmabuf_fd = base::ScopedFD(handle.GetHandle()); -} - -V4L2VideoDecodeAccelerator::BitstreamBufferRef::~BitstreamBufferRef() { - if (input_id >= 0) { - client_task_runner->PostTask( - FROM_HERE, - base::Bind(&Client::NotifyEndOfBitstreamBuffer, client, input_id)); - } -} - -V4L2VideoDecodeAccelerator::OutputRecord::OutputRecord() - : state(kFree), - picture_id(-1), - cleared(false) {} - -V4L2VideoDecodeAccelerator::OutputRecord::~OutputRecord() {} - -V4L2VideoDecodeAccelerator::PictureRecord::PictureRecord(bool cleared, - const Picture& picture) - : cleared(cleared), picture(picture) {} - -V4L2VideoDecodeAccelerator::PictureRecord::~PictureRecord() {} - -V4L2VideoDecodeAccelerator::V4L2VideoDecodeAccelerator( - const scoped_refptr<V4L2Device>& device) - : child_task_runner_(base::ThreadTaskRunnerHandle::Get()), - decoder_thread_("V4L2DecoderThread"), - decoder_state_(kUninitialized), - output_mode_(Config::OutputMode::ALLOCATE), - device_(device), - decoder_delay_bitstream_buffer_id_(-1), - decoder_decode_buffer_tasks_scheduled_(0), - decoder_frames_at_client_(0), - decoder_flushing_(false), - decoder_cmd_supported_(false), - flush_awaiting_last_output_buffer_(false), - reset_pending_(false), - input_streamon_(false), - input_buffer_queued_count_(0), - input_buffer_size_(0), - output_streamon_(false), - output_buffer_queued_count_(0), - output_dpb_size_(0), - output_planes_count_(0), - picture_clearing_count_(0), - device_poll_thread_("V4L2DevicePollThread"), - video_profile_(VIDEO_CODEC_PROFILE_UNKNOWN), - input_format_fourcc_(0), - output_format_fourcc_(0), - weak_this_factory_(this) { - weak_this_ = weak_this_factory_.GetWeakPtr(); -} - -V4L2VideoDecodeAccelerator::~V4L2VideoDecodeAccelerator() { - DCHECK(!decoder_thread_.IsRunning()); - DCHECK(!device_poll_thread_.IsRunning()); - DVLOGF(2); - - // These maps have members that should be manually destroyed, e.g. file - // descriptors, mmap() segments, etc. - DCHECK(input_buffer_map_.empty()); - DCHECK(output_buffer_map_.empty()); -} - -bool V4L2VideoDecodeAccelerator::Initialize(const Config& config, - Client* client) { - VLOGF(2) << "profile: " << config.profile - << ", output_mode=" << static_cast<int>(config.output_mode); - DCHECK(child_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kUninitialized); - - if (config.output_mode != Config::OutputMode::IMPORT) { - NOTREACHED() << "Only IMPORT OutputModes are supported"; - return false; - } - - client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client)); - client_ = client_ptr_factory_->GetWeakPtr(); - // If we haven't been set up to decode on separate thread via - // TryToSetupDecodeOnSeparateThread(), use the main thread/client for - // decode tasks. - if (!decode_task_runner_) { - decode_task_runner_ = child_task_runner_; - DCHECK(!decode_client_); - decode_client_ = client_; - } - - video_profile_ = config.profile; - - input_format_fourcc_ = - V4L2Device::VideoCodecProfileToV4L2PixFmt(video_profile_, false); - - if (!device_->Open(V4L2Device::Type::kDecoder, input_format_fourcc_)) { - VLOGF(1) << "Failed to open device for profile: " << config.profile - << " fourcc: " << std::hex << "0x" << input_format_fourcc_; - return false; - } - - // Capabilities check. - struct v4l2_capability caps; - const __u32 kCapsRequired = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QUERYCAP, &caps); - if ((caps.capabilities & kCapsRequired) != kCapsRequired) { - VLOGF(1) << "ioctl() failed: VIDIOC_QUERYCAP" - << ", caps check failed: 0x" << std::hex << caps.capabilities; - return false; - } - - if (!SetupFormats()) - return false; - - if (!decoder_thread_.Start()) { - VLOGF(1) << "decoder thread failed to start"; - return false; - } - - decoder_state_ = kInitialized; - output_mode_ = config.output_mode; - - // InitializeTask will NOTIFY_ERROR on failure. - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::InitializeTask, - base::Unretained(this))); - - return true; -} - -void V4L2VideoDecodeAccelerator::InitializeTask() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kInitialized); - - // Subscribe to the resolution change event. - struct v4l2_event_subscription sub; - memset(&sub, 0, sizeof(sub)); - sub.type = V4L2_EVENT_SOURCE_CHANGE; - IOCTL_OR_ERROR_RETURN(VIDIOC_SUBSCRIBE_EVENT, &sub); - - if (!CreateInputBuffers()) { - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - decoder_cmd_supported_ = IsDecoderCmdSupported(); - - if (!StartDevicePoll()) - return; -} - -void V4L2VideoDecodeAccelerator::Decode( - const BitstreamBuffer& bitstream_buffer) { - DVLOGF(4) << "input_id=" << bitstream_buffer.id() - << ", size=" << bitstream_buffer.size(); - DCHECK(decode_task_runner_->BelongsToCurrentThread()); - - if (bitstream_buffer.id() < 0) { - VLOGF(1) << "Invalid bitstream_buffer, id: " << bitstream_buffer.id(); - if (base::SharedMemory::IsHandleValid(bitstream_buffer.handle())) - base::SharedMemory::CloseHandle(bitstream_buffer.handle()); - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - // DecodeTask() will take care of running a DecodeBufferTask(). - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::DecodeTask, - base::Unretained(this), bitstream_buffer)); -} - -void V4L2VideoDecodeAccelerator::AssignPictureBuffers( - const std::vector<PictureBuffer>& buffers) { - VLOGF(2) << "buffer_count=" << buffers.size(); - DCHECK(child_task_runner_->BelongsToCurrentThread()); - - decoder_thread_.task_runner()->PostTask( - FROM_HERE, - base::Bind(&V4L2VideoDecodeAccelerator::AssignPictureBuffersTask, - base::Unretained(this), buffers)); -} - -void V4L2VideoDecodeAccelerator::AssignPictureBuffersTask( - const std::vector<PictureBuffer>& buffers) { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kAwaitingPictureBuffers); - DCHECK(!output_streamon_); - - uint32_t req_buffer_count = output_dpb_size_ + kDpbOutputBufferExtraCount; - - if (buffers.size() < req_buffer_count) { - VLOGF(1) << "Failed to provide requested picture buffers. (Got " - << buffers.size() << ", requested " << req_buffer_count << ")"; - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - // S_FMT on output queue if frame size allocated by gralloc is different from - // the frame size given by driver. NOTE: This S_FMT is not needed if memory - // type in output queue is MMAP because the driver allocates memory. - const Size& allocated_coded_size = buffers[0].size(); - if (allocated_coded_size != coded_size_) { - struct v4l2_format format = {}; - format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - format.fmt.pix_mp.width = allocated_coded_size.width(); - format.fmt.pix_mp.height = allocated_coded_size.height(); - format.fmt.pix_mp.pixelformat = output_format_fourcc_; - format.fmt.pix_mp.num_planes = output_planes_count_; - IOCTL_OR_ERROR_RETURN(VIDIOC_S_FMT, &format); - coded_size_.SetSize(format.fmt.pix_mp.width, format.fmt.pix_mp.height); - const Size& new_visible_size = GetVisibleSize(coded_size_); - if (new_visible_size != visible_size_) { - VLOGF(1) << "Visible size is changed by resetting coded_size," - << "the previous visible size=" << visible_size_.ToString() - << "the current visible size=" << new_visible_size.ToString(); - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - } - - // Allocate the output buffers. - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = buffers.size(); - reqbufs.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - reqbufs.memory = V4L2_MEMORY_DMABUF; - IOCTL_OR_ERROR_RETURN(VIDIOC_REQBUFS, &reqbufs); - - if (reqbufs.count < buffers.size()) { - VLOGF(1) << "Could not allocate enough output buffers"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - DCHECK(free_output_buffers_.empty()); - DCHECK(output_buffer_map_.empty()); - output_buffer_map_.resize(buffers.size()); - - // Always use IMPORT output mode for Android solution. - DCHECK_EQ(output_mode_, Config::OutputMode::IMPORT); - - for (size_t i = 0; i < output_buffer_map_.size(); ++i) { - OutputRecord& output_record = output_buffer_map_[i]; - DCHECK_EQ(output_record.state, kFree); - DCHECK_EQ(output_record.picture_id, -1); - DCHECK_EQ(output_record.cleared, false); - - output_record.picture_id = buffers[i].id(); - - // This will remain kAtClient until ImportBufferForPicture is called, either - // by the client, or by ourselves, if we are allocating. - output_record.state = kAtClient; - - DVLOGF(3) << "buffer[" << i << "]: picture_id=" << output_record.picture_id; - } -} - -void V4L2VideoDecodeAccelerator::ImportBufferForPicture( - int32_t picture_buffer_id, - VideoPixelFormat pixel_format, - const NativePixmapHandle& native_pixmap_handle) { - DVLOGF(3) << "picture_buffer_id=" << picture_buffer_id; - DCHECK(child_task_runner_->BelongsToCurrentThread()); - - if (output_mode_ != Config::OutputMode::IMPORT) { - VLOGF(1) << "Cannot import in non-import mode"; - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - if (pixel_format != V4L2PixFmtToVideoPixelFormat(output_format_fourcc_)) { - VLOGF(1) << "Unsupported import format: " << pixel_format; - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - std::vector<base::ScopedFD> dmabuf_fds; - std::vector<size_t> offsets; - for (const auto& plane : native_pixmap_handle.planes) - offsets.push_back(plane.offset); - - for (const auto& fd : native_pixmap_handle.fds) { - DCHECK_NE(fd.fd, -1); - dmabuf_fds.push_back(base::ScopedFD(fd.fd)); - } - - decoder_thread_.task_runner()->PostTask( - FROM_HERE, - base::Bind(&V4L2VideoDecodeAccelerator::ImportBufferForPictureTask, - base::Unretained(this), picture_buffer_id, - std::move(offsets), base::Passed(&dmabuf_fds))); -} - -void V4L2VideoDecodeAccelerator::ImportBufferForPictureTask( - int32_t picture_buffer_id, - std::vector<size_t> offsets, - std::vector<base::ScopedFD> dmabuf_fds) { - DVLOGF(3) << "picture_buffer_id=" << picture_buffer_id - << ", dmabuf_fds.size()=" << dmabuf_fds.size(); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - const auto iter = - std::find_if(output_buffer_map_.begin(), output_buffer_map_.end(), - [picture_buffer_id](const OutputRecord& output_record) { - return output_record.picture_id == picture_buffer_id; - }); - if (iter == output_buffer_map_.end()) { - // It's possible that we've already posted a DismissPictureBuffer for this - // picture, but it has not yet executed when this ImportBufferForPicture was - // posted to us by the client. In that case just ignore this (we've already - // dismissed it and accounted for that). - DVLOGF(3) << "got picture id=" << picture_buffer_id - << " not in use (anymore?)."; - return; - } - - if (iter->state != kAtClient) { - VLOGF(1) << "Cannot import buffer not owned by client"; - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - size_t index = iter - output_buffer_map_.begin(); - DCHECK_EQ(std::count(free_output_buffers_.begin(), free_output_buffers_.end(), - index), - 0); - - iter->state = kFree; - - DCHECK_LE(output_planes_count_, dmabuf_fds.size()); - - iter->output_fds = std::move(dmabuf_fds); - iter->offsets = std::move(offsets); - - if (decoder_state_ == kAwaitingPictureBuffers) - decoder_state_ = kDecoding; - - free_output_buffers_.push_back(index); - if (decoder_state_ != kChangingResolution) { - Enqueue(); - ScheduleDecodeBufferTaskIfNeeded(); - } -} - -void V4L2VideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_buffer_id) { - DVLOGF(4) << "picture_buffer_id=" << picture_buffer_id; - // Must be run on child thread, as we'll insert a sync in the EGL context. - DCHECK(child_task_runner_->BelongsToCurrentThread()); - - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::ReusePictureBufferTask, - base::Unretained(this), picture_buffer_id)); -} - -void V4L2VideoDecodeAccelerator::Flush() { - VLOGF(2); - DCHECK(child_task_runner_->BelongsToCurrentThread()); - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::FlushTask, - base::Unretained(this))); -} - -void V4L2VideoDecodeAccelerator::Reset() { - VLOGF(2); - DCHECK(child_task_runner_->BelongsToCurrentThread()); - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::ResetTask, - base::Unretained(this))); -} - -void V4L2VideoDecodeAccelerator::Destroy() { - VLOGF(2); - DCHECK(child_task_runner_->BelongsToCurrentThread()); - - // We're destroying; cancel all callbacks. - client_ptr_factory_.reset(); - weak_this_factory_.InvalidateWeakPtrs(); - - // If the decoder thread is running, destroy using posted task. - if (decoder_thread_.IsRunning()) { - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::DestroyTask, - base::Unretained(this))); - // DestroyTask() will cause the decoder_thread_ to flush all tasks. - decoder_thread_.Stop(); - } else { - // Otherwise, call the destroy task directly. - DestroyTask(); - } - - delete this; - VLOGF(2) << "Destroyed."; -} - -bool V4L2VideoDecodeAccelerator::TryToSetupDecodeOnSeparateThread( - const base::WeakPtr<Client>& decode_client, - const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) { - VLOGF(2); - decode_client_ = decode_client; - decode_task_runner_ = decode_task_runner; - return true; -} - -// static -VideoDecodeAccelerator::SupportedProfiles -V4L2VideoDecodeAccelerator::GetSupportedProfiles() { - scoped_refptr<V4L2Device> device(new GenericV4L2Device()); - if (!device) - return SupportedProfiles(); - - return device->GetSupportedDecodeProfiles(arraysize(supported_input_fourccs_), - supported_input_fourccs_); -} - -void V4L2VideoDecodeAccelerator::DecodeTask( - const BitstreamBuffer& bitstream_buffer) { - DVLOGF(4) << "input_id=" << bitstream_buffer.id(); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - - // Invalid handle. - if (!bitstream_buffer.handle().IsValid()) { - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - int bitstream_id = bitstream_buffer.id(); - std::unique_ptr<BitstreamBufferRef> bitstream_record(new BitstreamBufferRef( - decode_client_, decode_task_runner_, - std::move(bitstream_buffer), bitstream_id)); - - // Skip empty buffer. - if (bitstream_record->size == 0) - return; - - if (decoder_state_ == kResetting || decoder_flushing_) { - // In the case that we're resetting or flushing, we need to delay decoding - // the BitstreamBuffers that come after the Reset() or Flush() call. When - // we're here, we know that this DecodeTask() was scheduled by a Decode() - // call that came after (in the client thread) the Reset() or Flush() call; - // thus set up the delay if necessary. - if (decoder_delay_bitstream_buffer_id_ == -1) - decoder_delay_bitstream_buffer_id_ = bitstream_record->input_id; - } else if (decoder_state_ == kError) { - VLOGF(2) << "early out: kError state"; - return; - } - - decoder_input_queue_.push(std::move(bitstream_record)); - decoder_decode_buffer_tasks_scheduled_++; - DecodeBufferTask(); -} - -void V4L2VideoDecodeAccelerator::DecodeBufferTask() { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - - decoder_decode_buffer_tasks_scheduled_--; - - if (decoder_state_ != kInitialized && decoder_state_ != kDecoding) { - DVLOGF(3) << "early out: state=" << decoder_state_; - return; - } - - if (decoder_current_bitstream_buffer_ == NULL) { - if (decoder_input_queue_.empty()) { - // We're waiting for a new buffer -- exit without scheduling a new task. - return; - } - const std::unique_ptr<BitstreamBufferRef>& buffer_ref = decoder_input_queue_.front(); - if (decoder_delay_bitstream_buffer_id_ == buffer_ref->input_id) { - // We're asked to delay decoding on this and subsequent buffers. - return; - } - - // Setup to use the next buffer. - decoder_current_bitstream_buffer_ = std::move(decoder_input_queue_.front()); - decoder_input_queue_.pop(); - const auto& dmabuf_fd = decoder_current_bitstream_buffer_->dmabuf_fd; - if (dmabuf_fd.is_valid()) { - DVLOGF(4) << "reading input_id=" - << decoder_current_bitstream_buffer_->input_id - << ", fd=" << dmabuf_fd.get() - << ", size=" << decoder_current_bitstream_buffer_->size; - } else { - DCHECK_EQ(decoder_current_bitstream_buffer_->input_id, kFlushBufferId); - DVLOGF(4) << "reading input_id=kFlushBufferId"; - } - } - bool schedule_task = false; - const auto& dmabuf_fd = decoder_current_bitstream_buffer_->dmabuf_fd; - if (!dmabuf_fd.is_valid()) { - // This is a dummy buffer, queued to flush the pipe. Flush. - DCHECK_EQ(decoder_current_bitstream_buffer_->input_id, kFlushBufferId); - if (TrySubmitInputFrame()) { - VLOGF(2) << "enqueued flush buffer"; - schedule_task = true; - } else { - // If we failed to enqueue the empty buffer (due to pipeline - // backpressure), don't advance the bitstream buffer queue, and don't - // schedule the next task. This bitstream buffer queue entry will get - // reprocessed when the pipeline frees up. - schedule_task = false; - } - } else { - DCHECK_GT(decoder_current_bitstream_buffer_->size, 0u); - switch (decoder_state_) { - case kInitialized: - schedule_task = DecodeBufferInitial(); - break; - case kDecoding: - schedule_task = DecodeBufferContinue(); - break; - default: - NOTIFY_ERROR(ILLEGAL_STATE); - return; - } - } - if (decoder_state_ == kError) { - // Failed during decode. - return; - } - - if (schedule_task) { - ScheduleDecodeBufferTaskIfNeeded(); - } -} - -void V4L2VideoDecodeAccelerator::ScheduleDecodeBufferTaskIfNeeded() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - // If we're behind on tasks, schedule another one. - int buffers_to_decode = decoder_input_queue_.size(); - if (decoder_current_bitstream_buffer_ != NULL) - buffers_to_decode++; - if (decoder_decode_buffer_tasks_scheduled_ < buffers_to_decode) { - decoder_decode_buffer_tasks_scheduled_++; - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::DecodeBufferTask, - base::Unretained(this))); - } -} - -bool V4L2VideoDecodeAccelerator::DecodeBufferInitial() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kInitialized); - // Initial decode. We haven't been able to get output stream format info yet. - // Get it, and start decoding. - - if (!TrySubmitInputFrame()) - return false; - - // Recycle buffers. - Dequeue(); - - // If an initial resolution change event is not done yet, a driver probably - // needs more stream to decode format. - // Return true and schedule next buffer without changing status to kDecoding. - // If the initial resolution change is done and coded size is known, we may - // still have to wait for AssignPictureBuffers() and output buffers to be - // allocated. - if (coded_size_.IsEmpty() || output_buffer_map_.empty()) { - // Need more stream to decode format, return true and schedule next buffer. - return true; - } - - decoder_state_ = kDecoding; - ScheduleDecodeBufferTaskIfNeeded(); - return true; -} - -bool V4L2VideoDecodeAccelerator::DecodeBufferContinue() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kDecoding); - - return TrySubmitInputFrame(); -} - -bool V4L2VideoDecodeAccelerator::TrySubmitInputFrame() { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - DCHECK_NE(decoder_state_, kResetting); - DCHECK_NE(decoder_state_, kError); - CHECK(decoder_current_bitstream_buffer_); - - // No free input buffer. - if (free_input_buffers_.empty()) - return false; - - const int input_buffer_index = free_input_buffers_.back(); - free_input_buffers_.pop_back(); - InputRecord& input_record = input_buffer_map_[input_buffer_index]; - DCHECK(!input_record.bitstream_buffer); - - // Pass the required info to InputRecord. - input_record.bitstream_buffer = std::move(decoder_current_bitstream_buffer_); - // Queue it. - input_ready_queue_.push(input_buffer_index); - DVLOGF(4) << "submitting input_id=" << input_record.bitstream_buffer->input_id; - // Enqueue once since there's new available input for it. - Enqueue(); - - return (decoder_state_ != kError); -} - -void V4L2VideoDecodeAccelerator::ServiceDeviceTask(bool event_pending) { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - - if (decoder_state_ == kResetting) { - DVLOGF(3) << "early out: kResetting state"; - return; - } else if (decoder_state_ == kError) { - DVLOGF(3) << "early out: kError state"; - return; - } else if (decoder_state_ == kChangingResolution) { - DVLOGF(3) << "early out: kChangingResolution state"; - return; - } - - bool resolution_change_pending = false; - if (event_pending) - resolution_change_pending = DequeueResolutionChangeEvent(); - - if (!resolution_change_pending && coded_size_.IsEmpty()) { - // Some platforms do not send an initial resolution change event. - // To work around this, we need to keep checking if the initial resolution - // is known already by explicitly querying the format after each decode, - // regardless of whether we received an event. - // This needs to be done on initial resolution change, - // i.e. when coded_size_.IsEmpty(). - - // Try GetFormatInfo to check if an initial resolution change can be done. - struct v4l2_format format; - Size visible_size; - bool again; - if (GetFormatInfo(&format, &visible_size, &again) && !again) { - resolution_change_pending = true; - DequeueResolutionChangeEvent(); - } - } - - Dequeue(); - Enqueue(); - - // Clear the interrupt fd. - if (!device_->ClearDevicePollInterrupt()) { - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - bool poll_device = false; - // Add fd, if we should poll on it. - // Can be polled as soon as either input or output buffers are queued. - if (input_buffer_queued_count_ + output_buffer_queued_count_ > 0) - poll_device = true; - - // ServiceDeviceTask() should only ever be scheduled from DevicePollTask(), - // so either: - // * device_poll_thread_ is running normally - // * device_poll_thread_ scheduled us, but then a ResetTask() or DestroyTask() - // shut it down, in which case we're either in kResetting or kError states - // respectively, and we should have early-outed already. - DCHECK(device_poll_thread_.message_loop()); - // Queue the DevicePollTask() now. - device_poll_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::DevicePollTask, - base::Unretained(this), poll_device)); - - DVLOGF(3) << "ServiceDeviceTask(): buffer counts: DEC[" - << decoder_input_queue_.size() << "->" - << input_ready_queue_.size() << "] => DEVICE[" - << free_input_buffers_.size() << "+" - << input_buffer_queued_count_ << "/" - << input_buffer_map_.size() << "->" - << free_output_buffers_.size() << "+" - << output_buffer_queued_count_ << "/" - << output_buffer_map_.size() << "] => CLIENT[" - << decoder_frames_at_client_ << "]"; - - ScheduleDecodeBufferTaskIfNeeded(); - if (resolution_change_pending) - StartResolutionChange(); -} - -void V4L2VideoDecodeAccelerator::Enqueue() { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - - // Drain the pipe of completed decode buffers. - const int old_inputs_queued = input_buffer_queued_count_; - while (!input_ready_queue_.empty()) { - const int buffer = input_ready_queue_.front(); - InputRecord& input_record = input_buffer_map_[buffer]; - if (input_record.bitstream_buffer->input_id == kFlushBufferId && decoder_cmd_supported_) { - // Send the flush command after all input buffers are dequeued. This makes - // sure all previous resolution changes have been handled because the - // driver must hold the input buffer that triggers resolution change. The - // driver cannot decode data in it without new output buffers. If we send - // the flush now and a queued input buffer triggers resolution change - // later, the driver will send an output buffer that has - // V4L2_BUF_FLAG_LAST. But some queued input buffer have not been decoded - // yet. Also, V4L2VDA calls STREAMOFF and STREAMON after resolution - // change. They implicitly send a V4L2_DEC_CMD_STOP and V4L2_DEC_CMD_START - // to the decoder. - if (input_buffer_queued_count_ == 0) { - if (!SendDecoderCmdStop()) - return; - input_ready_queue_.pop(); - free_input_buffers_.push_back(buffer); - input_record.bitstream_buffer.reset(); - } else { - break; - } - } else if (!EnqueueInputRecord()) - return; - } - if (old_inputs_queued == 0 && input_buffer_queued_count_ != 0) { - // We just started up a previously empty queue. - // Queue state changed; signal interrupt. - if (!device_->SetDevicePollInterrupt()) { - VPLOGF(1) << "SetDevicePollInterrupt failed"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - // Start VIDIOC_STREAMON if we haven't yet. - if (!input_streamon_) { - __u32 type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMON, &type); - input_streamon_ = true; - } - } - - // Enqueue all the outputs we can. - const int old_outputs_queued = output_buffer_queued_count_; - while (!free_output_buffers_.empty()) { - if (!EnqueueOutputRecord()) - return; - } - if (old_outputs_queued == 0 && output_buffer_queued_count_ != 0) { - // We just started up a previously empty queue. - // Queue state changed; signal interrupt. - if (!device_->SetDevicePollInterrupt()) { - VPLOGF(1) << "SetDevicePollInterrupt(): failed"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - // Start VIDIOC_STREAMON if we haven't yet. - if (!output_streamon_) { - __u32 type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMON, &type); - output_streamon_ = true; - } - } -} - -bool V4L2VideoDecodeAccelerator::DequeueResolutionChangeEvent() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - DVLOGF(3); - - struct v4l2_event ev; - memset(&ev, 0, sizeof(ev)); - - while (device_->Ioctl(VIDIOC_DQEVENT, &ev) == 0) { - if (ev.type == V4L2_EVENT_SOURCE_CHANGE) { - if (ev.u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { - VLOGF(2) << "got resolution change event."; - return true; - } - } else { - VLOGF(1) << "got an event (" << ev.type << ") we haven't subscribed to."; - } - } - return false; -} - -void V4L2VideoDecodeAccelerator::Dequeue() { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - - while (input_buffer_queued_count_ > 0) { - if (!DequeueInputBuffer()) - break; - } - while (output_buffer_queued_count_ > 0) { - if (!DequeueOutputBuffer()) - break; - } - NotifyFlushDoneIfNeeded(); -} - -bool V4L2VideoDecodeAccelerator::DequeueInputBuffer() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_GT(input_buffer_queued_count_, 0); - DCHECK(input_streamon_); - - // Dequeue a completed input (VIDEO_OUTPUT) buffer, and recycle to the free - // list. - struct v4l2_buffer dqbuf; - struct v4l2_plane planes[1]; - memset(&dqbuf, 0, sizeof(dqbuf)); - memset(planes, 0, sizeof(planes)); - dqbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - dqbuf.memory = V4L2_MEMORY_DMABUF; - dqbuf.m.planes = planes; - dqbuf.length = 1; - if (device_->Ioctl(VIDIOC_DQBUF, &dqbuf) != 0) { - if (errno == EAGAIN) { - // EAGAIN if we're just out of buffers to dequeue. - return false; - } - VPLOGF(1) << "ioctl() failed: VIDIOC_DQBUF"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - InputRecord& input_record = input_buffer_map_[dqbuf.index]; - DCHECK(input_record.at_device); - free_input_buffers_.push_back(dqbuf.index); - input_record.at_device = false; - // This will trigger NotifyEndOfBitstreamBuffer(). - input_record.bitstream_buffer.reset(); - input_buffer_queued_count_--; - - return true; -} - -bool V4L2VideoDecodeAccelerator::DequeueOutputBuffer() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_GT(output_buffer_queued_count_, 0); - DCHECK(output_streamon_); - - // Dequeue a completed output (VIDEO_CAPTURE) buffer, and queue to the - // completed queue. - struct v4l2_buffer dqbuf {}; - struct v4l2_plane dqbuf_planes[VIDEO_MAX_PLANES] = {}; - dqbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - dqbuf.memory = V4L2_MEMORY_DMABUF; - dqbuf.m.planes = dqbuf_planes; - dqbuf.length = output_planes_count_; - if (device_->Ioctl(VIDIOC_DQBUF, &dqbuf) != 0) { - if (errno == EAGAIN) { - // EAGAIN if we're just out of buffers to dequeue. - return false; - } else if (errno == EPIPE) { - DVLOGF(3) << "Got EPIPE. Last output buffer was already dequeued."; - return false; - } - VPLOGF(1) << "ioctl() failed: VIDIOC_DQBUF"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - OutputRecord& output_record = output_buffer_map_[dqbuf.index]; - DCHECK_EQ(output_record.state, kAtDevice); - DCHECK_NE(output_record.picture_id, -1); - output_buffer_queued_count_--; - - // Zero-bytes buffers are returned as part of a flush and can be dismissed. - if (dqbuf.m.planes[0].bytesused > 0) { - int32_t bitstream_buffer_id = dqbuf.timestamp.tv_sec; - DCHECK_GE(bitstream_buffer_id, 0); - DVLOGF(4) << "Dequeue output buffer: dqbuf index=" << dqbuf.index - << " bitstream input_id=" << bitstream_buffer_id; - output_record.state = kAtClient; - decoder_frames_at_client_++; - - const Picture picture(output_record.picture_id, bitstream_buffer_id, - Rect(visible_size_), false); - pending_picture_ready_.push(PictureRecord(output_record.cleared, picture)); - SendPictureReady(); - output_record.cleared = true; - } - - if (dqbuf.flags & V4L2_BUF_FLAG_LAST) { - DVLOGF(3) << "Got last output buffer. Waiting last buffer=" - << flush_awaiting_last_output_buffer_; - if (flush_awaiting_last_output_buffer_) { - flush_awaiting_last_output_buffer_ = false; - struct v4l2_decoder_cmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.cmd = V4L2_DEC_CMD_START; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_DECODER_CMD, &cmd); - } - } - return true; -} - -bool V4L2VideoDecodeAccelerator::EnqueueInputRecord() { - DVLOGF(4); - DCHECK(!input_ready_queue_.empty()); - - // Enqueue an input (VIDEO_OUTPUT) buffer. - const int v4l2_buffer_index = input_ready_queue_.front(); - InputRecord& input_record = input_buffer_map_[v4l2_buffer_index]; - DCHECK(!input_record.at_device); - struct v4l2_buffer qbuf {}; - struct v4l2_plane qbuf_plane = {}; - qbuf.index = v4l2_buffer_index; - qbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - qbuf.timestamp.tv_sec = input_record.bitstream_buffer->input_id; - qbuf.memory = V4L2_MEMORY_DMABUF; - qbuf.m.planes = &qbuf_plane; - const std::unique_ptr<BitstreamBufferRef>& buffer = input_record.bitstream_buffer; - if (!buffer->dmabuf_fd.is_valid()) { - // This is a flush case. A driver must handle Flush with V4L2_DEC_CMD_STOP. - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - if (buffer->offset + buffer->size > input_buffer_size_) { - VLOGF(1) << "offset + size of input buffer is larger than buffer size" - << ", offset=" << buffer->offset - << ", size=" << buffer->size - << ", buffer size=" << input_buffer_size_; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - - // TODO(crbug.com/901264): The way to pass an offset within a DMA-buf is - // not defined in V4L2 specification, so we abuse data_offset for now. - // Fix it when we have the right interface, including any necessary - // validation and potential alignment. - qbuf.m.planes[0].m.fd = buffer->dmabuf_fd.get(); - qbuf.m.planes[0].data_offset = buffer->offset; - qbuf.m.planes[0].bytesused = buffer->offset + buffer->size; - // Workaround: filling length should not be needed. This is a bug of - // videobuf2 library. - qbuf.m.planes[0].length = input_buffer_size_; - qbuf.length = 1; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QBUF, &qbuf); - DVLOGF(4) << "enqueued input_id=" << buffer->input_id; - input_ready_queue_.pop(); - - input_record.at_device = true; - input_buffer_queued_count_++; - - return true; -} - -bool V4L2VideoDecodeAccelerator::EnqueueOutputRecord() { - DCHECK(!free_output_buffers_.empty()); - - // Enqueue an output (VIDEO_CAPTURE) buffer. - const int buffer = free_output_buffers_.front(); - DVLOGF(4) << "buffer " << buffer; - OutputRecord& output_record = output_buffer_map_[buffer]; - DCHECK_EQ(output_record.state, kFree); - DCHECK_NE(output_record.picture_id, -1); - struct v4l2_buffer qbuf {}; - struct v4l2_plane qbuf_planes[VIDEO_MAX_PLANES] = {}; - qbuf.index = buffer; - qbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - qbuf.memory = V4L2_MEMORY_DMABUF; - qbuf.m.planes = qbuf_planes; - qbuf.length = output_planes_count_; - DVLOGF(4) << "qbuf.index=" << qbuf.index; - DCHECK_LE(output_planes_count_, output_record.output_fds.size()); - DCHECK_LE(output_planes_count_, output_record.offsets.size()); - // Pass fd and offset info. - for (size_t i = 0; i < output_planes_count_; i++) { - // output_record.output_fds is repeatedly used. We will not close the fd of - // output buffer unless new fds are assigned in ImportBufferForPicture(). - qbuf.m.planes[i].m.fd = output_record.output_fds[i].get(); - qbuf.m.planes[i].data_offset = output_record.offsets[i]; - } - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QBUF, &qbuf); - free_output_buffers_.pop_front(); - output_record.state = kAtDevice; - output_buffer_queued_count_++; - return true; -} - -void V4L2VideoDecodeAccelerator::ReusePictureBufferTask(int32_t picture_buffer_id) { - DVLOGF(4) << "picture_buffer_id=" << picture_buffer_id; - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - // We run ReusePictureBufferTask even if we're in kResetting. - if (decoder_state_ == kError) { - DVLOGF(4) << "early out: kError state"; - return; - } - - if (decoder_state_ == kChangingResolution) { - DVLOGF(4) << "early out: kChangingResolution"; - return; - } - - size_t index; - for (index = 0; index < output_buffer_map_.size(); ++index) - if (output_buffer_map_[index].picture_id == picture_buffer_id) - break; - - if (index >= output_buffer_map_.size()) { - // It's possible that we've already posted a DismissPictureBuffer for this - // picture, but it has not yet executed when this ReusePictureBuffer was - // posted to us by the client. In that case just ignore this (we've already - // dismissed it and accounted for that) and let the sync object get - // destroyed. - DVLOGF(3) << "got picture id= " << picture_buffer_id - << " not in use (anymore?)."; - return; - } - - OutputRecord& output_record = output_buffer_map_[index]; - if (output_record.state != kAtClient) { - VLOGF(1) << "picture_buffer_id not reusable"; - NOTIFY_ERROR(INVALID_ARGUMENT); - return; - } - - output_record.state = kFree; - free_output_buffers_.push_back(index); - decoder_frames_at_client_--; - // We got a buffer back, so enqueue it back. - Enqueue(); -} - -void V4L2VideoDecodeAccelerator::FlushTask() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - if (decoder_state_ == kError) { - VLOGF(2) << "early out: kError state"; - return; - } - - // We don't support stacked flushing. - DCHECK(!decoder_flushing_); - - // Queue up an empty buffer -- this triggers the flush. - // BitstreamBufferRef::dmabuf_fd becomes invalid. - decoder_input_queue_.push(std::make_unique<BitstreamBufferRef>( - decode_client_, decode_task_runner_, BitstreamBuffer(), kFlushBufferId)); - decoder_flushing_ = true; - SendPictureReady(); // Send all pending PictureReady. - - ScheduleDecodeBufferTaskIfNeeded(); -} - -void V4L2VideoDecodeAccelerator::NotifyFlushDoneIfNeeded() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - if (!decoder_flushing_) - return; - - // Pipeline is empty when: - // * Decoder input queue is empty of non-delayed buffers. - // * There is no currently filling input buffer. - // * Input holding queue is empty. - // * All input (VIDEO_OUTPUT) buffers are returned. - // * All image processor buffers are returned. - if (!decoder_input_queue_.empty()) { - if (decoder_input_queue_.front()->input_id != - decoder_delay_bitstream_buffer_id_) { - DVLOGF(3) << "Some input bitstream buffers are not queued."; - return; - } - } - - if ((input_ready_queue_.size() + input_buffer_queued_count_) != 0) { - DVLOGF(3) << "Some input buffers are not dequeued."; - return; - } - if (flush_awaiting_last_output_buffer_) { - DVLOGF(3) << "Waiting for last output buffer."; - return; - } - - // TODO(posciak): https://crbug.com/270039. Exynos requires a - // streamoff-streamon sequence after flush to continue, even if we are not - // resetting. This would make sense, because we don't really want to resume - // from a non-resume point (e.g. not from an IDR) if we are flushed. - // MSE player however triggers a Flush() on chunk end, but never Reset(). One - // could argue either way, or even say that Flush() is not needed/harmful when - // transitioning to next chunk. - // For now, do the streamoff-streamon cycle to satisfy Exynos and not freeze - // when doing MSE. This should be harmless otherwise. - if (!(StopDevicePoll() && StopOutputStream() && StopInputStream())) - return; - - if (!StartDevicePoll()) - return; - - decoder_delay_bitstream_buffer_id_ = -1; - decoder_flushing_ = false; - VLOGF(2) << "returning flush"; - child_task_runner_->PostTask(FROM_HERE, - base::Bind(&Client::NotifyFlushDone, client_)); - - // While we were flushing, we early-outed DecodeBufferTask()s. - ScheduleDecodeBufferTaskIfNeeded(); -} - -bool V4L2VideoDecodeAccelerator::IsDecoderCmdSupported() { - // CMD_STOP should always succeed. If the decoder is started, the command can - // flush it. If the decoder is stopped, the command does nothing. We use this - // to know if a driver supports V4L2_DEC_CMD_STOP to flush. - struct v4l2_decoder_cmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.cmd = V4L2_DEC_CMD_STOP; - if (device_->Ioctl(VIDIOC_TRY_DECODER_CMD, &cmd) != 0) { - VLOGF(2) << "V4L2_DEC_CMD_STOP is not supported."; - return false; - } - - return true; -} - -bool V4L2VideoDecodeAccelerator::SendDecoderCmdStop() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK(!flush_awaiting_last_output_buffer_); - - struct v4l2_decoder_cmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.cmd = V4L2_DEC_CMD_STOP; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_DECODER_CMD, &cmd); - flush_awaiting_last_output_buffer_ = true; - - return true; -} - -void V4L2VideoDecodeAccelerator::ResetTask() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - if (decoder_state_ == kError) { - VLOGF(2) << "early out: kError state"; - return; - } - decoder_current_bitstream_buffer_.reset(); - while (!decoder_input_queue_.empty()) - decoder_input_queue_.pop(); - - // If we are in the middle of switching resolutions or awaiting picture - // buffers, postpone reset until it's done. We don't have to worry about - // timing of this wrt to decoding, because output pipe is already - // stopped if we are changing resolution. We will come back here after - // we are done. - DCHECK(!reset_pending_); - if (decoder_state_ == kChangingResolution || - decoder_state_ == kAwaitingPictureBuffers) { - reset_pending_ = true; - return; - } - FinishReset(); -} - -void V4L2VideoDecodeAccelerator::FinishReset() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - reset_pending_ = false; - // After the output stream is stopped, the codec should not post any - // resolution change events. So we dequeue the resolution change event - // afterwards. The event could be posted before or while stopping the output - // stream. The codec will expect the buffer of new size after the seek, so - // we need to handle the resolution change event first. - if (!(StopDevicePoll() && StopOutputStream())) - return; - - if (DequeueResolutionChangeEvent()) { - reset_pending_ = true; - StartResolutionChange(); - return; - } - - if (!StopInputStream()) - return; - - // If we were flushing, we'll never return any more BitstreamBuffers or - // PictureBuffers; they have all been dropped and returned by now. - NotifyFlushDoneIfNeeded(); - - // Mark that we're resetting, then enqueue a ResetDoneTask(). All intervening - // jobs will early-out in the kResetting state. - decoder_state_ = kResetting; - SendPictureReady(); // Send all pending PictureReady. - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::ResetDoneTask, - base::Unretained(this))); -} - -void V4L2VideoDecodeAccelerator::ResetDoneTask() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - if (decoder_state_ == kError) { - VLOGF(2) << "early out: kError state"; - return; - } - - // Start poll thread if NotifyFlushDoneIfNeeded has not already. - if (!device_poll_thread_.IsRunning()) { - if (!StartDevicePoll()) - return; - } - - // Jobs drained, we're finished resetting. - DCHECK_EQ(decoder_state_, kResetting); - decoder_state_ = kInitialized; - - decoder_delay_bitstream_buffer_id_ = -1; - child_task_runner_->PostTask(FROM_HERE, - base::Bind(&Client::NotifyResetDone, client_)); - - // While we were resetting, we early-outed DecodeBufferTask()s. - ScheduleDecodeBufferTaskIfNeeded(); -} - -void V4L2VideoDecodeAccelerator::DestroyTask() { - VLOGF(2); - - // DestroyTask() should run regardless of decoder_state_. - - StopDevicePoll(); - StopOutputStream(); - StopInputStream(); - - decoder_current_bitstream_buffer_.reset(); - decoder_decode_buffer_tasks_scheduled_ = 0; - decoder_frames_at_client_ = 0; - while (!decoder_input_queue_.empty()) - decoder_input_queue_.pop(); - decoder_flushing_ = false; - - // Set our state to kError. Just in case. - decoder_state_ = kError; - - DestroyInputBuffers(); - DestroyOutputBuffers(); -} - -bool V4L2VideoDecodeAccelerator::StartDevicePoll() { - DVLOGF(3); - DCHECK(!device_poll_thread_.IsRunning()); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - // Start up the device poll thread and schedule its first DevicePollTask(). - if (!device_poll_thread_.Start()) { - VLOGF(1) << "Device thread failed to start"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - device_poll_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::DevicePollTask, - base::Unretained(this), 0)); - - return true; -} - -bool V4L2VideoDecodeAccelerator::StopDevicePoll() { - DVLOGF(3); - - if (!device_poll_thread_.IsRunning()) - return true; - - if (decoder_thread_.IsRunning()) - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - // Signal the DevicePollTask() to stop, and stop the device poll thread. - if (!device_->SetDevicePollInterrupt()) { - VPLOGF(1) << "SetDevicePollInterrupt(): failed"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - device_poll_thread_.Stop(); - // Clear the interrupt now, to be sure. - if (!device_->ClearDevicePollInterrupt()) { - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - DVLOGF(3) << "device poll stopped"; - return true; -} - -bool V4L2VideoDecodeAccelerator::StopOutputStream() { - VLOGF(2); - if (!output_streamon_) - return true; - - __u32 type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_STREAMOFF, &type); - output_streamon_ = false; - - // Output stream is stopped. No need to wait for the buffer anymore. - flush_awaiting_last_output_buffer_ = false; - - for (size_t i = 0; i < output_buffer_map_.size(); ++i) { - // After streamoff, the device drops ownership of all buffers, even if we - // don't dequeue them explicitly. Some of them may still be owned by the - // client however. Reuse only those that aren't. - OutputRecord& output_record = output_buffer_map_[i]; - if (output_record.state == kAtDevice) { - output_record.state = kFree; - free_output_buffers_.push_back(i); - } - } - output_buffer_queued_count_ = 0; - return true; -} - -bool V4L2VideoDecodeAccelerator::StopInputStream() { - VLOGF(2); - if (!input_streamon_) - return true; - - __u32 type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_STREAMOFF, &type); - input_streamon_ = false; - - // Reset accounting info for input. - while (!input_ready_queue_.empty()) - input_ready_queue_.pop(); - free_input_buffers_.clear(); - for (size_t i = 0; i < input_buffer_map_.size(); ++i) { - free_input_buffers_.push_back(i); - input_buffer_map_[i].at_device = false; - input_buffer_map_[i].bitstream_buffer.reset(); - } - input_buffer_queued_count_ = 0; - - return true; -} - -void V4L2VideoDecodeAccelerator::StartResolutionChange() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_NE(decoder_state_, kUninitialized); - DCHECK_NE(decoder_state_, kResetting); - - VLOGF(2) << "Initiate resolution change"; - - if (!(StopDevicePoll() && StopOutputStream())) - return; - - decoder_state_ = kChangingResolution; - SendPictureReady(); // Send all pending PictureReady. - - if (!DestroyOutputBuffers()) { - VLOGF(1) << "Failed destroying output buffers."; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - FinishResolutionChange(); -} - -void V4L2VideoDecodeAccelerator::FinishResolutionChange() { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kChangingResolution); - VLOGF(2); - - if (decoder_state_ == kError) { - VLOGF(2) << "early out: kError state"; - return; - } - - struct v4l2_format format; - bool again; - Size visible_size; - bool ret = GetFormatInfo(&format, &visible_size, &again); - if (!ret || again) { - VLOGF(1) << "Couldn't get format information after resolution change"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - if (!CreateBuffersForFormat(format, visible_size)) { - VLOGF(1) << "Couldn't reallocate buffers after resolution change"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - if (!StartDevicePoll()) - return; -} - -void V4L2VideoDecodeAccelerator::DevicePollTask(bool poll_device) { - DVLOGF(4); - DCHECK(device_poll_thread_.task_runner()->BelongsToCurrentThread()); - - bool event_pending = false; - - if (!device_->Poll(poll_device, &event_pending)) { - NOTIFY_ERROR(PLATFORM_FAILURE); - return; - } - - // All processing should happen on ServiceDeviceTask(), since we shouldn't - // touch decoder state from this thread. - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::ServiceDeviceTask, - base::Unretained(this), event_pending)); -} - -void V4L2VideoDecodeAccelerator::NotifyError(Error error) { - VLOGF(1); - - if (!child_task_runner_->BelongsToCurrentThread()) { - child_task_runner_->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::NotifyError, - weak_this_, error)); - return; - } - - if (client_) { - client_->NotifyError(error); - client_ptr_factory_.reset(); - } -} - -void V4L2VideoDecodeAccelerator::SetErrorState(Error error) { - // We can touch decoder_state_ only if this is the decoder thread or the - // decoder thread isn't running. - if (decoder_thread_.task_runner() && - !decoder_thread_.task_runner()->BelongsToCurrentThread()) { - decoder_thread_.task_runner()->PostTask( - FROM_HERE, base::Bind(&V4L2VideoDecodeAccelerator::SetErrorState, - base::Unretained(this), error)); - return; - } - - // Post NotifyError only if we are already initialized, as the API does - // not allow doing so before that. - if (decoder_state_ != kError && decoder_state_ != kUninitialized) - NotifyError(error); - - decoder_state_ = kError; -} - -bool V4L2VideoDecodeAccelerator::GetFormatInfo(struct v4l2_format* format, - Size* visible_size, - bool* again) { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - *again = false; - memset(format, 0, sizeof(*format)); - format->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (device_->Ioctl(VIDIOC_G_FMT, format) != 0) { - if (errno == EINVAL) { - // EINVAL means we haven't seen sufficient stream to decode the format. - *again = true; - return true; - } else { - VPLOGF(1) << "ioctl() failed: VIDIOC_G_FMT"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - } - - // Make sure we are still getting the format we set on initialization. - if (format->fmt.pix_mp.pixelformat != output_format_fourcc_) { - VLOGF(1) << "Unexpected format from G_FMT on output"; - return false; - } - - Size coded_size(format->fmt.pix_mp.width, format->fmt.pix_mp.height); - if (visible_size != nullptr) - *visible_size = GetVisibleSize(coded_size); - - return true; -} - -bool V4L2VideoDecodeAccelerator::CreateBuffersForFormat( - const struct v4l2_format& format, - const Size& visible_size) { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - output_planes_count_ = format.fmt.pix_mp.num_planes; - coded_size_.SetSize(format.fmt.pix_mp.width, format.fmt.pix_mp.height); - visible_size_ = visible_size; - - VLOGF(2) << "new resolution: " << coded_size_.ToString() - << ", visible size: " << visible_size_.ToString() - << ", decoder output planes count: " << output_planes_count_; - - return CreateOutputBuffers(); -} - -Size V4L2VideoDecodeAccelerator::GetVisibleSize( - const Size& coded_size) { - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - - struct v4l2_rect* visible_rect; - struct v4l2_selection selection_arg; - memset(&selection_arg, 0, sizeof(selection_arg)); - selection_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - selection_arg.target = V4L2_SEL_TGT_COMPOSE; - - if (device_->Ioctl(VIDIOC_G_SELECTION, &selection_arg) == 0) { - VLOGF(2) << "VIDIOC_G_SELECTION is supported"; - visible_rect = &selection_arg.r; - } else { - VLOGF(2) << "Fallback to VIDIOC_G_CROP"; - struct v4l2_crop crop_arg; - memset(&crop_arg, 0, sizeof(crop_arg)); - crop_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - - if (device_->Ioctl(VIDIOC_G_CROP, &crop_arg) != 0) { - VPLOGF(1) << "ioctl() VIDIOC_G_CROP failed"; - return coded_size; - } - visible_rect = &crop_arg.c; - } - - Rect rect(visible_rect->left, visible_rect->top, visible_rect->width, - visible_rect->height); - VLOGF(2) << "visible rectangle is " << rect.ToString(); - if (!Rect(coded_size).Contains(rect)) { - DVLOGF(3) << "visible rectangle " << rect.ToString() - << " is not inside coded size " << coded_size.ToString(); - return coded_size; - } - if (rect.IsEmpty()) { - VLOGF(1) << "visible size is empty"; - return coded_size; - } - - // Chrome assume picture frame is coded at (0, 0). - if (rect.x() != 0 || rect.y() != 0) { - VLOGF(1) << "Unexpected visible rectangle " << rect.ToString() - << ", top-left is not origin"; - return coded_size; - } - - return rect.size(); -} - -bool V4L2VideoDecodeAccelerator::CreateInputBuffers() { - VLOGF(2); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - // We always run this as we prepare to initialize. - DCHECK_EQ(decoder_state_, kInitialized); - DCHECK(!input_streamon_); - DCHECK(input_buffer_map_.empty()); - - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = kInputBufferCount; - reqbufs.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - reqbufs.memory = V4L2_MEMORY_DMABUF; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_REQBUFS, &reqbufs); - if (reqbufs.count < kInputBufferCount) { - VLOGF(1) << "Could not allocate enough output buffers"; - NOTIFY_ERROR(PLATFORM_FAILURE); - return false; - } - input_buffer_map_.resize(reqbufs.count); - free_input_buffers_.resize(reqbufs.count); - std::iota(free_input_buffers_.begin(), free_input_buffers_.end(), 0); - return true; -} - -static bool IsSupportedOutputFormat(uint32_t v4l2_format) { - // Only support V4L2_PIX_FMT_NV12 output format for now. - // TODO(johnylin): add more supported format if necessary. - uint32_t kSupportedOutputFmtFourcc[] = { V4L2_PIX_FMT_NV12 }; - return std::find( - kSupportedOutputFmtFourcc, - kSupportedOutputFmtFourcc + arraysize(kSupportedOutputFmtFourcc), - v4l2_format) != - kSupportedOutputFmtFourcc + arraysize(kSupportedOutputFmtFourcc); -} - -bool V4L2VideoDecodeAccelerator::SetupFormats() { - // We always run this as we prepare to initialize. - DCHECK(child_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(decoder_state_, kUninitialized); - DCHECK(!input_streamon_); - DCHECK(!output_streamon_); - - size_t input_size; - Size max_resolution, min_resolution; - device_->GetSupportedResolution(input_format_fourcc_, &min_resolution, - &max_resolution); - if (max_resolution.width() > 1920 && max_resolution.height() > 1088) - input_size = kInputBufferMaxSizeFor4k; - else - input_size = kInputBufferMaxSizeFor1080p; - - struct v4l2_fmtdesc fmtdesc; - memset(&fmtdesc, 0, sizeof(fmtdesc)); - fmtdesc.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - bool is_format_supported = false; - while (device_->Ioctl(VIDIOC_ENUM_FMT, &fmtdesc) == 0) { - if (fmtdesc.pixelformat == input_format_fourcc_) { - is_format_supported = true; - break; - } - ++fmtdesc.index; - } - - if (!is_format_supported) { - VLOGF(1) << "Input fourcc " << input_format_fourcc_ - << " not supported by device."; - return false; - } - - struct v4l2_format format; - memset(&format, 0, sizeof(format)); - format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - format.fmt.pix_mp.pixelformat = input_format_fourcc_; - format.fmt.pix_mp.plane_fmt[0].sizeimage = input_size; - format.fmt.pix_mp.num_planes = 1; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_S_FMT, &format); - // V4L2 driver adjusts input size that the driver may access. Store the size - // in order to specify it in QBUF later. - input_buffer_size_ = format.fmt.pix_mp.plane_fmt[0].sizeimage; - - - // We have to set up the format for output, because the driver may not allow - // changing it once we start streaming; whether it can support our chosen - // output format or not may depend on the input format. - memset(&fmtdesc, 0, sizeof(fmtdesc)); - fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - while (device_->Ioctl(VIDIOC_ENUM_FMT, &fmtdesc) == 0) { - if (IsSupportedOutputFormat(fmtdesc.pixelformat)) { - output_format_fourcc_ = fmtdesc.pixelformat; - break; - } - ++fmtdesc.index; - } - - if (output_format_fourcc_ == 0) { - VLOGF(2) << "Image processor not available"; - return false; - } - VLOGF(2) << "Output format=" << output_format_fourcc_; - - // Just set the fourcc for output; resolution, etc., will come from the - // driver once it extracts it from the stream. - memset(&format, 0, sizeof(format)); - format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - format.fmt.pix_mp.pixelformat = output_format_fourcc_; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_S_FMT, &format); - - return true; -} - -bool V4L2VideoDecodeAccelerator::CreateOutputBuffers() { - VLOGF(2); - DCHECK(decoder_state_ == kInitialized || - decoder_state_ == kChangingResolution); - DCHECK(!output_streamon_); - DCHECK(output_buffer_map_.empty()); - DCHECK_EQ(output_mode_, Config::OutputMode::IMPORT); - - // Number of output buffers we need. - struct v4l2_control ctrl; - memset(&ctrl, 0, sizeof(ctrl)); - ctrl.id = V4L2_CID_MIN_BUFFERS_FOR_CAPTURE; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_G_CTRL, &ctrl); - output_dpb_size_ = ctrl.value; - - // Output format setup in Initialize(). - - uint32_t buffer_count = output_dpb_size_ + kDpbOutputBufferExtraCount; - - VideoPixelFormat pixel_format = - V4L2PixFmtToVideoPixelFormat(output_format_fourcc_); - - child_task_runner_->PostTask( - FROM_HERE, base::Bind(&Client::ProvidePictureBuffers, client_, - buffer_count, pixel_format, coded_size_)); - - - // Go into kAwaitingPictureBuffers to prevent us from doing any more decoding - // or event handling while we are waiting for AssignPictureBuffers(). Not - // having Pictures available would not have prevented us from making decoding - // progress entirely e.g. in the case of H.264 where we could further decode - // non-slice NALUs and could even get another resolution change before we were - // done with this one. After we get the buffers, we'll go back into kIdle and - // kick off further event processing, and eventually go back into kDecoding - // once no more events are pending (if any). - decoder_state_ = kAwaitingPictureBuffers; - - return true; -} - -void V4L2VideoDecodeAccelerator::DestroyInputBuffers() { - VLOGF(2); - DCHECK(!decoder_thread_.IsRunning() || - decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK(!input_streamon_); - - if (input_buffer_map_.empty()) - return; - - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = 0; - reqbufs.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - reqbufs.memory = V4L2_MEMORY_DMABUF; - IOCTL_OR_LOG_ERROR(VIDIOC_REQBUFS, &reqbufs); - - input_buffer_map_.clear(); - free_input_buffers_.clear(); -} - -bool V4L2VideoDecodeAccelerator::DestroyOutputBuffers() { - VLOGF(2); - DCHECK(!decoder_thread_.IsRunning() || - decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK(!output_streamon_); - bool success = true; - - if (output_buffer_map_.empty()) - return true; - - for (size_t i = 0; i < output_buffer_map_.size(); ++i) { - OutputRecord& output_record = output_buffer_map_[i]; - - DVLOGF(3) << "dismissing PictureBuffer id=" << output_record.picture_id; - child_task_runner_->PostTask( - FROM_HERE, base::Bind(&Client::DismissPictureBuffer, client_, - output_record.picture_id)); - } - - struct v4l2_requestbuffers reqbufs; - memset(&reqbufs, 0, sizeof(reqbufs)); - reqbufs.count = 0; - reqbufs.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - reqbufs.memory = V4L2_MEMORY_DMABUF; - if (device_->Ioctl(VIDIOC_REQBUFS, &reqbufs) != 0) { - VPLOGF(1) << "ioctl() failed: VIDIOC_REQBUFS"; - NOTIFY_ERROR(PLATFORM_FAILURE); - success = false; - } - - output_buffer_map_.clear(); - while (!free_output_buffers_.empty()) - free_output_buffers_.pop_front(); - output_buffer_queued_count_ = 0; - // The client may still hold some buffers. The texture holds a reference to - // the buffer. It is OK to free the buffer and destroy EGLImage here. - decoder_frames_at_client_ = 0; - - return success; -} - -void V4L2VideoDecodeAccelerator::SendPictureReady() { - DVLOGF(4); - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - bool send_now = (decoder_state_ == kChangingResolution || - decoder_state_ == kResetting || decoder_flushing_); - while (pending_picture_ready_.size() > 0) { - bool cleared = pending_picture_ready_.front().cleared; - const Picture& picture = pending_picture_ready_.front().picture; - if (cleared && picture_clearing_count_ == 0) { - // This picture is cleared. It can be posted to a thread different than - // the main GPU thread to reduce latency. This should be the case after - // all pictures are cleared at the beginning. - decode_task_runner_->PostTask( - FROM_HERE, - base::Bind(&Client::PictureReady, decode_client_, picture)); - pending_picture_ready_.pop(); - } else if (!cleared || send_now) { - DVLOGF(4) << "cleared=" << pending_picture_ready_.front().cleared - << ", decoder_state_=" << decoder_state_ - << ", decoder_flushing_=" << decoder_flushing_ - << ", picture_clearing_count_=" << picture_clearing_count_; - // If the picture is not cleared, post it to the child thread because it - // has to be cleared in the child thread. A picture only needs to be - // cleared once. If the decoder is changing resolution, resetting or - // flushing, send all pictures to ensure PictureReady arrive before - // ProvidePictureBuffers, NotifyResetDone, or NotifyFlushDone. - child_task_runner_->PostTaskAndReply( - FROM_HERE, base::Bind(&Client::PictureReady, client_, picture), - // Unretained is safe. If Client::PictureReady gets to run, |this| is - // alive. Destroy() will wait the decode thread to finish. - base::Bind(&V4L2VideoDecodeAccelerator::PictureCleared, - base::Unretained(this))); - picture_clearing_count_++; - pending_picture_ready_.pop(); - } else { - // This picture is cleared. But some pictures are about to be cleared on - // the child thread. To preserve the order, do not send this until those - // pictures are cleared. - break; - } - } -} - -void V4L2VideoDecodeAccelerator::PictureCleared() { - DVLOGF(4) << "clearing count=" << picture_clearing_count_; - DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - DCHECK_GT(picture_clearing_count_, 0); - picture_clearing_count_--; - SendPictureReady(); -} - -} // namespace media diff --git a/accel/v4l2_video_decode_accelerator.h b/accel/v4l2_video_decode_accelerator.h deleted file mode 100644 index 99076ed..0000000 --- a/accel/v4l2_video_decode_accelerator.h +++ /dev/null @@ -1,497 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of VideoDecodeAccelerator -// that utilizes hardware video decoders, which expose Video4Linux 2 API -// (http://linuxtv.org/downloads/v4l-dvb-apis/). -// Note: ported from Chromium commit head: 85fdf90 -// Note: image processor is not ported. - -#ifndef MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ -#define MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <list> -#include <memory> -#include <queue> -#include <vector> - -#include "base/callback_forward.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/synchronization/waitable_event.h" -#include "base/threading/thread.h" -#include "picture.h" -#include "size.h" -#include "v4l2_device.h" -#include "video_decode_accelerator.h" - -namespace media { - -// This class handles video accelerators directly through a V4L2 device exported -// by the hardware blocks. -// -// The threading model of this class is driven by the fact that it needs to -// interface two fundamentally different event queues -- the one Chromium -// provides through MessageLoop, and the one driven by the V4L2 devices which -// is waited on with epoll(). There are three threads involved in this class: -// -// * The child thread, which is the main GPU process thread which calls the -// VideoDecodeAccelerator entry points. Calls from this thread -// generally do not block (with the exception of Initialize() and Destroy()). -// They post tasks to the decoder_thread_, which actually services the task -// and calls back when complete through the -// VideoDecodeAccelerator::Client interface. -// * The decoder_thread_, owned by this class. It services API tasks, through -// the *Task() routines, as well as V4L2 device events, through -// ServiceDeviceTask(). Almost all state modification is done on this thread -// (this doesn't include buffer (re)allocation sequence, see below). -// * The device_poll_thread_, owned by this class. All it does is epoll() on -// the V4L2 in DevicePollTask() and schedule a ServiceDeviceTask() on the -// decoder_thread_ when something interesting happens. -// TODO(sheu): replace this thread with an TYPE_IO decoder_thread_. -// -// Note that this class has (almost) no locks, apart from the pictures_assigned_ -// WaitableEvent. Everything (apart from buffer (re)allocation) is serviced on -// the decoder_thread_, so there are no synchronization issues. -// ... well, there are, but it's a matter of getting messages posted in the -// right order, not fiddling with locks. -// Buffer creation is a two-step process that is serviced partially on the -// Child thread, because we need to wait for the client to provide textures -// for the buffers we allocate. We cannot keep the decoder thread running while -// the client allocates Pictures for us, because we need to REQBUFS first to get -// the required number of output buffers from the device and that cannot be done -// unless we free the previous set of buffers, leaving the decoding in a -// inoperable state for the duration of the wait for Pictures. So to prevent -// subtle races (esp. if we get Reset() in the meantime), we block the decoder -// thread while we wait for AssignPictureBuffers from the client. -// -// V4L2VideoDecodeAccelerator may use image processor to convert the output. -// There are three cases: -// Flush: V4L2VDA should wait until image processor returns all processed -// frames. -// Reset: V4L2VDA doesn't need to wait for image processor. When image processor -// returns an old frame, drop it. -// Resolution change: V4L2VDA destroy image processor when destroying output -// buffrers. We cannot drop any frame during resolution change. So V4L2VDA -// should destroy output buffers after image processor returns all the frames. -class V4L2VideoDecodeAccelerator - : public VideoDecodeAccelerator { - public: - V4L2VideoDecodeAccelerator( - const scoped_refptr<V4L2Device>& device); - ~V4L2VideoDecodeAccelerator() override; - - // VideoDecodeAccelerator implementation. - // Note: Initialize() and Destroy() are synchronous. - bool Initialize(const Config& config, Client* client) override; - void Decode(const BitstreamBuffer& bitstream_buffer) override; - void AssignPictureBuffers(const std::vector<PictureBuffer>& buffers) override; - void ImportBufferForPicture( - int32_t picture_buffer_id, - VideoPixelFormat pixel_format, - const NativePixmapHandle& native_pixmap_handle) override; - void ReusePictureBuffer(int32_t picture_buffer_id) override; - void Flush() override; - void Reset() override; - void Destroy() override; - bool TryToSetupDecodeOnSeparateThread( - const base::WeakPtr<Client>& decode_client, - const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) - override; - - static VideoDecodeAccelerator::SupportedProfiles GetSupportedProfiles(); - - private: - // These are rather subjectively tuned. - enum { - kInputBufferCount = 8, - // TODO(posciak): determine input buffer size based on level limits. - // See http://crbug.com/255116. - // Input bitstream buffer size for up to 1080p streams. - kInputBufferMaxSizeFor1080p = 1024 * 1024, - // Input bitstream buffer size for up to 4k streams. - kInputBufferMaxSizeFor4k = 4 * kInputBufferMaxSizeFor1080p, - // This is originally from media/base/limits.h in Chromium. - kMaxVideoFrames = 4, - // Number of output buffers to use for each VDA stage above what's required - // by the decoder (e.g. DPB size, in H264). We need - // limits::kMaxVideoFrames to fill up the GpuVideoDecode pipeline, - // and +1 for a frame in transit. - kDpbOutputBufferExtraCount = kMaxVideoFrames + 1, - // Number of extra output buffers if image processor is used. - kDpbOutputBufferExtraCountForImageProcessor = 1, - }; - - // Internal state of the decoder. - enum State { - kUninitialized, // Initialize() not yet called. - kInitialized, // Initialize() returned true; ready to start decoding. - kDecoding, // DecodeBufferInitial() successful; decoding frames. - kResetting, // Presently resetting. - // Performing resolution change and waiting for image processor to return - // all frames. - kChangingResolution, - // Requested new PictureBuffers via ProvidePictureBuffers(), awaiting - // AssignPictureBuffers(). - kAwaitingPictureBuffers, - kError, // Error in kDecoding state. - }; - - enum OutputRecordState { - kFree, // Ready to be queued to the device. - kAtDevice, // Held by device. - kAtProcessor, // Held by image processor. - kAtClient, // Held by client of V4L2VideoDecodeAccelerator. - }; - - enum BufferId { - kFlushBufferId = -2 // Buffer id for flush buffer, queued by FlushTask(). - }; - - // Auto-destruction reference for BitstreamBuffer, for message-passing from - // Decode() to DecodeTask(). - struct BitstreamBufferRef; - - // Record for decoded pictures that can be sent to PictureReady. - struct PictureRecord { - PictureRecord(bool cleared, const Picture& picture); - ~PictureRecord(); - bool cleared; // Whether the texture is cleared and safe to render from. - Picture picture; // The decoded picture. - }; - - // Record for input buffers. - struct InputRecord { - bool at_device = false; // held by device. - std::unique_ptr<BitstreamBufferRef> bitstream_buffer; - }; - - // Record for output buffers. - struct OutputRecord { - OutputRecord(); - OutputRecord(OutputRecord&&) = default; - ~OutputRecord(); - OutputRecordState state; - int32_t picture_id; // picture buffer id as returned to PictureReady(). - bool cleared; // Whether the texture is cleared and safe to render - // from. See TextureManager for details. - // Output fds of the decoded frame. - std::vector<base::ScopedFD> output_fds; - // offsets of each decoded frame from each fd in |output_fds|. - std::vector<size_t> offsets; - }; - - // - // Decoding tasks, to be run on decode_thread_. - // - - // Task to finish initialization on decoder_thread_. - void InitializeTask(); - - // Enqueue a BitstreamBuffer to decode. This will enqueue a buffer to the - // decoder_input_queue_, then queue a DecodeBufferTask() to actually decode - // the buffer. - void DecodeTask(const BitstreamBuffer& bitstream_buffer); - - // Decode from the buffers queued in decoder_input_queue_. Calls - // DecodeBufferInitial() or DecodeBufferContinue() as appropriate. - void DecodeBufferTask(); - // Schedule another DecodeBufferTask() if we're behind. - void ScheduleDecodeBufferTaskIfNeeded(); - - // Return true if we should continue to schedule DecodeBufferTask()s after - // completion. - bool DecodeBufferInitial(); - bool DecodeBufferContinue(); - - // Flush data for one decoded frame. - bool TrySubmitInputFrame(); - - // Allocate V4L2 buffers and assign them to |buffers| provided by the client - // via AssignPictureBuffers() on decoder thread. - void AssignPictureBuffersTask(const std::vector<PictureBuffer>& buffers); - - // Use buffer backed by dmabuf file descriptors in |dmabuf_fds| for the - // OutputRecord associated with |picture_buffer_id|, taking ownership of the - // file descriptors. - void ImportBufferForPictureTask(int32_t picture_buffer_id, - std::vector<size_t> offsets, - std::vector<base::ScopedFD> dmabuf_fds); - - // Service I/O on the V4L2 devices. This task should only be scheduled from - // DevicePollTask(). If |event_pending| is true, one or more events - // on file descriptor are pending. - void ServiceDeviceTask(bool event_pending); - // Handle the various device queues. - void Enqueue(); - void Dequeue(); - // Dequeue one input buffer. Return true if success. - bool DequeueInputBuffer(); - // Dequeue one output buffer. Return true if success. - bool DequeueOutputBuffer(); - - // Return true if there is a resolution change event pending. - bool DequeueResolutionChangeEvent(); - - // Enqueue a buffer on the corresponding queue. - bool EnqueueInputRecord(); - bool EnqueueOutputRecord(); - - // Process a ReusePictureBuffer() API call. The API call create an EGLSync - // object on the main (GPU process) thread; we will record this object so we - // can wait on it before reusing the buffer. - void ReusePictureBufferTask(int32_t picture_buffer_id); - - // Flush() task. Child thread should not submit any more buffers until it - // receives the NotifyFlushDone callback. This task will schedule an empty - // BitstreamBufferRef (with input_id == kFlushBufferId) to perform the flush. - void FlushTask(); - // Notify the client of a flush completion, if required. This should be - // called any time a relevant queue could potentially be emptied: see - // function definition. - void NotifyFlushDoneIfNeeded(); - // Returns true if VIDIOC_DECODER_CMD is supported. - bool IsDecoderCmdSupported(); - // Send V4L2_DEC_CMD_START to the driver. Return true if success. - bool SendDecoderCmdStop(); - - // Reset() task. Drop all input buffers. If V4L2VDA is not doing resolution - // change or waiting picture buffers, call FinishReset. - void ResetTask(); - // This will schedule a ResetDoneTask() that will send the NotifyResetDone - // callback, then set the decoder state to kResetting so that all intervening - // tasks will drain. - void FinishReset(); - void ResetDoneTask(); - - // Device destruction task. - void DestroyTask(); - - // Start |device_poll_thread_|. - bool StartDevicePoll(); - - // Stop |device_poll_thread_|. - bool StopDevicePoll(); - - bool StopInputStream(); - bool StopOutputStream(); - - void StartResolutionChange(); - void FinishResolutionChange(); - - // Try to get output format and visible size, detected after parsing the - // beginning of the stream. Sets |again| to true if more parsing is needed. - // |visible_size| could be nullptr and ignored. - bool GetFormatInfo(struct v4l2_format* format, - Size* visible_size, - bool* again); - // Create output buffers for the given |format| and |visible_size|. - bool CreateBuffersForFormat(const struct v4l2_format& format, - const Size& visible_size); - - // Try to get |visible_size|. Return visible size, or, if querying it is not - // supported or produces invalid size, return |coded_size| instead. - Size GetVisibleSize(const Size& coded_size); - - // - // Device tasks, to be run on device_poll_thread_. - // - - // The device task. - void DevicePollTask(bool poll_device); - - // - // Safe from any thread. - // - - // Error notification (using PostTask() to child thread, if necessary). - void NotifyError(Error error); - - // Set the decoder_state_ to kError and notify the client (if necessary). - void SetErrorState(Error error); - - // - // Other utility functions. Called on decoder_thread_, unless - // decoder_thread_ is not yet started, in which case the child thread can call - // these (e.g. in Initialize() or Destroy()). - // - - // Create the buffers we need. - bool CreateInputBuffers(); - bool CreateOutputBuffers(); - - // Destroy buffers. - void DestroyInputBuffers(); - // In contrast to DestroyInputBuffers, which is called only on destruction, - // we call DestroyOutputBuffers also during playback, on resolution change. - // Even if anything fails along the way, we still want to go on and clean - // up as much as possible, so return false if this happens, so that the - // caller can error out on resolution change. - bool DestroyOutputBuffers(); - - // Set input and output formats before starting decode. - bool SetupFormats(); - - // - // Methods run on child thread. - // - - // Send decoded pictures to PictureReady. - void SendPictureReady(); - - // Callback that indicates a picture has been cleared. - void PictureCleared(); - - // Our original calling task runner for the child thread. - scoped_refptr<base::SingleThreadTaskRunner> child_task_runner_; - - // Task runner Decode() and PictureReady() run on. - scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; - - // WeakPtr<> pointing to |this| for use in posting tasks from the decoder or - // device worker threads back to the child thread. Because the worker threads - // are members of this class, any task running on those threads is guaranteed - // that this object is still alive. As a result, tasks posted from the child - // thread to the decoder or device thread should use base::Unretained(this), - // and tasks posted the other way should use |weak_this_|. - base::WeakPtr<V4L2VideoDecodeAccelerator> weak_this_; - - // To expose client callbacks from VideoDecodeAccelerator. - // NOTE: all calls to these objects *MUST* be executed on - // child_task_runner_. - std::unique_ptr<base::WeakPtrFactory<Client>> client_ptr_factory_; - base::WeakPtr<Client> client_; - // Callbacks to |decode_client_| must be executed on |decode_task_runner_|. - base::WeakPtr<Client> decode_client_; - - // - // Decoder state, owned and operated by decoder_thread_. - // Before decoder_thread_ has started, the decoder state is managed by - // the child (main) thread. After decoder_thread_ has started, the decoder - // thread should be the only one managing these. - // - - // This thread services tasks posted from the VDA API entry points by the - // child thread and device service callbacks posted from the device thread. - base::Thread decoder_thread_; - // Decoder state machine state. - State decoder_state_; - - Config::OutputMode output_mode_; - - // BitstreamBuffer we're presently reading. - std::unique_ptr<BitstreamBufferRef> decoder_current_bitstream_buffer_; - // The V4L2Device this class is operating upon. - scoped_refptr<V4L2Device> device_; - // FlushTask() and ResetTask() should not affect buffers that have been - // queued afterwards. For flushing or resetting the pipeline then, we will - // delay these buffers until after the flush or reset completes. - int decoder_delay_bitstream_buffer_id_; - // We track the number of buffer decode tasks we have scheduled, since each - // task execution should complete one buffer. If we fall behind (due to - // resource backpressure, etc.), we'll have to schedule more to catch up. - int decoder_decode_buffer_tasks_scheduled_; - // Picture buffers held by the client. - int decoder_frames_at_client_; - - // Are we flushing? - bool decoder_flushing_; - // True if VIDIOC_DECODER_CMD is supported. - bool decoder_cmd_supported_; - // True if flushing is waiting for last output buffer. After - // VIDIOC_DECODER_CMD is sent to the driver, this flag will be set to true to - // wait for the last output buffer. When this flag is true, flush done will - // not be sent. After an output buffer that has the flag V4L2_BUF_FLAG_LAST is - // received, this is set to false. - bool flush_awaiting_last_output_buffer_; - - // Got a reset request while we were performing resolution change or waiting - // picture buffers. - bool reset_pending_; - // Input queue for decoder_thread_: BitstreamBuffers in. - std::queue<std::unique_ptr<BitstreamBufferRef>> decoder_input_queue_; - - // - // Hardware state and associated queues. Since decoder_thread_ services - // the hardware, decoder_thread_ owns these too. - // output_buffer_map_, free_output_buffers_ and output_planes_count_ are an - // exception during the buffer (re)allocation sequence, when the - // decoder_thread_ is blocked briefly while the Child thread manipulates - // them. - // - - // Completed decode buffers. - std::queue<int> input_ready_queue_; - - // Input buffer state. - bool input_streamon_; - // Input buffers enqueued to device. - int input_buffer_queued_count_; - // Input buffers ready to use, as a LIFO since we don't care about ordering. - std::vector<int> free_input_buffers_; - // Mapping of int index to input buffer record. - std::vector<InputRecord> input_buffer_map_; - // The size of input buffer that bitstream buffer can be copied. - size_t input_buffer_size_; - - // Output buffer state. - bool output_streamon_; - // Output buffers enqueued to device. - int output_buffer_queued_count_; - // Output buffers ready to use, as a FIFO since we want oldest-first to hide - // synchronization latency with GL. - std::list<int> free_output_buffers_; - // Mapping of int index to output buffer record. - std::vector<OutputRecord> output_buffer_map_; - // Required size of DPB for decoding. - int output_dpb_size_; - - // Number of planes (i.e. separate memory buffers) for output. - size_t output_planes_count_; - - // Pictures that are ready but not sent to PictureReady yet. - std::queue<PictureRecord> pending_picture_ready_; - - // The number of pictures that are sent to PictureReady and will be cleared. - int picture_clearing_count_; - - // Output picture coded size. - Size coded_size_; - - // Output picture visible size. - Size visible_size_; - - // - // The device polling thread handles notifications of V4L2 device changes. - // - - // The thread. - base::Thread device_poll_thread_; - - // - // Other state, held by the child (main) thread. - // - - // The codec we'll be decoding for. - VideoCodecProfile video_profile_; - // Chosen input format for video_profile_. - uint32_t input_format_fourcc_; - // Chosen output format. - uint32_t output_format_fourcc_; - - // Input format V4L2 fourccs this class supports. - static const uint32_t supported_input_fourccs_[]; - - // The WeakPtrFactory for |weak_this_|. - base::WeakPtrFactory<V4L2VideoDecodeAccelerator> weak_this_factory_; - - DISALLOW_COPY_AND_ASSIGN(V4L2VideoDecodeAccelerator); -}; - -} // namespace media - -#endif // MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ diff --git a/accel/video_codecs.cc b/accel/video_codecs.cc deleted file mode 100644 index 8e4d4a7..0000000 --- a/accel/video_codecs.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 23236dc929bc -// Note: only necessary functions are ported. - -#include "video_codecs.h" - -#include "base/logging.h" - -namespace media { - -std::string GetProfileName(VideoCodecProfile profile) { - switch (profile) { - case VIDEO_CODEC_PROFILE_UNKNOWN: - return "unknown"; - case H264PROFILE_BASELINE: - return "h264 baseline"; - case H264PROFILE_MAIN: - return "h264 main"; - case H264PROFILE_EXTENDED: - return "h264 extended"; - case H264PROFILE_HIGH: - return "h264 high"; - case H264PROFILE_HIGH10PROFILE: - return "h264 high 10"; - case H264PROFILE_HIGH422PROFILE: - return "h264 high 4:2:2"; - case H264PROFILE_HIGH444PREDICTIVEPROFILE: - return "h264 high 4:4:4 predictive"; - case H264PROFILE_SCALABLEBASELINE: - return "h264 scalable baseline"; - case H264PROFILE_SCALABLEHIGH: - return "h264 scalable high"; - case H264PROFILE_STEREOHIGH: - return "h264 stereo high"; - case H264PROFILE_MULTIVIEWHIGH: - return "h264 multiview high"; - case HEVCPROFILE_MAIN: - return "hevc main"; - case HEVCPROFILE_MAIN10: - return "hevc main 10"; - case HEVCPROFILE_MAIN_STILL_PICTURE: - return "hevc main still-picture"; - case VP8PROFILE_ANY: - return "vp8"; - case VP9PROFILE_PROFILE0: - return "vp9 profile0"; - case VP9PROFILE_PROFILE1: - return "vp9 profile1"; - case VP9PROFILE_PROFILE2: - return "vp9 profile2"; - case VP9PROFILE_PROFILE3: - return "vp9 profile3"; - case DOLBYVISION_PROFILE0: - return "dolby vision profile 0"; - case DOLBYVISION_PROFILE4: - return "dolby vision profile 4"; - case DOLBYVISION_PROFILE5: - return "dolby vision profile 5"; - case DOLBYVISION_PROFILE7: - return "dolby vision profile 7"; - case DOLBYVISION_PROFILE8: - return "dolby vision profile 8"; - case DOLBYVISION_PROFILE9: - return "dolby vision profile 9"; - case THEORAPROFILE_ANY: - return "theora"; - case AV1PROFILE_PROFILE_MAIN: - return "av1 profile main"; - case AV1PROFILE_PROFILE_HIGH: - return "av1 profile high"; - case AV1PROFILE_PROFILE_PRO: - return "av1 profile pro"; - } - NOTREACHED(); - return ""; -} - -} // namespace media diff --git a/accel/video_codecs.h b/accel/video_codecs.h deleted file mode 100644 index 44d631b..0000000 --- a/accel/video_codecs.h +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: becc5bbb0aa6 -// Note: only necessary functions are ported. - -#ifndef VIDEO_CODECS_H_ -#define VIDEO_CODECS_H_ - -#include <string> - -namespace media { - -// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.media -enum VideoCodec { - // These values are histogrammed over time; do not change their ordinal - // values. When deleting a codec replace it with a dummy value; when adding a - // codec, do so at the bottom (and update kVideoCodecMax). - kUnknownVideoCodec = 0, - kCodecH264, - kCodecVC1, - kCodecMPEG2, - kCodecMPEG4, - kCodecTheora, - kCodecVP8, - kCodecVP9, - kCodecHEVC, - kCodecDolbyVision, - kCodecAV1, - // DO NOT ADD RANDOM VIDEO CODECS! - // - // The only acceptable time to add a new codec is if there is production code - // that uses said codec in the same CL. - - kVideoCodecMax = kCodecAV1, // Must equal the last "real" codec above. -}; - -// Video codec profiles. Keep in sync with mojo::VideoCodecProfile (see -// media/mojo/mojom/media_types.mojom), gpu::VideoCodecProfile (see -// gpu/config/gpu_info.h), and PP_VideoDecoder_Profile (translation is performed -// in content/renderer/pepper/ppb_video_decoder_impl.cc). -// NOTE: These values are histogrammed over time in UMA so the values must never -// ever change (add new values to tools/metrics/histograms/histograms.xml) -// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.media -enum VideoCodecProfile { - // Keep the values in this enum unique, as they imply format (h.264 vs. VP8, - // for example), and keep the values for a particular format grouped - // together for clarity. - VIDEO_CODEC_PROFILE_UNKNOWN = -1, - VIDEO_CODEC_PROFILE_MIN = VIDEO_CODEC_PROFILE_UNKNOWN, - H264PROFILE_MIN = 0, - H264PROFILE_BASELINE = H264PROFILE_MIN, - H264PROFILE_MAIN = 1, - H264PROFILE_EXTENDED = 2, - H264PROFILE_HIGH = 3, - H264PROFILE_HIGH10PROFILE = 4, - H264PROFILE_HIGH422PROFILE = 5, - H264PROFILE_HIGH444PREDICTIVEPROFILE = 6, - H264PROFILE_SCALABLEBASELINE = 7, - H264PROFILE_SCALABLEHIGH = 8, - H264PROFILE_STEREOHIGH = 9, - H264PROFILE_MULTIVIEWHIGH = 10, - H264PROFILE_MAX = H264PROFILE_MULTIVIEWHIGH, - VP8PROFILE_MIN = 11, - VP8PROFILE_ANY = VP8PROFILE_MIN, - VP8PROFILE_MAX = VP8PROFILE_ANY, - VP9PROFILE_MIN = 12, - VP9PROFILE_PROFILE0 = VP9PROFILE_MIN, - VP9PROFILE_PROFILE1 = 13, - VP9PROFILE_PROFILE2 = 14, - VP9PROFILE_PROFILE3 = 15, - VP9PROFILE_MAX = VP9PROFILE_PROFILE3, - HEVCPROFILE_MIN = 16, - HEVCPROFILE_MAIN = HEVCPROFILE_MIN, - HEVCPROFILE_MAIN10 = 17, - HEVCPROFILE_MAIN_STILL_PICTURE = 18, - HEVCPROFILE_MAX = HEVCPROFILE_MAIN_STILL_PICTURE, - DOLBYVISION_PROFILE0 = 19, - DOLBYVISION_PROFILE4 = 20, - DOLBYVISION_PROFILE5 = 21, - DOLBYVISION_PROFILE7 = 22, - THEORAPROFILE_MIN = 23, - THEORAPROFILE_ANY = THEORAPROFILE_MIN, - THEORAPROFILE_MAX = THEORAPROFILE_ANY, - AV1PROFILE_MIN = 24, - AV1PROFILE_PROFILE_MAIN = AV1PROFILE_MIN, - AV1PROFILE_PROFILE_HIGH = 25, - AV1PROFILE_PROFILE_PRO = 26, - AV1PROFILE_MAX = AV1PROFILE_PROFILE_PRO, - DOLBYVISION_PROFILE8 = 27, - DOLBYVISION_PROFILE9 = 28, - VIDEO_CODEC_PROFILE_MAX = DOLBYVISION_PROFILE9, -}; - -std::string GetProfileName(VideoCodecProfile profile); - -} // namespace media - -#endif // VIDEO_CODECS_H_ diff --git a/accel/video_decode_accelerator.cc b/accel/video_decode_accelerator.cc deleted file mode 100644 index e74d1ec..0000000 --- a/accel/video_decode_accelerator.cc +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 85fdf90 - -#include "base/logging.h" - -#include "video_decode_accelerator.h" - -namespace media { - -VideoDecodeAccelerator::Config::Config() = default; -VideoDecodeAccelerator::Config::Config(const Config& config) = default; - -VideoDecodeAccelerator::Config::Config(VideoCodecProfile video_codec_profile) - : profile(video_codec_profile) {} - -VideoDecodeAccelerator::Config::~Config() = default; - -std::string VideoDecodeAccelerator::Config::AsHumanReadableString() const { - std::ostringstream s; - s << "profile: " << GetProfileName(profile); - return s.str(); -} - -void VideoDecodeAccelerator::Client::NotifyInitializationComplete( - bool success) { - NOTREACHED() << "By default deferred initialization is not supported."; -} - -VideoDecodeAccelerator::~VideoDecodeAccelerator() = default; - -bool VideoDecodeAccelerator::TryToSetupDecodeOnSeparateThread( - const base::WeakPtr<Client>& decode_client, - const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) { - // Implementations in the process that VDA runs in must override this. - LOG(FATAL) << "This may only be called in the same process as VDA impl."; - return false; -} - -void VideoDecodeAccelerator::ImportBufferForPicture( - int32_t picture_buffer_id, - VideoPixelFormat pixel_format, - const NativePixmapHandle& native_pixmap_handle) { - NOTREACHED() << "Buffer import not supported."; -} - -VideoDecodeAccelerator::SupportedProfile::SupportedProfile() - : profile(VIDEO_CODEC_PROFILE_UNKNOWN), encrypted_only(false) {} - -VideoDecodeAccelerator::SupportedProfile::~SupportedProfile() = default; - -VideoDecodeAccelerator::Capabilities::Capabilities() : flags(NO_FLAGS) {} - -VideoDecodeAccelerator::Capabilities::Capabilities(const Capabilities& other) = - default; - -VideoDecodeAccelerator::Capabilities::~Capabilities() = default; - -std::string VideoDecodeAccelerator::Capabilities::AsHumanReadableString() - const { - std::ostringstream s; - s << "["; - for (const SupportedProfile& sp : supported_profiles) { - s << " " << GetProfileName(sp.profile) << ": " << sp.min_resolution.width() - << "x" << sp.min_resolution.height() << "->" << sp.max_resolution.width() - << "x" << sp.max_resolution.height(); - } - s << "]"; - return s.str(); -} - -} // namespace media - -namespace std { - -void default_delete<media::VideoDecodeAccelerator>::operator()( - media::VideoDecodeAccelerator* vda) const { - vda->Destroy(); -} - -} // namespace std diff --git a/accel/video_decode_accelerator.h b/accel/video_decode_accelerator.h deleted file mode 100644 index 10601be..0000000 --- a/accel/video_decode_accelerator.h +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 85fdf90 - -#ifndef VIDEO_DECODE_ACCELERATOR_H_ -#define VIDEO_DECODE_ACCELERATOR_H_ - -#include <vector> - -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" - -#include "bitstream_buffer.h" -#include "native_pixmap_handle.h" -#include "picture.h" -#include "size.h" -#include "video_codecs.h" -#include "video_pixel_format.h" - -namespace base { -class SingleThreadTaskRunner; -} - -namespace media { - -// Video decoder interface. -// This interface is extended by the various components that ultimately -// implement the backend of PPB_VideoDecoder_Dev. -class VideoDecodeAccelerator { - public: - // Specification of a decoding profile supported by an decoder. - // |max_resolution| and |min_resolution| are inclusive. - struct SupportedProfile { - SupportedProfile(); - ~SupportedProfile(); - VideoCodecProfile profile; - Size max_resolution; - Size min_resolution; - bool encrypted_only; - }; - using SupportedProfiles = std::vector<SupportedProfile>; - - struct Capabilities { - Capabilities(); - Capabilities(const Capabilities& other); - ~Capabilities(); - - std::string AsHumanReadableString() const; - - // Flags that can be associated with a VDA. - enum Flags { - NO_FLAGS = 0, - - // Normally, the VDA is required to be able to provide all PictureBuffers - // to the client via PictureReady(), even if the client does not return - // any of them via ReusePictureBuffer(). The client is only required to - // return PictureBuffers when it holds all of them, if it wants to get - // more decoded output. See VideoDecoder::CanReadWithoutStalling for - // more context. - // If this flag is set, then the VDA does not make this guarantee. The - // client must return PictureBuffers to be sure that new frames will be - // provided via PictureReady. - NEEDS_ALL_PICTURE_BUFFERS_TO_DECODE = 1 << 0, - - // Whether the VDA supports being configured with an output surface for - // it to render frames to. For example, SurfaceViews on Android. - SUPPORTS_EXTERNAL_OUTPUT_SURFACE = 1 << 1, - - // If set, the VDA will use deferred initialization if the config - // indicates that the client supports it as well. Refer to - // NotifyInitializationComplete for more details. - SUPPORTS_DEFERRED_INITIALIZATION = 1 << 2, - - // If set, video frames will have COPY_REQUIRED flag which will cause - // an extra texture copy during composition. - REQUIRES_TEXTURE_COPY = 1 << 3, - - // Whether the VDA supports encrypted streams or not. - SUPPORTS_ENCRYPTED_STREAMS = 1 << 4, - - // If set the decoder does not require a restart in order to switch to - // using an external output surface. - SUPPORTS_SET_EXTERNAL_OUTPUT_SURFACE = 1 << 5, - }; - - SupportedProfiles supported_profiles; - uint32_t flags; - }; - - // Enumeration of potential errors generated by the API. - // Note: Keep these in sync with PP_VideoDecodeError_Dev. Also do not - // rearrange, reuse or remove values as they are used for gathering UMA - // statistics. - enum Error { - // An operation was attempted during an incompatible decoder state. - ILLEGAL_STATE = 1, - // Invalid argument was passed to an API method. - INVALID_ARGUMENT, - // Encoded input is unreadable. - UNREADABLE_INPUT, - // A failure occurred at the browser layer or one of its dependencies. - // Examples of such failures include GPU hardware failures, GPU driver - // failures, GPU library failures, browser programming errors, and so on. - PLATFORM_FAILURE, - // Largest used enum. This should be adjusted when new errors are added. - ERROR_MAX = PLATFORM_FAILURE, - }; - - // Config structure contains parameters required for the VDA initialization. - struct Config { - // Specifies the allocation and handling mode for output PictureBuffers. - // When set to ALLOCATE, the VDA is expected to allocate backing memory - // for PictureBuffers at the time of AssignPictureBuffers() call. - // When set to IMPORT, the VDA will not allocate, but after receiving - // AssignPictureBuffers() call, it will expect a call to - // ImportBufferForPicture() for each PictureBuffer before use. - enum class OutputMode { - ALLOCATE, - IMPORT, - }; - - Config(); - Config(const Config& config); - - explicit Config(VideoCodecProfile profile); - - ~Config(); - - std::string AsHumanReadableString() const; - - // The video codec and profile. - VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN; - - // Whether the client supports deferred initialization. - bool is_deferred_initialization_allowed = false; - - // Coded size of the video frame hint, subject to change. - Size initial_expected_coded_size = Size(320, 240); - - OutputMode output_mode = OutputMode::ALLOCATE; - - // The list of picture buffer formats that the client knows how to use. An - // empty list means any format is supported. - std::vector<VideoPixelFormat> supported_output_formats; - - // The H264 SPS and PPS configuration data. Not all clients populate these - // fields, so they should be parsed from the bitstream instead, if required. - // Each SPS and PPS is prefixed with the Annex B framing bytes: 0, 0, 0, 1. - std::vector<uint8_t> sps; - std::vector<uint8_t> pps; - }; - - // Interface for collaborating with picture interface to provide memory for - // output picture and blitting them. These callbacks will not be made unless - // Initialize() has returned successfully. - // This interface is extended by the various layers that relay messages back - // to the plugin, through the PPP_VideoDecoder_Dev interface the plugin - // implements. - class Client { - public: - // Notify the client that deferred initialization has completed successfully - // or not. This is required if and only if deferred initialization is - // supported by the VDA (see Capabilities), and it is supported by the - // client (see Config::is_deferred_initialization_allowed), and the initial - // call to VDA::Initialize returns true. - // The default implementation is a NOTREACHED, since deferred initialization - // is not supported by default. - virtual void NotifyInitializationComplete(bool success); - - // Callback to tell client how many and what size of buffers to provide. - // Note that the actual count provided through AssignPictureBuffers() can be - // larger than the value requested. - // |format| indicates what format the decoded frames will be produced in - // by the VDA, or PIXEL_FORMAT_UNKNOWN if the underlying platform handles - // this transparently. - virtual void ProvidePictureBuffers(uint32_t requested_num_of_buffers, - VideoPixelFormat format, - const Size& dimensions) = 0; - - // Callback to dismiss picture buffer that was assigned earlier. - virtual void DismissPictureBuffer(int32_t picture_buffer_id) = 0; - - // Callback to deliver decoded pictures ready to be displayed. - virtual void PictureReady(const Picture& picture) = 0; - - // Callback to notify that decoded has decoded the end of the current - // bitstream buffer. - virtual void NotifyEndOfBitstreamBuffer(int32_t bitstream_buffer_id) = 0; - - // Flush completion callback. - virtual void NotifyFlushDone() = 0; - - // Reset completion callback. - virtual void NotifyResetDone() = 0; - - // Callback to notify about decoding errors. Note that errors in - // Initialize() will not be reported here, but will instead be indicated by - // a false return value there. - virtual void NotifyError(Error error) = 0; - - protected: - virtual ~Client() {} - }; - - // Video decoder functions. - - // Initializes the video decoder with specific configuration. Called once per - // decoder construction. This call is synchronous and returns true iff - // initialization is successful, unless deferred initialization is used. - // - // By default, deferred initialization is not used. However, if Config:: - // is_deferred_initialization_allowed is set by the client, and if - // Capabilities::Flags::SUPPORTS_DEFERRED_INITIALIZATION is set by the VDA, - // and if VDA::Initialize returns true, then the client can expect a call to - // NotifyInitializationComplete with the actual success / failure of - // initialization. Note that a return value of false from VDA::Initialize - // indicates that initialization definitely failed, and no callback is needed. - // - // For encrypted video, only deferred initialization is supported and |config| - // must contain a valid |cdm_id|. - // - // Parameters: - // |config| contains the initialization parameters. - // |client| is the client of this video decoder. Does not take ownership of - // |client| which must be valid until Destroy() is called. - virtual bool Initialize(const Config& config, Client* client) = 0; - - // Decodes given bitstream buffer that contains at most one frame. Once - // decoder is done with processing |bitstream_buffer| it will call - // NotifyEndOfBitstreamBuffer() with the bitstream buffer id. - // Parameters: - // |bitstream_buffer| is the input bitstream that is sent for decoding. - virtual void Decode(const BitstreamBuffer& bitstream_buffer) = 0; - - // Assigns a set of texture-backed picture buffers to the video decoder. - // - // Ownership of each picture buffer remains with the client, but the client - // is not allowed to deallocate the buffer before the DismissPictureBuffer - // callback has been initiated for a given buffer. - // - // Parameters: - // |buffers| contains the allocated picture buffers for the output. Note - // that the count of buffers may be larger than the count requested through - // the call to Client::ProvidePictureBuffers(). - virtual void AssignPictureBuffers( - const std::vector<PictureBuffer>& buffers) = 0; - - // Imports |gpu_memory_buffer_handle|, pointing to a buffer in |pixel_format|, - // as backing memory for picture buffer associated with |picture_buffer_id|. - // This can only be be used if the VDA has been Initialize()d with - // config.output_mode = IMPORT, and should be preceded by a call to - // AssignPictureBuffers() to set up the number of PictureBuffers and their - // details. - // The |pixel_format| used here may be different from the |pixel_format| - // required in ProvidePictureBuffers(). If the buffer cannot be imported an - // error should be notified via NotifyError(). - // After this call, the VDA becomes the owner of those file descriptors, - // and is responsible for closing it after use, also on import failure. - virtual void ImportBufferForPicture( - int32_t picture_buffer_id, - VideoPixelFormat pixel_format, - const NativePixmapHandle& native_pixmap_handle); - - // Sends picture buffers to be reused by the decoder. This needs to be called - // for each buffer that has been processed so that decoder may know onto which - // picture buffers it can write the output to. - // - // Parameters: - // |picture_buffer_id| id of the picture buffer that is to be reused. - virtual void ReusePictureBuffer(int32_t picture_buffer_id) = 0; - - // Flushes the decoder: all pending inputs will be decoded and pictures handed - // back to the client, followed by NotifyFlushDone() being called on the - // client. Can be used to implement "end of stream" notification. - virtual void Flush() = 0; - - // Resets the decoder: all pending inputs are dropped immediately and the - // decoder returned to a state ready for further Decode()s, followed by - // NotifyResetDone() being called on the client. Can be used to implement - // "seek". After Flush is called, it is OK to call Reset before receiving - // NotifyFlushDone() and VDA should cancel the flush. Note NotifyFlushDone() - // may be on the way to the client. If client gets NotifyFlushDone(), it - // should be before NotifyResetDone(). - virtual void Reset() = 0; - - // Destroys the decoder: all pending inputs are dropped immediately and the - // component is freed. This call may asynchornously free system resources, - // but its client-visible effects are synchronous. After this method returns - // no more callbacks will be made on the client. Deletes |this| - // unconditionally, so make sure to drop all pointers to it! - virtual void Destroy() = 0; - - // TO BE CALLED IN THE SAME PROCESS AS THE VDA IMPLEMENTATION ONLY. - // - // A decode "task" is a sequence that includes a Decode() call from Client, - // as well as corresponding callbacks to return the input BitstreamBuffer - // after use, and the resulting output Picture(s). - // - // If the Client can support running these three calls on a separate thread, - // it may call this method to try to set up the VDA implementation to do so. - // If the VDA can support this as well, return true, otherwise return false. - // If true is returned, the client may submit each Decode() call (but no other - // calls) on |decode_task_runner|, and should then expect that - // NotifyEndOfBitstreamBuffer() and PictureReady() callbacks may come on - // |decode_task_runner| as well, called on |decode_client|, instead of client - // provided to Initialize(). - // - // This method may be called at any time. - // - // NOTE 1: some callbacks may still have to come on the main thread and the - // Client should handle both callbacks coming on main and |decode_task_runner| - // thread. - // - // NOTE 2: VDA implementations of Decode() must return as soon as possible and - // never block, as |decode_task_runner| may be a latency critical thread - // (such as the GPU IO thread). - // - // One application of this is offloading the GPU Child thread. In general, - // calls to VDA in GPU process have to be done on the GPU Child thread, as - // they may require GL context to be current. However, some VDAs may be able - // to run decode operations without GL context, which helps reduce latency and - // offloads the GPU Child thread. - virtual bool TryToSetupDecodeOnSeparateThread( - const base::WeakPtr<Client>& decode_client, - const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner); - - protected: - // Do not delete directly; use Destroy() or own it with a scoped_ptr, which - // will Destroy() it properly by default. - virtual ~VideoDecodeAccelerator(); -}; - -} // namespace media - -namespace std { - -// Specialize std::default_delete so that -// std::unique_ptr<VideoDecodeAccelerator> uses "Destroy()" instead of trying to -// use the destructor. -template <> -struct default_delete<media::VideoDecodeAccelerator> { - void operator()(media::VideoDecodeAccelerator* vda) const; -}; - -} // namespace std - -#endif // VIDEO_DECODE_ACCELERATOR_H_ diff --git a/accel/video_encode_accelerator.cc b/accel/video_encode_accelerator.cc deleted file mode 100644 index f35f4b2..0000000 --- a/accel/video_encode_accelerator.cc +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 9e40822e3a3d -// Note: only necessary functions are ported. - -#include "video_encode_accelerator.h" - -namespace media { - -Vp8Metadata::Vp8Metadata() - : non_reference(false), temporal_idx(0), layer_sync(false) {} -Vp8Metadata::Vp8Metadata(const Vp8Metadata& other) = default; -Vp8Metadata::Vp8Metadata(Vp8Metadata&& other) = default; -Vp8Metadata::~Vp8Metadata() = default; - -BitstreamBufferMetadata::BitstreamBufferMetadata() - : payload_size_bytes(0), key_frame(false) {} -BitstreamBufferMetadata::BitstreamBufferMetadata( - BitstreamBufferMetadata&& other) = default; -BitstreamBufferMetadata::BitstreamBufferMetadata(size_t payload_size_bytes, - bool key_frame, - base::TimeDelta timestamp) - : payload_size_bytes(payload_size_bytes), - key_frame(key_frame), - timestamp(timestamp) {} -BitstreamBufferMetadata::~BitstreamBufferMetadata() = default; - -VideoEncodeAccelerator::SupportedProfile::SupportedProfile() - : profile(media::VIDEO_CODEC_PROFILE_UNKNOWN), - max_framerate_numerator(0), - max_framerate_denominator(0) {} - -VideoEncodeAccelerator::SupportedProfile::SupportedProfile( - VideoCodecProfile profile, - const Size& max_resolution, - uint32_t max_framerate_numerator, - uint32_t max_framerate_denominator) - : profile(profile), - max_resolution(max_resolution), - max_framerate_numerator(max_framerate_numerator), - max_framerate_denominator(max_framerate_denominator) {} - -VideoEncodeAccelerator::SupportedProfile::~SupportedProfile() = default; - -} // namespace media diff --git a/accel/video_encode_accelerator.h b/accel/video_encode_accelerator.h deleted file mode 100644 index 200930a..0000000 --- a/accel/video_encode_accelerator.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 9e40822e3a3d -// Note: only necessary functions are ported. - -#ifndef MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_ -#define MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> -#include <vector> - -#include "base/macros.h" -#include "base/optional.h" -#include "base/time/time.h" - -#include "size.h" -#include "video_codecs.h" - -namespace media { - -// class BitstreamBuffer; -// class VideoFrame; - -// Metadata for a VP8 bitstream buffer. -// |non_reference| is true iff this frame does not update any reference buffer, -// meaning dropping this frame still results in a decodable -// stream. -// |temporal_idx| indicates the temporal index for this frame. -// |layer_sync| if true iff this frame has |temporal_idx| > 0 and does NOT -// reference any reference buffer containing a frame with -// temporal_idx > 0. -struct Vp8Metadata final { - Vp8Metadata(); - Vp8Metadata(const Vp8Metadata& other); - Vp8Metadata(Vp8Metadata&& other); - ~Vp8Metadata(); - bool non_reference; - uint8_t temporal_idx; - bool layer_sync; -}; - -// Metadata associated with a bitstream buffer. -// |payload_size| is the byte size of the used portion of the buffer. -// |key_frame| is true if this delivered frame is a keyframe. -// |timestamp| is the same timestamp as in VideoFrame passed to Encode(). -// |vp8|, if set, contains metadata specific to VP8. See above. -struct BitstreamBufferMetadata final { - BitstreamBufferMetadata(); - BitstreamBufferMetadata(BitstreamBufferMetadata&& other); - BitstreamBufferMetadata(size_t payload_size_bytes, - bool key_frame, - base::TimeDelta timestamp); - ~BitstreamBufferMetadata(); - size_t payload_size_bytes; - bool key_frame; - base::TimeDelta timestamp; - base::Optional<Vp8Metadata> vp8; -}; - -// Video encoder interface. -class VideoEncodeAccelerator { - public: - // Specification of an encoding profile supported by an encoder. - struct SupportedProfile { - SupportedProfile(); - SupportedProfile(VideoCodecProfile profile, - const Size& max_resolution, - uint32_t max_framerate_numerator = 0u, - uint32_t max_framerate_denominator = 1u); - ~SupportedProfile(); - - VideoCodecProfile profile; - Size min_resolution; - Size max_resolution; - uint32_t max_framerate_numerator; - uint32_t max_framerate_denominator; - }; - using SupportedProfiles = std::vector<SupportedProfile>; -}; - -} // namespace media - -#endif // MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_ diff --git a/accel/video_frame.cc b/accel/video_frame.cc deleted file mode 100644 index 2b1c7a6..0000000 --- a/accel/video_frame.cc +++ /dev/null @@ -1,821 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 602bc8fa60fa -// Note: only necessary functions are ported. -// Note: some shared memory-related functionality here is no longer present in -// Chromium. - -#include "video_frame.h" - -#include <algorithm> -#include <climits> -#include <limits> -#include <numeric> -#include <utility> - -#include "base/atomic_sequence_num.h" -#include "base/bind.h" -#include "base/bits.h" -#include "base/callback_helpers.h" -#include "base/logging.h" -#include "base/memory/aligned_memory.h" -#include "base/stl_util.h" -#include "base/strings/string_piece.h" -#include "base/strings/stringprintf.h" -#include "base/time/time.h" -#include "media_limits.h" - -namespace media { - -// Note: moved from Chromium media/base/timestamp_constants.h -// Indicates an invalid or missing timestamp. -constexpr base::TimeDelta kNoTimestamp = - base::TimeDelta::FromMicroseconds(std::numeric_limits<int64_t>::min()); - -namespace { - -// Helper to provide Rect::Intersect() as an expression. -Rect Intersection(Rect a, const Rect& b) { - a.Intersect(b); - return a; -} - -// Note: moved from Chromium base/bits.h which is not included in libchrome. -// Round down |size| to a multiple of alignment, which must be a power of two. -size_t AlignDown(size_t size, size_t alignment) { - DCHECK(base::bits::IsPowerOfTwo(alignment)); - return size & ~(alignment - 1); -} - -} // namespace - -// Static constexpr class for generating unique identifiers for each VideoFrame. -static base::AtomicSequenceNumber g_unique_id_generator; - -static std::string StorageTypeToString( - const VideoFrame::StorageType storage_type) { - switch (storage_type) { - case VideoFrame::STORAGE_UNKNOWN: - return "UNKNOWN"; - case VideoFrame::STORAGE_OPAQUE: - return "OPAQUE"; - case VideoFrame::STORAGE_UNOWNED_MEMORY: - return "UNOWNED_MEMORY"; - case VideoFrame::STORAGE_OWNED_MEMORY: - return "OWNED_MEMORY"; - case VideoFrame::STORAGE_SHMEM: - return "SHMEM"; - case VideoFrame::STORAGE_DMABUFS: - return "DMABUFS"; - case VideoFrame::STORAGE_MOJO_SHARED_BUFFER: - return "MOJO_SHARED_BUFFER"; - } - - NOTREACHED() << "Invalid StorageType provided: " << storage_type; - return "INVALID"; -} - -// static -bool VideoFrame::IsStorageTypeMappable(VideoFrame::StorageType storage_type) { - return - // This is not strictly needed but makes explicit that, at VideoFrame - // level, DmaBufs are not mappable from userspace. - storage_type != VideoFrame::STORAGE_DMABUFS && - (storage_type == VideoFrame::STORAGE_UNOWNED_MEMORY || - storage_type == VideoFrame::STORAGE_OWNED_MEMORY || - storage_type == VideoFrame::STORAGE_SHMEM || - storage_type == VideoFrame::STORAGE_MOJO_SHARED_BUFFER); -} - -// If it is required to allocate aligned to multiple-of-two size overall for the -// frame of pixel |format|. -static bool RequiresEvenSizeAllocation(VideoPixelFormat format) { - switch (format) { - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - case PIXEL_FORMAT_BGRA: - return false; - case PIXEL_FORMAT_NV12: - case PIXEL_FORMAT_NV21: - case PIXEL_FORMAT_I420: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_YV12: - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_P016LE: - return true; - case PIXEL_FORMAT_UNKNOWN: - break; - } - NOTREACHED() << "Unsupported video frame format: " << format; - return false; -} - -// Creates VideoFrameLayout for tightly packed frame. -static base::Optional<VideoFrameLayout> GetDefaultLayout( - VideoPixelFormat format, - const Size& coded_size) { - std::vector<ColorPlaneLayout> planes; - - switch (format) { - case PIXEL_FORMAT_I420: { - int uv_width = (coded_size.width() + 1) / 2; - int uv_height = (coded_size.height() + 1) / 2; - int uv_stride = uv_width; - int uv_size = uv_stride * uv_height; - planes = std::vector<ColorPlaneLayout>{ - ColorPlaneLayout(coded_size.width(), 0, coded_size.GetArea()), - ColorPlaneLayout(uv_stride, coded_size.GetArea(), uv_size), - ColorPlaneLayout(uv_stride, coded_size.GetArea() + uv_size, uv_size), - }; - break; - } - - case PIXEL_FORMAT_Y16: - planes = std::vector<ColorPlaneLayout>{ColorPlaneLayout( - coded_size.width() * 2, 0, coded_size.GetArea() * 2)}; - break; - - case PIXEL_FORMAT_ARGB: - planes = std::vector<ColorPlaneLayout>{ColorPlaneLayout( - coded_size.width() * 4, 0, coded_size.GetArea() * 4)}; - break; - - case PIXEL_FORMAT_NV12: { - int uv_width = (coded_size.width() + 1) / 2; - int uv_height = (coded_size.height() + 1) / 2; - int uv_stride = uv_width * 2; - int uv_size = uv_stride * uv_height; - planes = std::vector<ColorPlaneLayout>{ - ColorPlaneLayout(coded_size.width(), 0, coded_size.GetArea()), - ColorPlaneLayout(uv_stride, coded_size.GetArea(), uv_size), - }; - break; - } - - default: - // TODO(miu): This function should support any pixel format. - // http://crbug.com/555909 . - DLOG(ERROR) - << "Only PIXEL_FORMAT_I420, PIXEL_FORMAT_Y16, PIXEL_FORMAT_NV12, " - "and PIXEL_FORMAT_ARGB formats are supported: " - << VideoPixelFormatToString(format); - return base::nullopt; - } - - return VideoFrameLayout::CreateWithPlanes(format, coded_size, planes); -} - -// static -bool VideoFrame::IsValidConfig(VideoPixelFormat format, - StorageType storage_type, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size) { - // Check maximum limits for all formats. - int coded_size_area = coded_size.GetCheckedArea().ValueOrDefault(INT_MAX); - int natural_size_area = natural_size.GetCheckedArea().ValueOrDefault(INT_MAX); - static_assert(limits::kMaxCanvas < INT_MAX, ""); - if (coded_size_area > limits::kMaxCanvas || - coded_size.width() > limits::kMaxDimension || - coded_size.height() > limits::kMaxDimension || visible_rect.x() < 0 || - visible_rect.y() < 0 || visible_rect.right() > coded_size.width() || - visible_rect.bottom() > coded_size.height() || - natural_size_area > limits::kMaxCanvas || - natural_size.width() > limits::kMaxDimension || - natural_size.height() > limits::kMaxDimension) { - return false; - } - - // TODO(mcasas): Remove parameter |storage_type| when the opaque storage types - // comply with the checks below. Right now we skip them. - if (!IsStorageTypeMappable(storage_type)) - return true; - - // Make sure new formats are properly accounted for in the method. - static_assert(PIXEL_FORMAT_MAX == 32, - "Added pixel format, please review IsValidConfig()"); - - if (format == PIXEL_FORMAT_UNKNOWN) { - return coded_size.IsEmpty() && visible_rect.IsEmpty() && - natural_size.IsEmpty(); - } - - // Check that software-allocated buffer formats are not empty. - return !coded_size.IsEmpty() && !visible_rect.IsEmpty() && - !natural_size.IsEmpty(); -} - -// static -scoped_refptr<VideoFrame> VideoFrame::CreateFrame(VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp) { - return CreateFrameInternal(format, coded_size, visible_rect, natural_size, - timestamp, false); -} - -// static -scoped_refptr<VideoFrame> VideoFrame::WrapExternalSharedMemory( - VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - uint8_t* data, - size_t data_size, - base::SharedMemoryHandle handle, - size_t data_offset, - base::TimeDelta timestamp) { - auto layout = GetDefaultLayout(format, coded_size); - if (!layout) - return nullptr; - return WrapExternalStorage(STORAGE_SHMEM, *layout, visible_rect, natural_size, - data, data_size, timestamp, nullptr, nullptr, - handle, data_offset); -} - -// static -scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() { - auto layout = VideoFrameLayout::Create(PIXEL_FORMAT_UNKNOWN, Size()); - if (!layout) { - DLOG(ERROR) << "Invalid layout."; - return nullptr; - } - scoped_refptr<VideoFrame> frame = - new VideoFrame(*layout, STORAGE_UNKNOWN, Rect(), Size(), kNoTimestamp); - frame->metadata()->SetBoolean(VideoFrameMetadata::END_OF_STREAM, true); - return frame; -} - -// static -size_t VideoFrame::NumPlanes(VideoPixelFormat format) { - return VideoFrameLayout::NumPlanes(format); -} - -// static -size_t VideoFrame::AllocationSize(VideoPixelFormat format, - const Size& coded_size) { - size_t total = 0; - for (size_t i = 0; i < NumPlanes(format); ++i) - total += PlaneSize(format, i, coded_size).GetArea(); - return total; -} - -// static -Size VideoFrame::PlaneSize(VideoPixelFormat format, - size_t plane, - const Size& coded_size) { - DCHECK(IsValidPlane(plane, format)); - - int width = coded_size.width(); - int height = coded_size.height(); - if (RequiresEvenSizeAllocation(format)) { - // Align to multiple-of-two size overall. This ensures that non-subsampled - // planes can be addressed by pixel with the same scaling as the subsampled - // planes. - width = base::bits::Align(width, 2); - height = base::bits::Align(height, 2); - } - - const Size subsample = SampleSize(format, plane); - DCHECK(width % subsample.width() == 0); - DCHECK(height % subsample.height() == 0); - return Size(BytesPerElement(format, plane) * width / subsample.width(), - height / subsample.height()); -} - -// static -int VideoFrame::PlaneHorizontalBitsPerPixel(VideoPixelFormat format, - size_t plane) { - DCHECK(IsValidPlane(plane, format)); - const int bits_per_element = 8 * BytesPerElement(format, plane); - const int horiz_pixels_per_element = SampleSize(format, plane).width(); - DCHECK_EQ(bits_per_element % horiz_pixels_per_element, 0); - return bits_per_element / horiz_pixels_per_element; -} - -// static -int VideoFrame::PlaneBitsPerPixel(VideoPixelFormat format, size_t plane) { - DCHECK(IsValidPlane(plane, format)); - return PlaneHorizontalBitsPerPixel(format, plane) / - SampleSize(format, plane).height(); -} - -// static -size_t VideoFrame::RowBytes(size_t plane, VideoPixelFormat format, int width) { - DCHECK(IsValidPlane(plane, format)); - return BytesPerElement(format, plane) * Columns(plane, format, width); -} - -// static -int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) { - DCHECK(IsValidPlane(format, plane)); - switch (format) { - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_BGRA: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - return 4; - case PIXEL_FORMAT_RGB24: - return 3; - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - case PIXEL_FORMAT_P016LE: - return 2; - case PIXEL_FORMAT_NV12: - case PIXEL_FORMAT_NV21: { - static const int bytes_per_element[] = {1, 2}; - DCHECK_LT(plane, base::size(bytes_per_element)); - return bytes_per_element[plane]; - } - case PIXEL_FORMAT_YV12: - case PIXEL_FORMAT_I420: - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_I444: - return 1; - case PIXEL_FORMAT_MJPEG: - return 0; - case PIXEL_FORMAT_UNKNOWN: - break; - } - NOTREACHED(); - return 0; -} - -// static -std::vector<int32_t> VideoFrame::ComputeStrides(VideoPixelFormat format, - const Size& coded_size) { - std::vector<int32_t> strides; - const size_t num_planes = NumPlanes(format); - if (num_planes == 1) { - strides.push_back(RowBytes(0, format, coded_size.width())); - } else { - for (size_t plane = 0; plane < num_planes; ++plane) { - strides.push_back(base::bits::Align( - RowBytes(plane, format, coded_size.width()), kFrameAddressAlignment)); - } - } - return strides; -} - -// static -size_t VideoFrame::Rows(size_t plane, VideoPixelFormat format, int height) { - DCHECK(IsValidPlane(plane, format)); - const int sample_height = SampleSize(format, plane).height(); - return base::bits::Align(height, sample_height) / sample_height; -} - -// static -size_t VideoFrame::Columns(size_t plane, VideoPixelFormat format, int width) { - DCHECK(IsValidPlane(plane, format)); - const int sample_width = SampleSize(format, plane).width(); - return base::bits::Align(width, sample_width) / sample_width; -} - -bool VideoFrame::IsMappable() const { - return IsStorageTypeMappable(storage_type_); -} - -int VideoFrame::row_bytes(size_t plane) const { - return RowBytes(plane, format(), coded_size().width()); -} - -int VideoFrame::rows(size_t plane) const { - return Rows(plane, format(), coded_size().height()); -} - -const uint8_t* VideoFrame::visible_data(size_t plane) const { - DCHECK(IsValidPlane(plane, format())); - DCHECK(IsMappable()); - - // Calculate an offset that is properly aligned for all planes. - const Size alignment = CommonAlignment(format()); - const int offset_x = AlignDown(visible_rect_.x(), alignment.width()); - const int offset_y = AlignDown(visible_rect_.y(), alignment.height()); - - const Size subsample = SampleSize(format(), plane); - DCHECK(offset_x % subsample.width() == 0); - DCHECK(offset_y % subsample.height() == 0); - return data(plane) + - stride(plane) * (offset_y / subsample.height()) + // Row offset. - BytesPerElement(format(), plane) * // Column offset. - (offset_x / subsample.width()); -} - -uint8_t* VideoFrame::visible_data(size_t plane) { - return const_cast<uint8_t*>( - static_cast<const VideoFrame*>(this)->visible_data(plane)); -} - -base::ReadOnlySharedMemoryRegion* VideoFrame::read_only_shared_memory_region() - const { - DCHECK_EQ(storage_type_, STORAGE_SHMEM); - DCHECK(read_only_shared_memory_region_ && - read_only_shared_memory_region_->IsValid()); - return read_only_shared_memory_region_; -} - -base::UnsafeSharedMemoryRegion* VideoFrame::unsafe_shared_memory_region() - const { - DCHECK_EQ(storage_type_, STORAGE_SHMEM); - DCHECK(unsafe_shared_memory_region_ && - unsafe_shared_memory_region_->IsValid()); - return unsafe_shared_memory_region_; -} - -base::SharedMemoryHandle VideoFrame::shared_memory_handle() const { - DCHECK_EQ(storage_type_, STORAGE_SHMEM); - DCHECK(shared_memory_handle_.IsValid()); - return shared_memory_handle_; -} - -size_t VideoFrame::shared_memory_offset() const { - DCHECK_EQ(storage_type_, STORAGE_SHMEM); - DCHECK((read_only_shared_memory_region_ && - read_only_shared_memory_region_->IsValid()) || - (unsafe_shared_memory_region_ && - unsafe_shared_memory_region_->IsValid()) || - shared_memory_handle_.IsValid()); - return shared_memory_offset_; -} - -const std::vector<base::ScopedFD>& VideoFrame::DmabufFds() const { - DCHECK_EQ(storage_type_, STORAGE_DMABUFS); - - return dmabuf_fds_; -} - -bool VideoFrame::HasDmaBufs() const { - return !dmabuf_fds_.empty(); -} - -void VideoFrame::AddReadOnlySharedMemoryRegion( - base::ReadOnlySharedMemoryRegion* region) { - storage_type_ = STORAGE_SHMEM; - DCHECK(SharedMemoryUninitialized()); - DCHECK(region && region->IsValid()); - read_only_shared_memory_region_ = region; -} - -void VideoFrame::AddUnsafeSharedMemoryRegion( - base::UnsafeSharedMemoryRegion* region) { - storage_type_ = STORAGE_SHMEM; - DCHECK(SharedMemoryUninitialized()); - DCHECK(region && region->IsValid()); - unsafe_shared_memory_region_ = region; -} - -void VideoFrame::AddSharedMemoryHandle(base::SharedMemoryHandle handle) { - storage_type_ = STORAGE_SHMEM; - DCHECK(SharedMemoryUninitialized()); - shared_memory_handle_ = handle; -} - -void VideoFrame::AddDestructionObserver(base::OnceClosure callback) { - DCHECK(!callback.is_null()); - done_callbacks_.push_back(std::move(callback)); -} - -std::string VideoFrame::AsHumanReadableString() { - if (metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)) - return "end of stream"; - - std::ostringstream s; - s << ConfigToString(format(), storage_type_, coded_size(), visible_rect_, - natural_size_) - << " timestamp:" << timestamp_.InMicroseconds(); - return s.str(); -} - -size_t VideoFrame::BitDepth() const { - return media::BitDepth(format()); -} - -// static -scoped_refptr<VideoFrame> VideoFrame::WrapExternalStorage( - StorageType storage_type, - const VideoFrameLayout& layout, - const Rect& visible_rect, - const Size& natural_size, - uint8_t* data, - size_t data_size, - base::TimeDelta timestamp, - base::ReadOnlySharedMemoryRegion* read_only_region, - base::UnsafeSharedMemoryRegion* unsafe_region, - base::SharedMemoryHandle handle, - size_t data_offset) { - DCHECK(IsStorageTypeMappable(storage_type)); - - if (!IsValidConfig(layout.format(), storage_type, layout.coded_size(), - visible_rect, natural_size)) { - DLOG(ERROR) << __func__ << " Invalid config." - << ConfigToString(layout.format(), storage_type, - layout.coded_size(), visible_rect, - natural_size); - return nullptr; - } - - scoped_refptr<VideoFrame> frame = new VideoFrame( - layout, storage_type, visible_rect, natural_size, timestamp); - - for (size_t i = 0; i < layout.planes().size(); ++i) { - frame->data_[i] = data + layout.planes()[i].offset; - } - - if (storage_type == STORAGE_SHMEM) { - if (read_only_region || unsafe_region) { - DCHECK(!handle.IsValid()); - DCHECK_NE(!!read_only_region, !!unsafe_region) - << "Expected exactly one read-only or unsafe region for " - << "STORAGE_SHMEM VideoFrame"; - if (read_only_region) { - frame->read_only_shared_memory_region_ = read_only_region; - DCHECK(frame->read_only_shared_memory_region_->IsValid()); - } else if (unsafe_region) { - frame->unsafe_shared_memory_region_ = unsafe_region; - DCHECK(frame->unsafe_shared_memory_region_->IsValid()); - } - frame->shared_memory_offset_ = data_offset; - } else { - frame->AddSharedMemoryHandle(handle); - frame->shared_memory_offset_ = data_offset; - } - } - - return frame; -} - -VideoFrame::VideoFrame(const VideoFrameLayout& layout, - StorageType storage_type, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp) - : layout_(layout), - storage_type_(storage_type), - visible_rect_(Intersection(visible_rect, Rect(layout.coded_size()))), - natural_size_(natural_size), - shared_memory_offset_(0), - timestamp_(timestamp), - unique_id_(g_unique_id_generator.GetNext()) { - DCHECK(IsValidConfig(format(), storage_type, coded_size(), visible_rect_, - natural_size_)); - DCHECK(visible_rect_ == visible_rect) - << "visible_rect " << visible_rect.ToString() << " exceeds coded_size " - << coded_size().ToString(); - memset(&data_, 0, sizeof(data_)); -} - -VideoFrame::~VideoFrame() { - for (auto& callback : done_callbacks_) - std::move(callback).Run(); -} - -// static -std::string VideoFrame::ConfigToString(const VideoPixelFormat format, - const StorageType storage_type, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size) { - return base::StringPrintf( - "format:%s storage_type:%s coded_size:%s visible_rect:%s natural_size:%s", - VideoPixelFormatToString(format).c_str(), - StorageTypeToString(storage_type).c_str(), coded_size.ToString().c_str(), - visible_rect.ToString().c_str(), natural_size.ToString().c_str()); -} - -// static -bool VideoFrame::IsValidPlane(size_t plane, VideoPixelFormat format) { - DCHECK_LE(NumPlanes(format), static_cast<size_t>(kMaxPlanes)); - return (plane < NumPlanes(format)); -} - -// static -Size VideoFrame::DetermineAlignedSize(VideoPixelFormat format, - const Size& dimensions) { - const Size alignment = CommonAlignment(format); - const Size adjusted = - Size(base::bits::Align(dimensions.width(), alignment.width()), - base::bits::Align(dimensions.height(), alignment.height())); - DCHECK((adjusted.width() % alignment.width() == 0) && - (adjusted.height() % alignment.height() == 0)); - return adjusted; -} - -// static -scoped_refptr<VideoFrame> VideoFrame::CreateFrameInternal( - VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp, - bool zero_initialize_memory) { - // Since we're creating a new frame (and allocating memory for it ourselves), - // we can pad the requested |coded_size| if necessary if the request does not - // line up on sample boundaries. See discussion at http://crrev.com/1240833003 - const Size new_coded_size = DetermineAlignedSize(format, coded_size); - auto layout = VideoFrameLayout::CreateWithStrides( - format, new_coded_size, ComputeStrides(format, coded_size)); - if (!layout) { - DLOG(ERROR) << "Invalid layout."; - return nullptr; - } - - return CreateFrameWithLayout(*layout, visible_rect, natural_size, timestamp, - zero_initialize_memory); -} - -scoped_refptr<VideoFrame> VideoFrame::CreateFrameWithLayout( - const VideoFrameLayout& layout, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp, - bool zero_initialize_memory) { - const StorageType storage = STORAGE_OWNED_MEMORY; - if (!IsValidConfig(layout.format(), storage, layout.coded_size(), - visible_rect, natural_size)) { - DLOG(ERROR) << __func__ << " Invalid config." - << ConfigToString(layout.format(), storage, layout.coded_size(), - visible_rect, natural_size); - return nullptr; - } - - scoped_refptr<VideoFrame> frame(new VideoFrame( - std::move(layout), storage, visible_rect, natural_size, timestamp)); - frame->AllocateMemory(zero_initialize_memory); - return frame; -} - -bool VideoFrame::SharedMemoryUninitialized() { - return !read_only_shared_memory_region_ && !unsafe_shared_memory_region_ && - !shared_memory_handle_.IsValid(); -} - -// static -bool VideoFrame::IsValidPlane(VideoPixelFormat format, size_t plane) { - DCHECK_LE(NumPlanes(format), static_cast<size_t>(kMaxPlanes)); - return plane < NumPlanes(format); -} - -// static -Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) { - DCHECK(IsValidPlane(format, plane)); - - switch (plane) { - case kYPlane: // and kARGBPlane: - case kAPlane: - return Size(1, 1); - - case kUPlane: // and kUVPlane: - case kVPlane: - switch (format) { - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV444P12: - case PIXEL_FORMAT_Y16: - return Size(1, 1); - - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV422P12: - return Size(2, 1); - - case PIXEL_FORMAT_YV12: - case PIXEL_FORMAT_I420: - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_NV12: - case PIXEL_FORMAT_NV21: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_P016LE: - return Size(2, 2); - - case PIXEL_FORMAT_UNKNOWN: - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - case PIXEL_FORMAT_BGRA: - break; - } - } - NOTREACHED(); - return Size(); -} - -// static -Size VideoFrame::CommonAlignment(VideoPixelFormat format) { - int max_sample_width = 0; - int max_sample_height = 0; - for (size_t plane = 0; plane < NumPlanes(format); ++plane) { - const Size sample_size = SampleSize(format, plane); - max_sample_width = std::max(max_sample_width, sample_size.width()); - max_sample_height = std::max(max_sample_height, sample_size.height()); - } - return Size(max_sample_width, max_sample_height); -} - -void VideoFrame::AllocateMemory(bool zero_initialize_memory) { - DCHECK_EQ(storage_type_, STORAGE_OWNED_MEMORY); - static_assert(0 == kYPlane, "y plane data must be index 0"); - - std::vector<size_t> plane_size = CalculatePlaneSize(); - const size_t total_buffer_size = - std::accumulate(plane_size.begin(), plane_size.end(), 0u); - - uint8_t* data = reinterpret_cast<uint8_t*>( - base::AlignedAlloc(total_buffer_size, layout_.buffer_addr_align())); - if (zero_initialize_memory) { - memset(data, 0, total_buffer_size); - } - AddDestructionObserver(base::BindOnce(&base::AlignedFree, data)); - - // Note that if layout.buffer_sizes is specified, color planes' layout is the - // same as buffers'. See CalculatePlaneSize() for detail. - for (size_t plane = 0, offset = 0; plane < NumPlanes(format()); ++plane) { - data_[plane] = data + offset; - offset += plane_size[plane]; - } -} - -std::vector<size_t> VideoFrame::CalculatePlaneSize() const { - // We have two cases for plane size mapping: - // 1) If plane size is specified: use planes' size. - // 2) VideoFrameLayout::size is unassigned: use legacy calculation formula. - - const size_t num_planes = NumPlanes(format()); - const auto& planes = layout_.planes(); - std::vector<size_t> plane_size(num_planes); - bool plane_size_assigned = true; - DCHECK_EQ(planes.size(), num_planes); - for (size_t i = 0; i < num_planes; ++i) { - plane_size[i] = planes[i].size; - plane_size_assigned &= plane_size[i] != 0; - } - - if (plane_size_assigned) - return plane_size; - - // Reset plane size. - std::fill(plane_size.begin(), plane_size.end(), 0u); - for (size_t plane = 0; plane < num_planes; ++plane) { - // These values were chosen to mirror ffmpeg's get_video_buffer(). - // TODO(dalecurtis): This should be configurable; eventually ffmpeg wants - // us to use av_cpu_max_align(), but... for now, they just hard-code 32. - const size_t height = - base::bits::Align(rows(plane), kFrameAddressAlignment); - const size_t width = std::abs(stride(plane)); - plane_size[plane] = width * height; - } - - if (num_planes > 1) { - // The extra line of UV being allocated is because h264 chroma MC - // overreads by one line in some cases, see libavcodec/utils.c: - // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm: - // put_h264_chroma_mc4_ssse3(). - DCHECK(IsValidPlane(format(), kUPlane)); - plane_size.back() += std::abs(stride(kUPlane)) + kFrameSizePadding; - } - return plane_size; -} - -} // namespace media diff --git a/accel/video_frame.h b/accel/video_frame.h deleted file mode 100644 index 468fe9e..0000000 --- a/accel/video_frame.h +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 602bc8fa60fa -// Note: only necessary functions are ported. -// Note: some OS-specific defines have been removed -// Note: WrapExternalSharedMemory() has been removed in Chromium, but is still -// present here. Porting the code to a newer version of VideoFrame is not -// useful, as this is only a temporary step and all usage of VideoFrame will -// be removed. - -#ifndef VIDEO_FRAME_H_ -#define VIDEO_FRAME_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "base/callback.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/memory/aligned_memory.h" -#include "base/memory/read_only_shared_memory_region.h" -#include "base/memory/ref_counted.h" -#include "base/memory/shared_memory.h" -#include "base/memory/shared_memory_handle.h" -#include "base/memory/unsafe_shared_memory_region.h" -#include "base/optional.h" -#include "base/synchronization/lock.h" -#include "base/thread_annotations.h" -#include "base/unguessable_token.h" -#include "rect.h" -#include "size.h" -#include "video_frame_layout.h" -#include "video_frame_metadata.h" -#include "video_pixel_format.h" - -#include "base/files/scoped_file.h" - -namespace media { - -class VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { - public: - enum { - kFrameSizeAlignment = 16, - kFrameSizePadding = 16, - - kFrameAddressAlignment = VideoFrameLayout::kBufferAddressAlignment - }; - - enum { - kMaxPlanes = 4, - - kYPlane = 0, - kARGBPlane = kYPlane, - kUPlane = 1, - kUVPlane = kUPlane, - kVPlane = 2, - kAPlane = 3, - }; - - // Defines the pixel storage type. Differentiates between directly accessible - // |data_| and pixels that are only indirectly accessible and not via mappable - // memory. - // Note that VideoFrames of any StorageType can also have Texture backing, - // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE. - enum StorageType { - STORAGE_UNKNOWN = 0, - STORAGE_OPAQUE = 1, // We don't know how VideoFrame's pixels are stored. - STORAGE_UNOWNED_MEMORY = 2, // External, non owned data pointers. - STORAGE_OWNED_MEMORY = 3, // VideoFrame has allocated its own data buffer. - STORAGE_SHMEM = 4, // Pixels are backed by Shared Memory. - // TODO(mcasas): Consider turning this type into STORAGE_NATIVE - // based on the idea of using this same enum value for both DMA - // buffers on Linux and CVPixelBuffers on Mac (which currently use - // STORAGE_UNOWNED_MEMORY) and handle it appropriately in all cases. - STORAGE_DMABUFS = 5, // Each plane is stored into a DmaBuf. - STORAGE_MOJO_SHARED_BUFFER = 6, - STORAGE_LAST = STORAGE_MOJO_SHARED_BUFFER, - }; - - // Call prior to CreateFrame to ensure validity of frame configuration. Called - // automatically by VideoDecoderConfig::IsValidConfig(). - static bool IsValidConfig(VideoPixelFormat format, - StorageType storage_type, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size); - - // Creates a new frame in system memory with given parameters. Buffers for the - // frame are allocated but not initialized. The caller must not make - // assumptions about the actual underlying size(s), but check the returned - // VideoFrame instead. - static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp); - - // Creates a new frame in system memory with given parameters. Buffers for the - // frame are allocated but not initialized. The caller should specify the - // physical buffer size and strides if needed in |layout| parameter. - static scoped_refptr<VideoFrame> CreateFrameWithLayout( - const VideoFrameLayout& layout, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp, - bool zero_initialize_memory); - - // Legacy wrapping of old SharedMemoryHandle objects. Deprecated, use one of - // the shared memory region wrappers above instead. - static scoped_refptr<VideoFrame> WrapExternalSharedMemory( - VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - uint8_t* data, - size_t data_size, - base::SharedMemoryHandle handle, - size_t shared_memory_offset, - base::TimeDelta timestamp); - - // Creates a frame which indicates end-of-stream. - static scoped_refptr<VideoFrame> CreateEOSFrame(); - - static size_t NumPlanes(VideoPixelFormat format); - - // Returns the required allocation size for a (tightly packed) frame of the - // given coded size and format. - static size_t AllocationSize(VideoPixelFormat format, const Size& coded_size); - - // Returns the plane Size (in bytes) for a plane of the given coded size - // and format. - static Size PlaneSize(VideoPixelFormat format, - size_t plane, - const Size& coded_size); - - // Returns horizontal bits per pixel for given |plane| and |format|. - static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane); - - // Returns bits per pixel for given |plane| and |format|. - static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane); - - // Returns the number of bytes per row for the given plane, format, and width. - // The width may be aligned to format requirements. - static size_t RowBytes(size_t plane, VideoPixelFormat format, int width); - - // Returns the number of bytes per element for given |plane| and |format|. - static int BytesPerElement(VideoPixelFormat format, size_t plane); - - // Calculates strides for each plane based on |format| and |coded_size|. - static std::vector<int32_t> ComputeStrides(VideoPixelFormat format, - const Size& coded_size); - - // Returns the number of rows for the given plane, format, and height. - // The height may be aligned to format requirements. - static size_t Rows(size_t plane, VideoPixelFormat format, int height); - - // Returns the number of columns for the given plane, format, and width. - // The width may be aligned to format requirements. - static size_t Columns(size_t plane, VideoPixelFormat format, int width); - - // Returns true if |frame| is accesible mapped in the VideoFrame memory space. - // static - static bool IsStorageTypeMappable(VideoFrame::StorageType storage_type); - - // Returns true if |frame| is accessible and mapped in the VideoFrame memory - // space. If false, clients should refrain from accessing data(), - // visible_data() etc. - bool IsMappable() const; - - const VideoFrameLayout& layout() const { return layout_; } - - VideoPixelFormat format() const { return layout_.format(); } - StorageType storage_type() const { return storage_type_; } - - // The full dimensions of the video frame data. - const Size& coded_size() const { return layout_.coded_size(); } - // A subsection of [0, 0, coded_size().width(), coded_size.height()]. This - // can be set to "soft-apply" a cropping. It determines the pointers into - // the data returned by visible_data(). - const Rect& visible_rect() const { return visible_rect_; } - // Specifies that the |visible_rect| section of the frame is supposed to be - // scaled to this size when being presented. This can be used to represent - // anamorphic frames, or to "soft-apply" any custom scaling. - const Size& natural_size() const { return natural_size_; } - - int stride(size_t plane) const { - DCHECK(IsValidPlane(plane, format())); - DCHECK_LT(plane, layout_.num_planes()); - return layout_.planes()[plane].stride; - } - - // Returns the number of bytes per row and number of rows for a given plane. - // - // As opposed to stride(), row_bytes() refers to the bytes representing - // frame data scanlines (coded_size.width() pixels, without stride padding). - int row_bytes(size_t plane) const; - int rows(size_t plane) const; - - // Returns pointer to the buffer for a given plane, if this is an - // IsMappable() frame type. The memory is owned by VideoFrame object and must - // not be freed by the caller. - const uint8_t* data(size_t plane) const { - DCHECK(IsValidPlane(plane, format())); - DCHECK(IsMappable()); - return data_[plane]; - } - uint8_t* data(size_t plane) { - DCHECK(IsValidPlane(plane, format())); - DCHECK(IsMappable()); - return data_[plane]; - } - - // Returns pointer to the data in the visible region of the frame, for - // IsMappable() storage types. The returned pointer is offsetted into the - // plane buffer specified by visible_rect().origin(). Memory is owned by - // VideoFrame object and must not be freed by the caller. - const uint8_t* visible_data(size_t plane) const; - uint8_t* visible_data(size_t plane); - - // Returns a pointer to the read-only shared-memory region, if present. - base::ReadOnlySharedMemoryRegion* read_only_shared_memory_region() const; - - // Returns a pointer to the unsafe shared memory handle, if present. - base::UnsafeSharedMemoryRegion* unsafe_shared_memory_region() const; - - // Returns the legacy SharedMemoryHandle, if present. - base::SharedMemoryHandle shared_memory_handle() const; - - // Returns the offset into the shared memory where the frame data begins. - size_t shared_memory_offset() const; - - // Returns a vector containing the backing DmaBufs for this frame. The number - // of returned DmaBufs will be equal or less than the number of planes of - // the frame. If there are less, this means that the last FD contains the - // remaining planes. - // Note that the returned FDs are still owned by the VideoFrame. This means - // that the caller shall not close them, or use them after the VideoFrame is - // destroyed. For such use cases, use media::DuplicateFDs() to obtain your - // own copy of the FDs. - const std::vector<base::ScopedFD>& DmabufFds() const; - - // Returns true if |frame| has DmaBufs. - bool HasDmaBufs() const; - - void AddReadOnlySharedMemoryRegion(base::ReadOnlySharedMemoryRegion* region); - void AddUnsafeSharedMemoryRegion(base::UnsafeSharedMemoryRegion* region); - - // Legacy, use one of the Add*SharedMemoryRegion methods above instead. - void AddSharedMemoryHandle(base::SharedMemoryHandle handle); - - // Adds a callback to be run when the VideoFrame is about to be destroyed. - // The callback may be run from ANY THREAD, and so it is up to the client to - // ensure thread safety. Although read-only access to the members of this - // VideoFrame is permitted while the callback executes (including - // VideoFrameMetadata), clients should not assume the data pointers are - // valid. - void AddDestructionObserver(base::OnceClosure callback); - - // Returns a dictionary of optional metadata. This contains information - // associated with the frame that downstream clients might use for frame-level - // logging, quality/performance optimizations, signaling, etc. - // - // TODO(miu): Move some of the "extra" members of VideoFrame (below) into - // here as a later clean-up step. - const VideoFrameMetadata* metadata() const { return &metadata_; } - VideoFrameMetadata* metadata() { return &metadata_; } - - // The time span between the current frame and the first frame of the stream. - // This is the media timestamp, and not the reference time. - // See VideoFrameMetadata::REFERENCE_TIME for details. - base::TimeDelta timestamp() const { return timestamp_; } - void set_timestamp(base::TimeDelta timestamp) { timestamp_ = timestamp; } - - // Returns a human-readable string describing |*this|. - std::string AsHumanReadableString(); - - // Unique identifier for this video frame; generated at construction time and - // guaranteed to be unique within a single process. - int unique_id() const { return unique_id_; } - - // Returns the number of bits per channel. - size_t BitDepth() const; - - protected: - friend class base::RefCountedThreadSafe<VideoFrame>; - - // Clients must use the static factory/wrapping methods to create a new frame. - // Derived classes should create their own factory/wrapping methods, and use - // this constructor to do basic initialization. - VideoFrame(const VideoFrameLayout& layout, - StorageType storage_type, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp); - - virtual ~VideoFrame(); - - // Creates a summary of the configuration settings provided as parameters. - static std::string ConfigToString(const VideoPixelFormat format, - const VideoFrame::StorageType storage_type, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size); - - // Returns true if |plane| is a valid plane index for the given |format|. - static bool IsValidPlane(size_t plane, VideoPixelFormat format); - - // Returns |dimensions| adjusted to appropriate boundaries based on |format|. - static Size DetermineAlignedSize(VideoPixelFormat format, - const Size& dimensions); - - void set_data(size_t plane, uint8_t* ptr) { - DCHECK(IsValidPlane(plane, format())); - DCHECK(ptr); - data_[plane] = ptr; - } - - private: - static scoped_refptr<VideoFrame> WrapExternalStorage( - StorageType storage_type, - const VideoFrameLayout& layout, - const Rect& visible_rect, - const Size& natural_size, - uint8_t* data, - size_t data_size, - base::TimeDelta timestamp, - base::ReadOnlySharedMemoryRegion* read_only_region, - base::UnsafeSharedMemoryRegion* unsafe_region, - base::SharedMemoryHandle handle, - size_t data_offset); - - static scoped_refptr<VideoFrame> CreateFrameInternal( - VideoPixelFormat format, - const Size& coded_size, - const Rect& visible_rect, - const Size& natural_size, - base::TimeDelta timestamp, - bool zero_initialize_memory); - - bool SharedMemoryUninitialized(); - - // Returns true if |plane| is a valid plane index for the given |format|. - static bool IsValidPlane(VideoPixelFormat format, size_t plane); - - // Returns the pixel size of each subsample for a given |plane| and |format|. - // E.g. 2x2 for the U-plane in PIXEL_FORMAT_I420. - static Size SampleSize(VideoPixelFormat format, size_t plane); - - // Return the alignment for the whole frame, calculated as the max of the - // alignment for each individual plane. - static Size CommonAlignment(VideoPixelFormat format); - - void AllocateMemory(bool zero_initialize_memory); - - // Calculates plane size. - // It first considers buffer size layout_ object provides. If layout's - // number of buffers equals to number of planes, and buffer size is assigned - // (non-zero), it returns buffers' size. - // Otherwise, it uses the first (num_buffers - 1) assigned buffers' size as - // plane size. Then for the rest unassigned planes, calculates their size - // based on format, coded size and stride for the plane. - std::vector<size_t> CalculatePlaneSize() const; - - // VideFrameLayout (includes format, coded_size, and strides). - const VideoFrameLayout layout_; - - // Storage type for the different planes. - StorageType storage_type_; // TODO(mcasas): make const - - // Width, height, and offsets of the visible portion of the video frame. Must - // be a subrect of |coded_size_|. Can be odd with respect to the sample - // boundaries, e.g. for formats with subsampled chroma. - const Rect visible_rect_; - - // Width and height of the visible portion of the video frame - // (|visible_rect_.size()|) with aspect ratio taken into account. - const Size natural_size_; - - // Array of data pointers to each plane. - // TODO(mcasas): we don't know on ctor if we own |data_| or not. Change - // to std::unique_ptr<uint8_t, AlignedFreeDeleter> after refactoring - // VideoFrame. - uint8_t* data_[kMaxPlanes]; - - // Shared memory handle and associated offset inside it, if this frame is a - // STORAGE_SHMEM one. Pointers to unowned shared memory regions. At most one - // of the memory regions will be set. - base::ReadOnlySharedMemoryRegion* read_only_shared_memory_region_ = nullptr; - base::UnsafeSharedMemoryRegion* unsafe_shared_memory_region_ = nullptr; - - // Legacy handle. - base::SharedMemoryHandle shared_memory_handle_; - - // If this is a STORAGE_SHMEM frame, the offset of the data within the shared - // memory. - size_t shared_memory_offset_; - - class DmabufHolder; - - // Dmabufs for the frame, used when storage is STORAGE_DMABUFS. Size is either - // equal or less than the number of planes of the frame. If it is less, then - // the memory area represented by the last FD contains the remaining planes. - std::vector<base::ScopedFD> dmabuf_fds_; - - std::vector<base::OnceClosure> done_callbacks_; - - base::TimeDelta timestamp_; - - VideoFrameMetadata metadata_; - - // Generated at construction time. - const int unique_id_; - - DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame); -}; - -} // namespace media - -#endif // VIDEO_FRAME_H_ diff --git a/accel/video_frame_layout.cc b/accel/video_frame_layout.cc deleted file mode 100644 index 3f38314..0000000 --- a/accel/video_frame_layout.cc +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 3b7ce92816e2 - -#include "video_frame_layout.h" - -#include <string.h> -#include <numeric> -#include <sstream> - -#include "base/logging.h" - -namespace media { - -namespace { - -template <class T> -std::string VectorToString(const std::vector<T>& vec) { - std::ostringstream result; - std::string delim; - result << "["; - for (auto& v : vec) { - result << delim; - result << v; - if (delim.size() == 0) - delim = ", "; - } - result << "]"; - return result.str(); -} - -std::vector<ColorPlaneLayout> PlanesFromStrides( - const std::vector<int32_t> strides) { - std::vector<ColorPlaneLayout> planes(strides.size()); - for (size_t i = 0; i < strides.size(); i++) { - planes[i].stride = strides[i]; - } - return planes; -} - -} // namespace - -// static -size_t VideoFrameLayout::NumPlanes(VideoPixelFormat format) { - switch (format) { - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_BGRA: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - return 1; - case PIXEL_FORMAT_NV12: - case PIXEL_FORMAT_NV21: - case PIXEL_FORMAT_P016LE: - return 2; - case PIXEL_FORMAT_I420: - case PIXEL_FORMAT_YV12: - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV444P9: - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - return 3; - case PIXEL_FORMAT_I420A: - return 4; - case PIXEL_FORMAT_UNKNOWN: - // Note: PIXEL_FORMAT_UNKNOWN is used for end-of-stream frame. - // Set its NumPlanes() to zero to avoid NOTREACHED(). - return 0; - } - NOTREACHED() << "Unsupported video frame format: " << format; - return 0; -} - -// static -base::Optional<VideoFrameLayout> VideoFrameLayout::Create( - VideoPixelFormat format, - const Size& coded_size) { - return CreateWithStrides(format, coded_size, - std::vector<int32_t>(NumPlanes(format), 0)); -} - -// static -base::Optional<VideoFrameLayout> VideoFrameLayout::CreateWithStrides( - VideoPixelFormat format, - const Size& coded_size, - std::vector<int32_t> strides) { - return CreateWithPlanes(format, coded_size, PlanesFromStrides(strides)); -} - -// static -base::Optional<VideoFrameLayout> VideoFrameLayout::CreateWithPlanes( - VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - size_t buffer_addr_align, - uint64_t modifier) { - // NOTE: Even if format is UNKNOWN, it is valid if coded_sizes is not Empty(). - // TODO(crbug.com/896135): Return base::nullopt, - // if (format != PIXEL_FORMAT_UNKNOWN || !coded_sizes.IsEmpty()) - // TODO(crbug.com/896135): Return base::nullopt, - // if (planes.size() != NumPlanes(format)) - return VideoFrameLayout(format, coded_size, std::move(planes), - false /*is_multi_planar */, buffer_addr_align, - modifier); -} - -base::Optional<VideoFrameLayout> VideoFrameLayout::CreateMultiPlanar( - VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - size_t buffer_addr_align, - uint64_t modifier) { - // NOTE: Even if format is UNKNOWN, it is valid if coded_sizes is not Empty(). - // TODO(crbug.com/896135): Return base::nullopt, - // if (format != PIXEL_FORMAT_UNKNOWN || !coded_sizes.IsEmpty()) - // TODO(crbug.com/896135): Return base::nullopt, - // if (planes.size() != NumPlanes(format)) - return VideoFrameLayout(format, coded_size, std::move(planes), - true /*is_multi_planar */, buffer_addr_align, - modifier); -} - -VideoFrameLayout::VideoFrameLayout(VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - bool is_multi_planar, - size_t buffer_addr_align, - uint64_t modifier) - : format_(format), - coded_size_(coded_size), - planes_(std::move(planes)), - is_multi_planar_(is_multi_planar), - buffer_addr_align_(buffer_addr_align), - modifier_(modifier) {} - -VideoFrameLayout::~VideoFrameLayout() = default; -VideoFrameLayout::VideoFrameLayout(const VideoFrameLayout&) = default; -VideoFrameLayout::VideoFrameLayout(VideoFrameLayout&&) = default; -VideoFrameLayout& VideoFrameLayout::operator=(const VideoFrameLayout&) = - default; - -bool VideoFrameLayout::operator==(const VideoFrameLayout& rhs) const { - return format_ == rhs.format_ && coded_size_ == rhs.coded_size_ && - planes_ == rhs.planes_ && is_multi_planar_ == rhs.is_multi_planar_ && - buffer_addr_align_ == rhs.buffer_addr_align_ && - modifier_ == rhs.modifier_; -} - -bool VideoFrameLayout::operator!=(const VideoFrameLayout& rhs) const { - return !(*this == rhs); -} - -std::ostream& operator<<(std::ostream& ostream, - const VideoFrameLayout& layout) { - ostream << "VideoFrameLayout(format: " << layout.format() - << ", coded_size: " << layout.coded_size().ToString() - << ", planes (stride, offset, size): " - << VectorToString(layout.planes()) - << ", is_multi_planar: " << layout.is_multi_planar() - << ", buffer_addr_align: " << layout.buffer_addr_align() - << ", modifier: " << layout.modifier() << ")"; - return ostream; -} - -} // namespace media diff --git a/accel/video_frame_layout.h b/accel/video_frame_layout.h deleted file mode 100644 index 1713abd..0000000 --- a/accel/video_frame_layout.h +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 61df9350f6de -// Note: Added kNoModifier define to remove dependency on native pixmap handle. - -#ifndef VIDEO_FRAME_LAYOUT_H_ -#define VIDEO_FRAME_LAYOUT_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <ostream> -#include <string> -#include <utility> -#include <vector> - -#include "base/optional.h" -#include "color_plane_layout.h" -#include "size.h" -#include "video_pixel_format.h" - -namespace media { - -// Copied from native_pixmap_handle.h: -// This is the same value as DRM_FORMAT_MOD_INVALID, which is not a valid -// modifier. We use this to indicate that layout information -// (tiling/compression) if any will be communicated out of band. -static constexpr uint64_t kNoModifier = 0x00ffffffffffffffULL; - -// A class to describes how physical buffer is allocated for video frame. -// In stores format, coded size of the frame and size of physical buffers -// which can be used to allocate buffer(s) hardware expected. -// It also stores stride (bytes per line) and offset per color plane as Plane. -// stride is to calculate each color plane's size (note that a buffer may -// contains multiple color planes.) -// offset is to describe a start point of each plane from buffer's dmabuf fd. -// Note that it is copyable. -class VideoFrameLayout { - public: - // Default alignment for buffers. - // Note: This value is dependent on what's used by ffmpeg, do not change - // without inspecting av_frame_get_buffer() first. - static constexpr size_t kBufferAddressAlignment = 32; - - // Factory functions. - // |format| and |coded_size| must always be specified. - // |planes| info is also optional but useful to represent the layout of a - // video frame buffer correctly. When omitted, its information is all set - // to zero, so clients should be wary not to use this information. - // |buffer_addr_align| can be specified to request a specific buffer memory - // alignment. - // |modifier| is the additional information of |format|. It will become some - // value else than gfx::NativePixmapHandle::kNoModifier when the underlying - // buffer format is different from a standard |format| due to tiling. - // The returned base::Optional will be base::nullopt if the configured values - // are invalid. - - // Create a layout suitable for |format| at |coded_size|. The stride, offsets - // and size of all planes are set to 0, since that information cannot reliably - // be infered from the arguments. - static base::Optional<VideoFrameLayout> Create(VideoPixelFormat format, - const Size& coded_size); - - // Create a layout suitable for |format| at |coded_size|, with the |strides| - // for each plane specified. The offsets and size of all planes are set to 0. - // The size of |strides| must be equal to NumPlanes(|format|). - static base::Optional<VideoFrameLayout> CreateWithStrides( - VideoPixelFormat format, - const Size& coded_size, - std::vector<int32_t> strides); - - // Create a layout suitable for |format| at |coded_size|, with the |planes| - // fully provided. - // The size of |planes| must be equal to NumPlanes(|format|). - static base::Optional<VideoFrameLayout> CreateWithPlanes( - VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - size_t buffer_addr_align = kBufferAddressAlignment, - uint64_t modifier = kNoModifier); - - // This constructor should be called for situations where the frames using - // this format are backed by multiple physical buffers, instead of having each - // plane at different offsets of the same buffer. Currently only used by V4L2. - static base::Optional<VideoFrameLayout> CreateMultiPlanar( - VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - size_t buffer_addr_align = kBufferAddressAlignment, - uint64_t modifier = kNoModifier); - - VideoFrameLayout() = delete; - VideoFrameLayout(const VideoFrameLayout&); - VideoFrameLayout(VideoFrameLayout&&); - VideoFrameLayout& operator=(const VideoFrameLayout&); - ~VideoFrameLayout(); - - static size_t NumPlanes(VideoPixelFormat format); - - VideoPixelFormat format() const { return format_; } - const Size& coded_size() const { return coded_size_; } - - // Returns number of planes. Note that num_planes >= num_buffers. - size_t num_planes() const { return planes_.size(); } - - const std::vector<ColorPlaneLayout>& planes() const { return planes_; } - - bool operator==(const VideoFrameLayout& rhs) const; - bool operator!=(const VideoFrameLayout& rhs) const; - - // Return true when a format uses multiple backing buffers to store its - // planes. - bool is_multi_planar() const { return is_multi_planar_; } - // Returns the required memory alignment for buffers. - size_t buffer_addr_align() const { return buffer_addr_align_; } - // Return the modifier of buffers. - uint64_t modifier() const { return modifier_; } - - private: - VideoFrameLayout(VideoPixelFormat format, - const Size& coded_size, - std::vector<ColorPlaneLayout> planes, - bool is_multi_planar, - size_t buffer_addr_align, - uint64_t modifier); - - VideoPixelFormat format_; - - // Width and height of the video frame in pixels. This must include pixel - // data for the whole image; i.e. for YUV formats with subsampled chroma - // planes, in the case that the visible portion of the image does not line up - // on a sample boundary, |coded_size_| must be rounded up appropriately and - // the pixel data provided for the odd pixels. - Size coded_size_; - - // Layout property for each color planes, e.g. stride and buffer offset. - std::vector<ColorPlaneLayout> planes_; - - // Set to true when a format uses multiple backing buffers to store its - // planes. Used by code for V4L2 API at the moment. - bool is_multi_planar_; - - // Memory address alignment of the buffers. This is only relevant when - // allocating physical memory for the buffer, so it doesn't need to be - // serialized when frames are passed through Mojo. - size_t buffer_addr_align_; - - // Modifier of buffers. The modifier is retrieved from GBM library. This - // can be a different value from kNoModifier only if the VideoFrame is created - // by using NativePixmap. - uint64_t modifier_; -}; - -// Outputs VideoFrameLayout to stream. -std::ostream& operator<<(std::ostream& ostream, const VideoFrameLayout& layout); - -} // namespace media - -#endif // VIDEO_FRAME_LAYOUT_H_ diff --git a/accel/video_frame_metadata.cc b/accel/video_frame_metadata.cc deleted file mode 100644 index c6c6601..0000000 --- a/accel/video_frame_metadata.cc +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 19cd1babcaff -// Note: only functions related to END_OF_STREAM are ported. - -#include "video_frame_metadata.h" - -#include <stdint.h> -#include <utility> - -#include "base/logging.h" -#include "base/strings/string_number_conversions.h" - -namespace media { - -namespace { - -// Map enum key to internal std::string key used by base::DictionaryValue. -inline std::string ToInternalKey(VideoFrameMetadata::Key key) { - DCHECK_LT(key, VideoFrameMetadata::NUM_KEYS); - return base::NumberToString(static_cast<int>(key)); -} - -} // namespace - -VideoFrameMetadata::VideoFrameMetadata() = default; - -VideoFrameMetadata::~VideoFrameMetadata() = default; - -bool VideoFrameMetadata::HasKey(Key key) const { - return dictionary_.HasKey(ToInternalKey(key)); -} - -void VideoFrameMetadata::SetBoolean(Key key, bool value) { - dictionary_.SetKey(ToInternalKey(key), base::Value(value)); -} - -bool VideoFrameMetadata::GetBoolean(Key key, bool* value) const { - DCHECK(value); - return dictionary_.GetBooleanWithoutPathExpansion(ToInternalKey(key), value); -} - -bool VideoFrameMetadata::IsTrue(Key key) const { - bool value = false; - return GetBoolean(key, &value) && value; -} - -} // namespace media diff --git a/accel/video_frame_metadata.h b/accel/video_frame_metadata.h deleted file mode 100644 index a6ac6e5..0000000 --- a/accel/video_frame_metadata.h +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 514536171be3 -// Note: only functions related to END_OF_STREAM are ported. - -#ifndef VIDEO_FRAME_METADATA_H_ -#define VIDEO_FRAME_METADATA_H_ - -#include "base/values.h" - -namespace media { - -class VideoFrameMetadata { - public: - enum Key { - // Sources of VideoFrames use this marker to indicate that the associated - // VideoFrame can be overlayed, case in which its contents do not need to be - // further composited but displayed directly. Use Get/SetBoolean() for - // this Key. - ALLOW_OVERLAY, - - // Video capture begin/end timestamps. Consumers can use these values for - // dynamic optimizations, logging stats, etc. Use Get/SetTimeTicks() for - // these keys. - CAPTURE_BEGIN_TIME, - CAPTURE_END_TIME, - - // A counter that is increased by the producer of video frames each time - // it pushes out a new frame. By looking for gaps in this counter, clients - // can determine whether or not any frames have been dropped on the way from - // the producer between two consecutively received frames. Note that the - // counter may start at arbitrary values, so the absolute value of it has no - // meaning. - CAPTURE_COUNTER, - - // A base::ListValue containing 4 integers representing x, y, width, height - // of the rectangular region of the frame that has changed since the frame - // with the directly preceding CAPTURE_COUNTER. If that frame was not - // received, typically because it was dropped during transport from the - // producer, clients must assume that the entire frame has changed. - // The rectangle is relative to the full frame data, i.e. [0, 0, - // coded_size().width(), coded_size().height()]. It does not have to be - // fully contained within visible_rect(). - CAPTURE_UPDATE_RECT, - - // Indicates that this frame must be copied to a new texture before use, - // rather than being used directly. Specifically this is required for - // WebView because of limitations about sharing surface textures between GL - // contexts. - COPY_REQUIRED, - - // Indicates if the current frame is the End of its current Stream. Use - // Get/SetBoolean() for this Key. - END_OF_STREAM, - - // The estimated duration of this frame (i.e., the amount of time between - // the media timestamp of this frame and the next). Note that this is not - // the same information provided by FRAME_RATE as the FRAME_DURATION can - // vary unpredictably for every frame. Consumers can use this to optimize - // playback scheduling, make encoding quality decisions, and/or compute - // frame-level resource utilization stats. Use Get/SetTimeDelta() for this - // key. - FRAME_DURATION, - - // Represents either the fixed frame rate, or the maximum frame rate to - // expect from a variable-rate source. This value generally remains the - // same for all frames in the same session. Use Get/SetDouble() for this - // key. - FRAME_RATE, - - // This is a boolean that signals that the video capture engine detects - // interactive content. One possible optimization that this signal can help - // with is remote content: adjusting end-to-end latency down to help the - // user better coordinate their actions. - // - // Use Get/SetBoolean for this key. - INTERACTIVE_CONTENT, - - // This field represents the local time at which either: 1) the frame was - // generated, if it was done so locally; or 2) the targeted play-out time - // of the frame, if it was generated from a remote source. This value is NOT - // a high-resolution timestamp, and so it should not be used as a - // presentation time; but, instead, it should be used for buffering playback - // and for A/V synchronization purposes. - // Use Get/SetTimeTicks() for this key. - REFERENCE_TIME, - - // A feedback signal that indicates the fraction of the tolerable maximum - // amount of resources that were utilized to process this frame. A producer - // can check this value after-the-fact, usually via a VideoFrame destruction - // observer, to determine whether the consumer can handle more or less data - // volume, and achieve the right quality versus performance trade-off. - // - // Use Get/SetDouble() for this key. Values are interpreted as follows: - // Less than 0.0 is meaningless and should be ignored. 1.0 indicates a - // maximum sustainable utilization. Greater than 1.0 indicates the consumer - // is likely to stall or drop frames if the data volume is not reduced. - // - // Example: In a system that encodes and transmits video frames over the - // network, this value can be used to indicate whether sufficient CPU - // is available for encoding and/or sufficient bandwidth is available for - // transmission over the network. The maximum of the two utilization - // measurements would be used as feedback. - RESOURCE_UTILIZATION, - - // Sources of VideoFrames use this marker to indicate that an instance of - // VideoFrameExternalResources produced from the associated video frame - // should use read lock fences. - READ_LOCK_FENCES_ENABLED, - - // Indicates that the frame is rotated. - ROTATION, - - // Android only: if set, then this frame is not suitable for overlay, even - // if ALLOW_OVERLAY is set. However, it allows us to process the overlay - // to see if it would have been promoted, if it were backed by a SurfaceView - // instead. This lets us figure out when SurfaceViews are appropriate. - TEXTURE_OWNER, - - // Android only: if set, then this frame's resource would like to be - // notified about its promotability to an overlay. - WANTS_PROMOTION_HINT, - - // This video frame comes from protected content. - PROTECTED_VIDEO, - - // This video frame is protected by hardware. This option is valid only if - // PROTECTED_VIDEO is also set to true. - HW_PROTECTED, - - // An UnguessableToken that identifies VideoOverlayFactory that created - // this VideoFrame. It's used by Cast to help with video hole punch. - // Use Get/SetUnguessableToken() for this key. - OVERLAY_PLANE_ID, - - // Whether this frame was decoded in a power efficient way. - POWER_EFFICIENT, - - // CompositorFrameMetadata variables associated with this frame. Used for - // remote debugging. - // Use Get/SetDouble() for these keys. - // TODO(crbug.com/832220): Use a customized dictionary value instead of - // using these keys directly. - DEVICE_SCALE_FACTOR, - PAGE_SCALE_FACTOR, - ROOT_SCROLL_OFFSET_X, - ROOT_SCROLL_OFFSET_Y, - TOP_CONTROLS_VISIBLE_HEIGHT, - - // If present, this field represents the local time at which the VideoFrame - // was decoded from whichever format it was encoded in. Sometimes only - // DECODE_END_TIME will be present. Use Get/SetTimeTicks() for this key. - DECODE_BEGIN_TIME, - DECODE_END_TIME, - - // If present, this field represents the elapsed time from the submission of - // the encoded packet with the same PTS as this frame to the decoder until - // the decoded frame was ready for presentation. Stored as base::TimeDelta. - PROCESSING_TIME, - - // The RTP timestamp associated with this video frame. Stored as a double - // since base::DictionaryValue doesn't have a uint32_t type. - // - // https://w3c.github.io/webrtc-pc/#dom-rtcrtpcontributingsource - RTP_TIMESTAMP, - - NUM_KEYS - }; - - VideoFrameMetadata(); - ~VideoFrameMetadata(); - - bool HasKey(Key key) const; - - void Clear() { dictionary_.Clear(); } - - // Setters. Overwrites existing value, if present. - void SetBoolean(Key key, bool value); - - // Getters. Returns true if |key| is present, and its value has been set. - bool GetBoolean(Key key, bool* value) const WARN_UNUSED_RESULT; - - // Convenience method that returns true if |key| exists and is set to true. - bool IsTrue(Key key) const WARN_UNUSED_RESULT; - - private: - base::DictionaryValue dictionary_; - - DISALLOW_COPY_AND_ASSIGN(VideoFrameMetadata); -}; - -} // namespace media - -#endif // VIDEO_FRAME_METADATA_H_ diff --git a/accel/video_pixel_format.cc b/accel/video_pixel_format.cc deleted file mode 100644 index 20b8537..0000000 --- a/accel/video_pixel_format.cc +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 3b7ce92816e2 -// Note: only necessary functions are ported from video_types.cc - -#include "video_pixel_format.h" - -#include "base/logging.h" -#include "base/strings/stringprintf.h" - -namespace media { - -std::string VideoPixelFormatToString(VideoPixelFormat format) { - switch (format) { - case PIXEL_FORMAT_UNKNOWN: - return "PIXEL_FORMAT_UNKNOWN"; - case PIXEL_FORMAT_I420: - return "PIXEL_FORMAT_I420"; - case PIXEL_FORMAT_YV12: - return "PIXEL_FORMAT_YV12"; - case PIXEL_FORMAT_I422: - return "PIXEL_FORMAT_I422"; - case PIXEL_FORMAT_I420A: - return "PIXEL_FORMAT_I420A"; - case PIXEL_FORMAT_I444: - return "PIXEL_FORMAT_I444"; - case PIXEL_FORMAT_NV12: - return "PIXEL_FORMAT_NV12"; - case PIXEL_FORMAT_NV21: - return "PIXEL_FORMAT_NV21"; - case PIXEL_FORMAT_YUY2: - return "PIXEL_FORMAT_YUY2"; - case PIXEL_FORMAT_ARGB: - return "PIXEL_FORMAT_ARGB"; - case PIXEL_FORMAT_XRGB: - return "PIXEL_FORMAT_XRGB"; - case PIXEL_FORMAT_RGB24: - return "PIXEL_FORMAT_RGB24"; - case PIXEL_FORMAT_MJPEG: - return "PIXEL_FORMAT_MJPEG"; - case PIXEL_FORMAT_YUV420P9: - return "PIXEL_FORMAT_YUV420P9"; - case PIXEL_FORMAT_YUV420P10: - return "PIXEL_FORMAT_YUV420P10"; - case PIXEL_FORMAT_YUV422P9: - return "PIXEL_FORMAT_YUV422P9"; - case PIXEL_FORMAT_YUV422P10: - return "PIXEL_FORMAT_YUV422P10"; - case PIXEL_FORMAT_YUV444P9: - return "PIXEL_FORMAT_YUV444P9"; - case PIXEL_FORMAT_YUV444P10: - return "PIXEL_FORMAT_YUV444P10"; - case PIXEL_FORMAT_YUV420P12: - return "PIXEL_FORMAT_YUV420P12"; - case PIXEL_FORMAT_YUV422P12: - return "PIXEL_FORMAT_YUV422P12"; - case PIXEL_FORMAT_YUV444P12: - return "PIXEL_FORMAT_YUV444P12"; - case PIXEL_FORMAT_Y16: - return "PIXEL_FORMAT_Y16"; - case PIXEL_FORMAT_ABGR: - return "PIXEL_FORMAT_ABGR"; - case PIXEL_FORMAT_XBGR: - return "PIXEL_FORMAT_XBGR"; - case PIXEL_FORMAT_P016LE: - return "PIXEL_FORMAT_P016LE"; - case PIXEL_FORMAT_XR30: - return "PIXEL_FORMAT_XR30"; - case PIXEL_FORMAT_XB30: - return "PIXEL_FORMAT_XB30"; - case PIXEL_FORMAT_BGRA: - return "PIXEL_FORMAT_BGRA"; - } - NOTREACHED() << "Invalid VideoPixelFormat provided: " << format; - return ""; -} - -std::string FourccToString(uint32_t fourcc) { - std::string result = "0000"; - for (size_t i = 0; i < 4; ++i, fourcc >>= 8) { - const char c = static_cast<char>(fourcc & 0xFF); - if (c <= 0x1f || c >= 0x7f) - return base::StringPrintf("0x%x", fourcc); - result[i] = c; - } - return result; -} - -size_t BitDepth(VideoPixelFormat format) { - switch (format) { - case PIXEL_FORMAT_UNKNOWN: - NOTREACHED(); - FALLTHROUGH; - case PIXEL_FORMAT_I420: - case PIXEL_FORMAT_YV12: - case PIXEL_FORMAT_I422: - case PIXEL_FORMAT_I420A: - case PIXEL_FORMAT_I444: - case PIXEL_FORMAT_NV12: - case PIXEL_FORMAT_NV21: - case PIXEL_FORMAT_YUY2: - case PIXEL_FORMAT_ARGB: - case PIXEL_FORMAT_XRGB: - case PIXEL_FORMAT_RGB24: - case PIXEL_FORMAT_MJPEG: - case PIXEL_FORMAT_ABGR: - case PIXEL_FORMAT_XBGR: - case PIXEL_FORMAT_BGRA: - return 8; - case PIXEL_FORMAT_YUV420P9: - case PIXEL_FORMAT_YUV422P9: - case PIXEL_FORMAT_YUV444P9: - return 9; - case PIXEL_FORMAT_YUV420P10: - case PIXEL_FORMAT_YUV422P10: - case PIXEL_FORMAT_YUV444P10: - case PIXEL_FORMAT_XR30: - case PIXEL_FORMAT_XB30: - return 10; - case PIXEL_FORMAT_YUV420P12: - case PIXEL_FORMAT_YUV422P12: - case PIXEL_FORMAT_YUV444P12: - return 12; - case PIXEL_FORMAT_Y16: - case PIXEL_FORMAT_P016LE: - return 16; - } - NOTREACHED(); - return 0; -} - -} // namespace media - diff --git a/accel/video_pixel_format.h b/accel/video_pixel_format.h deleted file mode 100644 index 8d80731..0000000 --- a/accel/video_pixel_format.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 3b7ce92816e2 -// Note: only necessary functions are ported from video_types.h - -#ifndef VIDEO_PIXEL_FORMAT_H_ -#define VIDEO_PIXEL_FORMAT_H_ - -#include <string> - -namespace media { - -// Pixel formats roughly based on FOURCC labels, see: -// http://www.fourcc.org/rgb.php and http://www.fourcc.org/yuv.php -// Logged to UMA, so never reuse values. Leave gaps if necessary. -// Ordered as planar, semi-planar, YUV-packed, and RGB formats. -// When a VideoFrame is backed by native textures, VideoPixelFormat describes -// how those textures should be sampled and combined to produce the final -// pixels. -enum VideoPixelFormat { - PIXEL_FORMAT_UNKNOWN = 0, // Unknown or unspecified format value. - PIXEL_FORMAT_I420 = - 1, // 12bpp YUV planar 1x1 Y, 2x2 UV samples, a.k.a. YU12. - - // Note: Chrome does not actually support YVU compositing, so you probably - // don't actually want to use this. See http://crbug.com/784627. - PIXEL_FORMAT_YV12 = 2, // 12bpp YVU planar 1x1 Y, 2x2 VU samples. - - PIXEL_FORMAT_I422 = 3, // 16bpp YUV planar 1x1 Y, 2x1 UV samples. - PIXEL_FORMAT_I420A = 4, // 20bpp YUVA planar 1x1 Y, 2x2 UV, 1x1 A samples. - PIXEL_FORMAT_I444 = 5, // 24bpp YUV planar, no subsampling. - PIXEL_FORMAT_NV12 = - 6, // 12bpp with Y plane followed by a 2x2 interleaved UV plane. - PIXEL_FORMAT_NV21 = - 7, // 12bpp with Y plane followed by a 2x2 interleaved VU plane. - /* PIXEL_FORMAT_UYVY = 8, Deprecated */ - PIXEL_FORMAT_YUY2 = - 9, // 16bpp interleaved 1x1 Y, 2x1 U, 1x1 Y, 2x1 V samples. - PIXEL_FORMAT_ARGB = 10, // 32bpp BGRA (byte-order), 1 plane. - PIXEL_FORMAT_XRGB = 11, // 24bpp BGRX (byte-order), 1 plane. - PIXEL_FORMAT_RGB24 = 12, // 24bpp BGR (byte-order), 1 plane. - - /* PIXEL_FORMAT_RGB32 = 13, Deprecated */ - PIXEL_FORMAT_MJPEG = 14, // MJPEG compressed. - /* PIXEL_FORMAT_MT21 = 15, Deprecated */ - - // The P* in the formats below designates the number of bits per pixel - // component. I.e. P9 is 9-bits per pixel component, P10 is 10-bits per pixel - // component, etc. - PIXEL_FORMAT_YUV420P9 = 16, - PIXEL_FORMAT_YUV420P10 = 17, - PIXEL_FORMAT_YUV422P9 = 18, - PIXEL_FORMAT_YUV422P10 = 19, - PIXEL_FORMAT_YUV444P9 = 20, - PIXEL_FORMAT_YUV444P10 = 21, - PIXEL_FORMAT_YUV420P12 = 22, - PIXEL_FORMAT_YUV422P12 = 23, - PIXEL_FORMAT_YUV444P12 = 24, - - /* PIXEL_FORMAT_Y8 = 25, Deprecated */ - PIXEL_FORMAT_Y16 = 26, // single 16bpp plane. - - PIXEL_FORMAT_ABGR = 27, // 32bpp RGBA (byte-order), 1 plane. - PIXEL_FORMAT_XBGR = 28, // 24bpp RGBX (byte-order), 1 plane. - - PIXEL_FORMAT_P016LE = 29, // 24bpp NV12, 16 bits per channel - - PIXEL_FORMAT_XR30 = - 30, // 32bpp BGRX, 10 bits per channel, 2 bits ignored, 1 plane - PIXEL_FORMAT_XB30 = - 31, // 32bpp RGBX, 10 bits per channel, 2 bits ignored, 1 plane - - PIXEL_FORMAT_BGRA = 32, // 32bpp ARGB (byte-order), 1 plane. - - // Please update UMA histogram enumeration when adding new formats here. - PIXEL_FORMAT_MAX = - PIXEL_FORMAT_BGRA, // Must always be equal to largest entry logged. -}; - -// Returns the name of a Format as a string. -std::string VideoPixelFormatToString(VideoPixelFormat format); - -// Returns human readable fourcc string. -// If any of the four characters is non-printable, it outputs -// "0x<32-bit integer in hex>", e.g. FourccToString(0x66616b00) returns -// "0x66616b00". -std::string FourccToString(uint32_t fourcc); - -// Returns the number of significant bits per channel. -size_t BitDepth(VideoPixelFormat format); - -} // namespace media - -#endif // VIDEO_PIXEL_FORMAT_H_ diff --git a/accel/vp8_bool_decoder.cc b/accel/vp8_bool_decoder.cc deleted file mode 100644 index 68f06d0..0000000 --- a/accel/vp8_bool_decoder.cc +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Note: ported from Chromium commit head: 9b6f429 - -/* - * Copyright (c) 2010, The WebM Project authors. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * * Neither the name of Google, nor the WebM Project, nor the names - * of its contributors may be used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// This file is modified from the dboolhuff.{c,h} from the WebM's libvpx -// project. (http://www.webmproject.org/code) -// It is used to decode bits from a vp8 stream. - -#include <limits.h> - -#include <algorithm> - -#include "base/numerics/safe_conversions.h" -#include "vp8_bool_decoder.h" - -namespace media { - -#define VP8_BD_VALUE_BIT \ - static_cast<int>(sizeof(Vp8BoolDecoder::value_) * CHAR_BIT) - -static const int kDefaultProbability = 0x80; // 0x80 / 256 = 0.5 - -// This is meant to be a large, positive constant that can still be efficiently -// loaded as an immediate (on platforms like ARM, for example). Even relatively -// modest values like 100 would work fine. -#define VP8_LOTS_OF_BITS (0x40000000) - -// The number of leading zeros. -static const unsigned char kVp8Norm[256] = { - 0, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -Vp8BoolDecoder::Vp8BoolDecoder() - : user_buffer_(NULL), - user_buffer_end_(NULL), - value_(0), - count_(-8), - range_(255) { -} - -bool Vp8BoolDecoder::Initialize(const uint8_t* data, size_t size) { - if (data == NULL || size == 0) - return false; - user_buffer_start_ = data; - user_buffer_ = data; - user_buffer_end_ = data + size; - value_ = 0; - count_ = -8; - range_ = 255; - return true; -} - -void Vp8BoolDecoder::FillDecoder() { - DCHECK(user_buffer_ != NULL); - int shift = VP8_BD_VALUE_BIT - CHAR_BIT - (count_ + CHAR_BIT); - size_t bytes_left = user_buffer_end_ - user_buffer_; - size_t bits_left = bytes_left * CHAR_BIT; - int x = shift + CHAR_BIT - static_cast<int>(bits_left); - int loop_end = 0; - - if (x >= 0) { - count_ += VP8_LOTS_OF_BITS; - loop_end = x; - } - - if (x < 0 || bits_left) { - while (shift >= loop_end) { - count_ += CHAR_BIT; - value_ |= static_cast<size_t>(*user_buffer_) << shift; - ++user_buffer_; - shift -= CHAR_BIT; - } - } -} - -int Vp8BoolDecoder::ReadBit(int probability) { - int bit = 0; - size_t split = 1 + (((range_ - 1) * probability) >> 8); - if (count_ < 0) - FillDecoder(); - size_t bigsplit = static_cast<size_t>(split) << (VP8_BD_VALUE_BIT - 8); - - if (value_ >= bigsplit) { - range_ -= split; - value_ -= bigsplit; - bit = 1; - } else { - range_ = split; - } - - size_t shift = kVp8Norm[range_]; - range_ <<= shift; - value_ <<= shift; - count_ -= static_cast<int>(shift); - - DCHECK_EQ(1U, (range_ >> 7)); // In the range [128, 255]. - - return bit; -} - -bool Vp8BoolDecoder::ReadLiteral(size_t num_bits, int* out) { - DCHECK_LE(num_bits, sizeof(int) * CHAR_BIT); - *out = 0; - for (; num_bits > 0; --num_bits) - *out = (*out << 1) | ReadBit(kDefaultProbability); - return !OutOfBuffer(); -} - -bool Vp8BoolDecoder::ReadBool(bool* out, uint8_t probability) { - *out = !!ReadBit(probability); - return !OutOfBuffer(); -} - -bool Vp8BoolDecoder::ReadBool(bool* out) { - return ReadBool(out, kDefaultProbability); -} - -bool Vp8BoolDecoder::ReadLiteralWithSign(size_t num_bits, int* out) { - ReadLiteral(num_bits, out); - // Read sign. - if (ReadBit(kDefaultProbability)) - *out = -*out; - return !OutOfBuffer(); -} - -size_t Vp8BoolDecoder::BitOffset() { - int bit_count = count_ + 8; - if (bit_count > VP8_BD_VALUE_BIT) - // Capped at 0 to ignore buffer underrun. - bit_count = std::max(0, bit_count - VP8_LOTS_OF_BITS); - return (user_buffer_ - user_buffer_start_) * 8 - bit_count; -} - -uint8_t Vp8BoolDecoder::GetRange() { - return base::checked_cast<uint8_t>(range_); -} - -uint8_t Vp8BoolDecoder::GetBottom() { - if (count_ < 0) - FillDecoder(); - return static_cast<uint8_t>(value_ >> (VP8_BD_VALUE_BIT - 8)); -} - -inline bool Vp8BoolDecoder::OutOfBuffer() { - // Check if we have reached the end of the buffer. - // - // Variable |count_| stores the number of bits in the |value_| buffer, minus - // 8. The top byte is part of the algorithm and the remainder is buffered to - // be shifted into it. So, if |count_| == 8, the top 16 bits of |value_| are - // occupied, 8 for the algorithm and 8 in the buffer. - // - // When reading a byte from the user's buffer, |count_| is filled with 8 and - // one byte is filled into the |value_| buffer. When we reach the end of the - // data, |count_| is additionally filled with VP8_LOTS_OF_BITS. So when - // |count_| == VP8_LOTS_OF_BITS - 1, the user's data has been exhausted. - return (count_ > VP8_BD_VALUE_BIT) && (count_ < VP8_LOTS_OF_BITS); -} - -} // namespace media diff --git a/accel/vp8_bool_decoder.h b/accel/vp8_bool_decoder.h deleted file mode 100644 index 4b8e3a5..0000000 --- a/accel/vp8_bool_decoder.h +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Note: ported from Chromium commit head: 1323b9c - -/* - * Copyright (c) 2010, The WebM Project authors. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * * Neither the name of Google, nor the WebM Project, nor the names - * of its contributors may be used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// This file is modified from the dboolhuff.{c,h} from the WebM's libvpx -// project. (http://www.webmproject.org/code) -// It is used to decode bits from a vp8 stream. - -#ifndef VP8_BOOL_DECODER_H_ -#define VP8_BOOL_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> -#include <sys/types.h> - -#include "base/logging.h" -#include "base/macros.h" - -namespace media { - -// A class to decode the VP8's boolean entropy coded stream. It's a variant of -// arithmetic coding. See RFC 6386 - Chapter 7. Boolean Entropy Decoder. -class Vp8BoolDecoder { - public: - Vp8BoolDecoder(); - - // Initializes the decoder to start decoding |data|, |size| being size - // of |data| in bytes. Returns false if |data| is NULL or empty. - bool Initialize(const uint8_t* data, size_t size); - - // Reads a boolean from the coded stream. Returns false if it has reached the - // end of |data| and failed to read the boolean. The probability of |out| to - // be true is |probability| / 256, e.g., when |probability| is 0x80, the - // chance is 1/2 (i.e., 0x80 / 256). - bool ReadBool(bool* out, uint8_t probability); - - // Reads a boolean from the coded stream with the default probability 1/2. - // Returns false if it has reached the end of |data| and failed to read the - // boolean. - bool ReadBool(bool* out); - - // Reads a "literal", that is, a "num_bits"-wide unsigned value whose bits - // come high- to low-order, with each bit encoded at probability 1/2. - // Returns false if it has reached the end of |data| and failed to read the - // literal. - bool ReadLiteral(size_t num_bits, int* out); - - // Reads a literal with sign from the coded stream. This is similar to - // the ReadListeral(), it first read a "num_bits"-wide unsigned value, and - // then read an extra bit as the sign of the literal. Returns false if it has - // reached the end of |data| and failed to read the literal or the sign. - // This is different from the "read_signed_literal(d, n)" defined in RFC 6386. - bool ReadLiteralWithSign(size_t num_bits, int* out); - - // The following methods are used to get the internal states of the decoder. - - // Returns the bit offset to the current top bit of the coded stream. It is - // also the number of bits that have been written in the corresponding - // encoding state. More specifically, we have the following constraint: - // w + (bottom * S) <= v < w + (bottom + range) * S, - // where "w" is for the bits already written, - // "v" is for the possible values of the coded number. - // "S" is the scale for the current bit position, - // i.e., S = pow(2, -(n + 8)), where "n" is the bit number of "w". - // BitOffset() returns the bit count of "w", i.e., "n". - size_t BitOffset(); - - // Gets the "bottom" of the current coded value. See BitOffset() for - // more details. - uint8_t GetBottom(); - - // Gets the "range" of the current coded value. See BitOffset() for - // more details. - uint8_t GetRange(); - - private: - // Reads the next bit from the coded stream. The probability of the bit to - // be one is |probability| / 256. - int ReadBit(int probability); - - // Fills more bits from |user_buffer_| to |value_|. We shall keep at least 8 - // bits of the current |user_buffer_| in |value_|. - void FillDecoder(); - - // Returns true iff we have ran out of bits. - bool OutOfBuffer(); - - const uint8_t* user_buffer_; - const uint8_t* user_buffer_start_; - const uint8_t* user_buffer_end_; - size_t value_; - int count_; - size_t range_; - - DISALLOW_COPY_AND_ASSIGN(Vp8BoolDecoder); -}; - -} // namespace media - -#endif // VP8_BOOL_DECODER_H_ diff --git a/accel/vp8_decoder.cc b/accel/vp8_decoder.cc deleted file mode 100644 index cd2d58b..0000000 --- a/accel/vp8_decoder.cc +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 7441087 - -#include "vp8_decoder.h" - -namespace media { - -VP8Decoder::VP8Accelerator::VP8Accelerator() {} - -VP8Decoder::VP8Accelerator::~VP8Accelerator() {} - -VP8Decoder::VP8Decoder(VP8Accelerator* accelerator) - : state_(kNeedStreamMetadata), - curr_frame_start_(nullptr), - frame_size_(0), - accelerator_(accelerator) { - DCHECK(accelerator_); -} - -VP8Decoder::~VP8Decoder() {} - -bool VP8Decoder::Flush() { - DVLOG(2) << "Decoder flush"; - Reset(); - return true; -} - -void VP8Decoder::SetStream(const uint8_t* ptr, size_t size) { - DCHECK(ptr); - DCHECK(size); - - curr_frame_start_ = ptr; - frame_size_ = size; - DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size; -} - -void VP8Decoder::Reset() { - curr_pic_ = nullptr; - curr_frame_hdr_ = nullptr; - curr_frame_start_ = nullptr; - frame_size_ = 0; - - last_frame_ = nullptr; - golden_frame_ = nullptr; - alt_frame_ = nullptr; - - if (state_ == kDecoding) - state_ = kAfterReset; -} - -VP8Decoder::DecodeResult VP8Decoder::Decode() { - if (!curr_frame_start_ || frame_size_ == 0) - return kRanOutOfStreamData; - - if (!curr_frame_hdr_) { - curr_frame_hdr_.reset(new Vp8FrameHeader()); - if (!parser_.ParseFrame(curr_frame_start_, frame_size_, - curr_frame_hdr_.get())) { - DVLOG(1) << "Error during decode"; - state_ = kError; - return kDecodeError; - } - } - - if (curr_frame_hdr_->IsKeyframe()) { - Size new_pic_size(curr_frame_hdr_->width, curr_frame_hdr_->height); - if (new_pic_size.IsEmpty()) - return kDecodeError; - - if (new_pic_size != pic_size_) { - DVLOG(2) << "New resolution: " << new_pic_size.ToString(); - pic_size_ = new_pic_size; - - DCHECK(!curr_pic_); - last_frame_ = nullptr; - golden_frame_ = nullptr; - alt_frame_ = nullptr; - - return kAllocateNewSurfaces; - } - - state_ = kDecoding; - } else { - if (state_ != kDecoding) { - // Need a resume point. - curr_frame_hdr_.reset(); - return kRanOutOfStreamData; - } - } - - curr_pic_ = accelerator_->CreateVP8Picture(); - if (!curr_pic_) - return kRanOutOfSurfaces; - - curr_pic_->visible_rect = Rect(pic_size_); - if (!DecodeAndOutputCurrentFrame()) - return kDecodeError; - - return kRanOutOfStreamData; -} - -void VP8Decoder::RefreshReferenceFrames() { - if (curr_frame_hdr_->IsKeyframe()) { - last_frame_ = curr_pic_; - golden_frame_ = curr_pic_; - alt_frame_ = curr_pic_; - return; - } - - // Save current golden since we overwrite it here, - // but may have to use it to update alt below. - scoped_refptr<VP8Picture> curr_golden = golden_frame_; - - if (curr_frame_hdr_->refresh_golden_frame) { - golden_frame_ = curr_pic_; - } else { - switch (curr_frame_hdr_->copy_buffer_to_golden) { - case Vp8FrameHeader::COPY_LAST_TO_GOLDEN: - DCHECK(last_frame_); - golden_frame_ = last_frame_; - break; - - case Vp8FrameHeader::COPY_ALT_TO_GOLDEN: - DCHECK(alt_frame_); - golden_frame_ = alt_frame_; - break; - } - } - - if (curr_frame_hdr_->refresh_alternate_frame) { - alt_frame_ = curr_pic_; - } else { - switch (curr_frame_hdr_->copy_buffer_to_alternate) { - case Vp8FrameHeader::COPY_LAST_TO_ALT: - DCHECK(last_frame_); - alt_frame_ = last_frame_; - break; - - case Vp8FrameHeader::COPY_GOLDEN_TO_ALT: - DCHECK(curr_golden); - alt_frame_ = curr_golden; - break; - } - } - - if (curr_frame_hdr_->refresh_last) - last_frame_ = curr_pic_; -} - -bool VP8Decoder::DecodeAndOutputCurrentFrame() { - DCHECK(!pic_size_.IsEmpty()); - DCHECK(curr_pic_); - DCHECK(curr_frame_hdr_); - - if (curr_frame_hdr_->IsKeyframe()) { - horizontal_scale_ = curr_frame_hdr_->horizontal_scale; - vertical_scale_ = curr_frame_hdr_->vertical_scale; - } else { - // Populate fields from decoder state instead. - curr_frame_hdr_->width = pic_size_.width(); - curr_frame_hdr_->height = pic_size_.height(); - curr_frame_hdr_->horizontal_scale = horizontal_scale_; - curr_frame_hdr_->vertical_scale = vertical_scale_; - } - - if (!accelerator_->SubmitDecode(curr_pic_, curr_frame_hdr_.get(), last_frame_, - golden_frame_, alt_frame_)) - return false; - - if (curr_frame_hdr_->show_frame) - if (!accelerator_->OutputPicture(curr_pic_)) - return false; - - RefreshReferenceFrames(); - - curr_pic_ = nullptr; - curr_frame_hdr_ = nullptr; - curr_frame_start_ = nullptr; - frame_size_ = 0; - return true; -} - -Size VP8Decoder::GetPicSize() const { - return pic_size_; -} - -size_t VP8Decoder::GetRequiredNumOfPictures() const { - const size_t kVP8NumFramesActive = 4; - // TODO(johnylin): see if we could get rid of kMaxVideoFrames. - const size_t kMaxVideoFrames = 4; - const size_t kPicsInPipeline = kMaxVideoFrames + 2; - return kVP8NumFramesActive + kPicsInPipeline; -} - -} // namespace media diff --git a/accel/vp8_decoder.h b/accel/vp8_decoder.h deleted file mode 100644 index 58211f6..0000000 --- a/accel/vp8_decoder.h +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 60f9667 - -#ifndef VP8_DECODER_H_ -#define VP8_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "accelerated_video_decoder.h" -#include "size.h" -#include "vp8_parser.h" -#include "vp8_picture.h" - -namespace media { - -// Clients of this class are expected to pass raw VP8 stream and are expected -// to provide an implementation of VP8Accelerator for offloading final steps -// of the decoding process. -// -// This class must be created, called and destroyed on a single thread, and -// does nothing internally on any other thread. -class VP8Decoder : public AcceleratedVideoDecoder { - public: - class VP8Accelerator { - public: - VP8Accelerator(); - virtual ~VP8Accelerator(); - - // Create a new VP8Picture that the decoder client can use for decoding - // and pass back to this accelerator for decoding or reference. - // When the picture is no longer needed by decoder, it will just drop - // its reference to it, and it may do so at any time. - // Note that this may return nullptr if accelerator is not able to provide - // any new pictures at given time. The decoder is expected to handle - // this situation as normal and return from Decode() with kRanOutOfSurfaces. - virtual scoped_refptr<VP8Picture> CreateVP8Picture() = 0; - - // Submit decode for |pic|, taking as arguments |frame_hdr| with parsed - // VP8 frame header information for current frame, and using |last_frame|, - // |golden_frame| and |alt_frame| as references, as per VP8 specification. - // Note that this runs the decode in hardware. - // Return true if successful. - virtual bool SubmitDecode(const scoped_refptr<VP8Picture>& pic, - const Vp8FrameHeader* frame_hdr, - const scoped_refptr<VP8Picture>& last_frame, - const scoped_refptr<VP8Picture>& golden_frame, - const scoped_refptr<VP8Picture>& alt_frame) = 0; - - // Schedule output (display) of |pic|. Note that returning from this - // method does not mean that |pic| has already been outputted (displayed), - // but guarantees that all pictures will be outputted in the same order - // as this method was called for them. Decoder may drop its reference - // to |pic| after calling this method. - // Return true if successful. - virtual bool OutputPicture(const scoped_refptr<VP8Picture>& pic) = 0; - - private: - DISALLOW_COPY_AND_ASSIGN(VP8Accelerator); - }; - - VP8Decoder(VP8Accelerator* accelerator); - ~VP8Decoder() override; - - // AcceleratedVideoDecoder implementation. - bool Flush() override WARN_UNUSED_RESULT; - void Reset() override; - void SetStream(const uint8_t* ptr, size_t size) override; - DecodeResult Decode() override WARN_UNUSED_RESULT; - Size GetPicSize() const override; - size_t GetRequiredNumOfPictures() const override; - - private: - bool DecodeAndOutputCurrentFrame(); - void RefreshReferenceFrames(); - - enum State { - kNeedStreamMetadata, // After initialization, need a keyframe. - kDecoding, // Ready to decode from any point. - kAfterReset, // After Reset(), need a resume point. - kError, // Error in decode, can't continue. - }; - - State state_; - - Vp8Parser parser_; - - std::unique_ptr<Vp8FrameHeader> curr_frame_hdr_; - scoped_refptr<VP8Picture> curr_pic_; - scoped_refptr<VP8Picture> last_frame_; - scoped_refptr<VP8Picture> golden_frame_; - scoped_refptr<VP8Picture> alt_frame_; - - const uint8_t* curr_frame_start_; - size_t frame_size_; - - Size pic_size_; - int horizontal_scale_; - int vertical_scale_; - - VP8Accelerator* accelerator_; - - DISALLOW_COPY_AND_ASSIGN(VP8Decoder); -}; - -} // namespace media - -#endif // VP8_DECODER_H_ diff --git a/accel/vp8_parser.cc b/accel/vp8_parser.cc deleted file mode 100644 index 5367545..0000000 --- a/accel/vp8_parser.cc +++ /dev/null @@ -1,877 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of a VP8 raw stream parser, -// as defined in RFC 6386. -// Note: ported from Chromium commit head: 2de6929 - - -#include "base/logging.h" -#include "vp8_parser.h" - -namespace media { - -#define ERROR_RETURN(what) \ - do { \ - DVLOG(1) << "Error while trying to read " #what; \ - return false; \ - } while (0) - -#define BD_READ_BOOL_OR_RETURN(out) \ - do { \ - if (!bd_.ReadBool(out)) \ - ERROR_RETURN(out); \ - } while (0) - -#define BD_READ_BOOL_WITH_PROB_OR_RETURN(out, prob) \ - do { \ - if (!bd_.ReadBool(out, prob)) \ - ERROR_RETURN(out); \ - } while (0) - -#define BD_READ_UNSIGNED_OR_RETURN(num_bits, out) \ - do { \ - int _out; \ - if (!bd_.ReadLiteral(num_bits, &_out)) \ - ERROR_RETURN(out); \ - *out = _out; \ - } while (0) - -#define BD_READ_SIGNED_OR_RETURN(num_bits, out) \ - do { \ - int _out; \ - if (!bd_.ReadLiteralWithSign(num_bits, &_out)) \ - ERROR_RETURN(out); \ - *out = _out; \ - } while (0) - -Vp8FrameHeader::Vp8FrameHeader() { - memset(this, 0, sizeof(*this)); -} - -Vp8Parser::Vp8Parser() : stream_(nullptr), bytes_left_(0) { -} - -Vp8Parser::~Vp8Parser() = default; - -bool Vp8Parser::ParseFrame(const uint8_t* ptr, - size_t frame_size, - Vp8FrameHeader* fhdr) { - stream_ = ptr; - bytes_left_ = frame_size; - - memset(fhdr, 0, sizeof(*fhdr)); - fhdr->data = stream_; - fhdr->frame_size = bytes_left_; - - if (!ParseFrameTag(fhdr)) - return false; - - fhdr->first_part_offset = stream_ - fhdr->data; - - if (!ParseFrameHeader(fhdr)) - return false; - - if (!ParsePartitions(fhdr)) - return false; - - DVLOG(4) << "Frame parsed, start: " << static_cast<const void*>(ptr) - << ", size: " << frame_size - << ", offsets: to first_part=" << fhdr->first_part_offset - << ", to macroblock data (in bits)=" << fhdr->macroblock_bit_offset; - - return true; -} - -static inline uint32_t GetBitsAt(uint32_t data, size_t shift, size_t num_bits) { - return ((data >> shift) & ((1 << num_bits) - 1)); -} - -bool Vp8Parser::ParseFrameTag(Vp8FrameHeader* fhdr) { - const size_t kFrameTagSize = 3; - if (bytes_left_ < kFrameTagSize) - return false; - - uint32_t frame_tag = (stream_[2] << 16) | (stream_[1] << 8) | stream_[0]; - fhdr->key_frame = - static_cast<Vp8FrameHeader::FrameType>(GetBitsAt(frame_tag, 0, 1)); - fhdr->version = GetBitsAt(frame_tag, 1, 2); - fhdr->is_experimental = !!GetBitsAt(frame_tag, 3, 1); - fhdr->show_frame =!!GetBitsAt(frame_tag, 4, 1); - fhdr->first_part_size = GetBitsAt(frame_tag, 5, 19); - - stream_ += kFrameTagSize; - bytes_left_ -= kFrameTagSize; - - if (fhdr->IsKeyframe()) { - const size_t kKeyframeTagSize = 7; - if (bytes_left_ < kKeyframeTagSize) - return false; - - static const uint8_t kVp8StartCode[] = {0x9d, 0x01, 0x2a}; - if (memcmp(stream_, kVp8StartCode, sizeof(kVp8StartCode)) != 0) - return false; - - stream_ += sizeof(kVp8StartCode); - bytes_left_ -= sizeof(kVp8StartCode); - - uint16_t data = (stream_[1] << 8) | stream_[0]; - fhdr->width = data & 0x3fff; - fhdr->horizontal_scale = data >> 14; - - data = (stream_[3] << 8) | stream_[2]; - fhdr->height = data & 0x3fff; - fhdr->vertical_scale = data >> 14; - - stream_ += 4; - bytes_left_ -= 4; - } - - return true; -} - -bool Vp8Parser::ParseFrameHeader(Vp8FrameHeader* fhdr) { - if (!bd_.Initialize(stream_, bytes_left_)) - return false; - - bool keyframe = fhdr->IsKeyframe(); - if (keyframe) { - unsigned int data; - BD_READ_UNSIGNED_OR_RETURN(1, &data); // color_space - BD_READ_UNSIGNED_OR_RETURN(1, &data); // clamping_type - } - - if (!ParseSegmentationHeader(keyframe)) - return false; - - fhdr->segmentation_hdr = curr_segmentation_hdr_; - - if (!ParseLoopFilterHeader(keyframe)) - return false; - - fhdr->loopfilter_hdr = curr_loopfilter_hdr_; - - int log2_nbr_of_dct_partitions; - BD_READ_UNSIGNED_OR_RETURN(2, &log2_nbr_of_dct_partitions); - fhdr->num_of_dct_partitions = static_cast<size_t>(1) - << log2_nbr_of_dct_partitions; - - if (!ParseQuantizationHeader(&fhdr->quantization_hdr)) - return false; - - if (keyframe) { - BD_READ_BOOL_OR_RETURN(&fhdr->refresh_entropy_probs); - } else { - BD_READ_BOOL_OR_RETURN(&fhdr->refresh_golden_frame); - BD_READ_BOOL_OR_RETURN(&fhdr->refresh_alternate_frame); - - int refresh_mode; - if (!fhdr->refresh_golden_frame) { - BD_READ_UNSIGNED_OR_RETURN(2, &refresh_mode); - fhdr->copy_buffer_to_golden = - static_cast<Vp8FrameHeader::GoldenRefreshMode>(refresh_mode); - } - - if (!fhdr->refresh_alternate_frame) { - BD_READ_UNSIGNED_OR_RETURN(2, &refresh_mode); - fhdr->copy_buffer_to_alternate = - static_cast<Vp8FrameHeader::AltRefreshMode>(refresh_mode); - } - - BD_READ_UNSIGNED_OR_RETURN(1, &fhdr->sign_bias_golden); - BD_READ_UNSIGNED_OR_RETURN(1, &fhdr->sign_bias_alternate); - BD_READ_BOOL_OR_RETURN(&fhdr->refresh_entropy_probs); - BD_READ_BOOL_OR_RETURN(&fhdr->refresh_last); - } - - if (keyframe) - ResetProbs(); - - fhdr->entropy_hdr = curr_entropy_hdr_; - - if (!ParseTokenProbs(&fhdr->entropy_hdr, fhdr->refresh_entropy_probs)) - return false; - - BD_READ_BOOL_OR_RETURN(&fhdr->mb_no_skip_coeff); - if (fhdr->mb_no_skip_coeff) - BD_READ_UNSIGNED_OR_RETURN(8, &fhdr->prob_skip_false); - - if (!keyframe) { - BD_READ_UNSIGNED_OR_RETURN(8, &fhdr->prob_intra); - BD_READ_UNSIGNED_OR_RETURN(8, &fhdr->prob_last); - BD_READ_UNSIGNED_OR_RETURN(8, &fhdr->prob_gf); - } - - if (!ParseIntraProbs(&fhdr->entropy_hdr, fhdr->refresh_entropy_probs, - keyframe)) - return false; - - if (!keyframe) { - if (!ParseMVProbs(&fhdr->entropy_hdr, fhdr->refresh_entropy_probs)) - return false; - } - - fhdr->macroblock_bit_offset = bd_.BitOffset(); - fhdr->bool_dec_range = bd_.GetRange(); - fhdr->bool_dec_value = bd_.GetBottom(); - fhdr->bool_dec_count = 7 - (bd_.BitOffset() + 7) % 8; - - return true; -} - -bool Vp8Parser::ParseSegmentationHeader(bool keyframe) { - Vp8SegmentationHeader* shdr = &curr_segmentation_hdr_; - - if (keyframe) - memset(shdr, 0, sizeof(*shdr)); - - BD_READ_BOOL_OR_RETURN(&shdr->segmentation_enabled); - if (!shdr->segmentation_enabled) - return true; - - BD_READ_BOOL_OR_RETURN(&shdr->update_mb_segmentation_map); - BD_READ_BOOL_OR_RETURN(&shdr->update_segment_feature_data); - if (shdr->update_segment_feature_data) { - int mode; - BD_READ_UNSIGNED_OR_RETURN(1, &mode); - shdr->segment_feature_mode = - static_cast<Vp8SegmentationHeader::SegmentFeatureMode>(mode); - - for (size_t i = 0; i < kMaxMBSegments; ++i) { - bool quantizer_update; - BD_READ_BOOL_OR_RETURN(&quantizer_update); - if (quantizer_update) - BD_READ_SIGNED_OR_RETURN(7, &shdr->quantizer_update_value[i]); - else - shdr->quantizer_update_value[i] = 0; - } - - for (size_t i = 0; i < kMaxMBSegments; ++i) { - bool loop_filter_update; - BD_READ_BOOL_OR_RETURN(&loop_filter_update); - if (loop_filter_update) - BD_READ_SIGNED_OR_RETURN(6, &shdr->lf_update_value[i]); - else - shdr->lf_update_value[i] = 0; - } - } - - if (shdr->update_mb_segmentation_map) { - for (size_t i = 0; i < kNumMBFeatureTreeProbs; ++i) { - bool segment_prob_update; - BD_READ_BOOL_OR_RETURN(&segment_prob_update); - if (segment_prob_update) - BD_READ_UNSIGNED_OR_RETURN(8, &shdr->segment_prob[i]); - else - shdr->segment_prob[i] = Vp8SegmentationHeader::kDefaultSegmentProb; - } - } - - return true; -} - -bool Vp8Parser::ParseLoopFilterHeader(bool keyframe) { - Vp8LoopFilterHeader* lfhdr = &curr_loopfilter_hdr_; - - if (keyframe) - memset(lfhdr, 0, sizeof(*lfhdr)); - - int type; - BD_READ_UNSIGNED_OR_RETURN(1, &type); - lfhdr->type = static_cast<Vp8LoopFilterHeader::Type>(type); - BD_READ_UNSIGNED_OR_RETURN(6, &lfhdr->level); - BD_READ_UNSIGNED_OR_RETURN(3, &lfhdr->sharpness_level); - BD_READ_BOOL_OR_RETURN(&lfhdr->loop_filter_adj_enable); - - if (lfhdr->loop_filter_adj_enable) { - BD_READ_BOOL_OR_RETURN(&lfhdr->mode_ref_lf_delta_update); - if (lfhdr->mode_ref_lf_delta_update) { - for (size_t i = 0; i < kNumBlockContexts; ++i) { - bool ref_frame_delta_update_flag; - BD_READ_BOOL_OR_RETURN(&ref_frame_delta_update_flag); - if (ref_frame_delta_update_flag) - BD_READ_SIGNED_OR_RETURN(6, &lfhdr->ref_frame_delta[i]); - } - - for (size_t i = 0; i < kNumBlockContexts; ++i) { - bool mb_mode_delta_update_flag; - BD_READ_BOOL_OR_RETURN(&mb_mode_delta_update_flag); - if (mb_mode_delta_update_flag) - BD_READ_SIGNED_OR_RETURN(6, &lfhdr->mb_mode_delta[i]); - } - } - } - - return true; -} - -bool Vp8Parser::ParseQuantizationHeader(Vp8QuantizationHeader* qhdr) { - // If any of the delta values is not present, the delta should be zero. - memset(qhdr, 0, sizeof(*qhdr)); - - BD_READ_UNSIGNED_OR_RETURN(7, &qhdr->y_ac_qi); - - bool delta_present; - - BD_READ_BOOL_OR_RETURN(&delta_present); - if (delta_present) - BD_READ_SIGNED_OR_RETURN(4, &qhdr->y_dc_delta); - - BD_READ_BOOL_OR_RETURN(&delta_present); - if (delta_present) - BD_READ_SIGNED_OR_RETURN(4, &qhdr->y2_dc_delta); - - BD_READ_BOOL_OR_RETURN(&delta_present); - if (delta_present) - BD_READ_SIGNED_OR_RETURN(4, &qhdr->y2_ac_delta); - - BD_READ_BOOL_OR_RETURN(&delta_present); - if (delta_present) - BD_READ_SIGNED_OR_RETURN(4, &qhdr->uv_dc_delta); - - BD_READ_BOOL_OR_RETURN(&delta_present); - if (delta_present) - BD_READ_SIGNED_OR_RETURN(4, &qhdr->uv_ac_delta); - - return true; -} - -// See spec for details on these values. -const uint8_t kCoeffUpdateProbs[kNumBlockTypes][kNumCoeffBands] - [kNumPrevCoeffContexts][kNumEntropyNodes] = { - { - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {176, 246, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {223, 241, 252, 255, 255, 255, 255, 255, 255, 255, 255}, - {249, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 244, 252, 255, 255, 255, 255, 255, 255, 255, 255}, - {234, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 246, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {239, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {251, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {251, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 253, 255, 254, 255, 255, 255, 255, 255, 255}, - {250, 255, 254, 255, 254, 255, 255, 255, 255, 255, 255}, - {254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - }, - { - { - {217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {225, 252, 241, 253, 255, 255, 254, 255, 255, 255, 255}, - {234, 250, 241, 250, 253, 255, 253, 254, 255, 255, 255}, - }, - { - {255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {223, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {238, 253, 254, 254, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {249, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {247, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {252, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - }, - { - { - {186, 251, 250, 255, 255, 255, 255, 255, 255, 255, 255}, - {234, 251, 244, 254, 255, 255, 255, 255, 255, 255, 255}, - {251, 251, 243, 253, 254, 255, 254, 255, 255, 255, 255}, - }, - { - {255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {236, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {251, 253, 253, 254, 254, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - }, - { - { - {248, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {250, 254, 252, 254, 255, 255, 255, 255, 255, 255, 255}, - {248, 254, 249, 253, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {246, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {252, 254, 251, 254, 254, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 254, 252, 255, 255, 255, 255, 255, 255, 255, 255}, - {248, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {253, 255, 254, 254, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {245, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {253, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 251, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {252, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 252, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {249, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 253, 255, 255, 255, 255, 255, 255, 255, 255}, - {250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - { - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, - }, - }, -}; - -const uint8_t kKeyframeYModeProbs[kNumYModeProbs] = {145, 156, 163, 128}; -const uint8_t kKeyframeUVModeProbs[kNumUVModeProbs] = {142, 114, 183}; - -const uint8_t kDefaultYModeProbs[kNumYModeProbs] = {112, 86, 140, 37}; -const uint8_t kDefaultUVModeProbs[kNumUVModeProbs] = {162, 101, 204}; - -const uint8_t kDefaultCoeffProbs[kNumBlockTypes][kNumCoeffBands] - [kNumPrevCoeffContexts][kNumEntropyNodes] = { - { - { - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - }, - { - {253, 136, 254, 255, 228, 219, 128, 128, 128, 128, 128}, - {189, 129, 242, 255, 227, 213, 255, 219, 128, 128, 128}, - {106, 126, 227, 252, 214, 209, 255, 255, 128, 128, 128}, - }, - { - { 1, 98, 248, 255, 236, 226, 255, 255, 128, 128, 128}, - {181, 133, 238, 254, 221, 234, 255, 154, 128, 128, 128}, - { 78, 134, 202, 247, 198, 180, 255, 219, 128, 128, 128}, - }, - { - { 1, 185, 249, 255, 243, 255, 128, 128, 128, 128, 128}, - {184, 150, 247, 255, 236, 224, 128, 128, 128, 128, 128}, - { 77, 110, 216, 255, 236, 230, 128, 128, 128, 128, 128}, - }, - { - { 1, 101, 251, 255, 241, 255, 128, 128, 128, 128, 128}, - {170, 139, 241, 252, 236, 209, 255, 255, 128, 128, 128}, - { 37, 116, 196, 243, 228, 255, 255, 255, 128, 128, 128}, - }, - { - { 1, 204, 254, 255, 245, 255, 128, 128, 128, 128, 128}, - {207, 160, 250, 255, 238, 128, 128, 128, 128, 128, 128}, - {102, 103, 231, 255, 211, 171, 128, 128, 128, 128, 128}, - }, - { - { 1, 152, 252, 255, 240, 255, 128, 128, 128, 128, 128}, - {177, 135, 243, 255, 234, 225, 128, 128, 128, 128, 128}, - { 80, 129, 211, 255, 194, 224, 128, 128, 128, 128, 128}, - }, - { - { 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {246, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {255, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - } - }, - { - { - {198, 35, 237, 223, 193, 187, 162, 160, 145, 155, 62}, - {131, 45, 198, 221, 172, 176, 220, 157, 252, 221, 1}, - { 68, 47, 146, 208, 149, 167, 221, 162, 255, 223, 128}, - }, - { - { 1, 149, 241, 255, 221, 224, 255, 255, 128, 128, 128}, - {184, 141, 234, 253, 222, 220, 255, 199, 128, 128, 128}, - { 81, 99, 181, 242, 176, 190, 249, 202, 255, 255, 128}, - }, - { - { 1, 129, 232, 253, 214, 197, 242, 196, 255, 255, 128}, - { 99, 121, 210, 250, 201, 198, 255, 202, 128, 128, 128}, - { 23, 91, 163, 242, 170, 187, 247, 210, 255, 255, 128}, - }, - { - { 1, 200, 246, 255, 234, 255, 128, 128, 128, 128, 128}, - {109, 178, 241, 255, 231, 245, 255, 255, 128, 128, 128}, - { 44, 130, 201, 253, 205, 192, 255, 255, 128, 128, 128}, - }, - { - { 1, 132, 239, 251, 219, 209, 255, 165, 128, 128, 128}, - { 94, 136, 225, 251, 218, 190, 255, 255, 128, 128, 128}, - { 22, 100, 174, 245, 186, 161, 255, 199, 128, 128, 128}, - }, - { - { 1, 182, 249, 255, 232, 235, 128, 128, 128, 128, 128}, - {124, 143, 241, 255, 227, 234, 128, 128, 128, 128, 128}, - { 35, 77, 181, 251, 193, 211, 255, 205, 128, 128, 128}, - }, - { - { 1, 157, 247, 255, 236, 231, 255, 255, 128, 128, 128}, - {121, 141, 235, 255, 225, 227, 255, 255, 128, 128, 128}, - { 45, 99, 188, 251, 195, 217, 255, 224, 128, 128, 128}, - }, - { - { 1, 1, 251, 255, 213, 255, 128, 128, 128, 128, 128}, - {203, 1, 248, 255, 255, 128, 128, 128, 128, 128, 128}, - {137, 1, 177, 255, 224, 255, 128, 128, 128, 128, 128}, - } - }, - { - { - {253, 9, 248, 251, 207, 208, 255, 192, 128, 128, 128}, - {175, 13, 224, 243, 193, 185, 249, 198, 255, 255, 128}, - { 73, 17, 171, 221, 161, 179, 236, 167, 255, 234, 128}, - }, - { - { 1, 95, 247, 253, 212, 183, 255, 255, 128, 128, 128}, - {239, 90, 244, 250, 211, 209, 255, 255, 128, 128, 128}, - {155, 77, 195, 248, 188, 195, 255, 255, 128, 128, 128}, - }, - { - { 1, 24, 239, 251, 218, 219, 255, 205, 128, 128, 128}, - {201, 51, 219, 255, 196, 186, 128, 128, 128, 128, 128}, - { 69, 46, 190, 239, 201, 218, 255, 228, 128, 128, 128}, - }, - { - { 1, 191, 251, 255, 255, 128, 128, 128, 128, 128, 128}, - {223, 165, 249, 255, 213, 255, 128, 128, 128, 128, 128}, - {141, 124, 248, 255, 255, 128, 128, 128, 128, 128, 128}, - }, - { - { 1, 16, 248, 255, 255, 128, 128, 128, 128, 128, 128}, - {190, 36, 230, 255, 236, 255, 128, 128, 128, 128, 128}, - {149, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - }, - { - { 1, 226, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {247, 192, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {240, 128, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - }, - { - { 1, 134, 252, 255, 255, 128, 128, 128, 128, 128, 128}, - {213, 62, 250, 255, 255, 128, 128, 128, 128, 128, 128}, - { 55, 93, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - }, - { - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, - } - }, - { - { - {202, 24, 213, 235, 186, 191, 220, 160, 240, 175, 255}, - {126, 38, 182, 232, 169, 184, 228, 174, 255, 187, 128}, - { 61, 46, 138, 219, 151, 178, 240, 170, 255, 216, 128}, - }, - { - { 1, 112, 230, 250, 199, 191, 247, 159, 255, 255, 128}, - {166, 109, 228, 252, 211, 215, 255, 174, 128, 128, 128}, - { 39, 77, 162, 232, 172, 180, 245, 178, 255, 255, 128}, - }, - { - { 1, 52, 220, 246, 198, 199, 249, 220, 255, 255, 128}, - {124, 74, 191, 243, 183, 193, 250, 221, 255, 255, 128}, - { 24, 71, 130, 219, 154, 170, 243, 182, 255, 255, 128}, - }, - { - { 1, 182, 225, 249, 219, 240, 255, 224, 128, 128, 128}, - {149, 150, 226, 252, 216, 205, 255, 171, 128, 128, 128}, - { 28, 108, 170, 242, 183, 194, 254, 223, 255, 255, 128} - }, - { - { 1, 81, 230, 252, 204, 203, 255, 192, 128, 128, 128}, - {123, 102, 209, 247, 188, 196, 255, 233, 128, 128, 128}, - { 20, 95, 153, 243, 164, 173, 255, 203, 128, 128, 128}, - }, - { - { 1, 222, 248, 255, 216, 213, 128, 128, 128, 128, 128}, - {168, 175, 246, 252, 235, 205, 255, 255, 128, 128, 128}, - { 47, 116, 215, 255, 211, 212, 255, 255, 128, 128, 128}, - }, - { - { 1, 121, 236, 253, 212, 214, 255, 255, 128, 128, 128}, - {141, 84, 213, 252, 201, 202, 255, 219, 128, 128, 128}, - { 42, 80, 160, 240, 162, 185, 255, 205, 128, 128, 128}, - }, - { - { 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {244, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - {238, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128}, - }, - }, -}; - -const uint8_t kMVUpdateProbs[kNumMVContexts][kNumMVProbs] = -{ - { - 237, 246, 253, 253, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 250, 250, 252, 254, 254, - }, - { - 231, 243, 245, 253, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 251, 251, 254, 254, 254, - }, -}; - -const uint8_t kDefaultMVProbs[kNumMVContexts][kNumMVProbs] = { - { - 162, 128, 225, 146, 172, 147, 214, 39, 156, - 128, 129, 132, 75, 145, 178, 206, 239, 254, 254, - }, - { - 164, 128, 204, 170, 119, 235, 140, 230, 228, - 128, 130, 130, 74, 148, 180, 203, 236, 254, 254, - }, -}; - -void Vp8Parser::ResetProbs() { - static_assert( - sizeof(curr_entropy_hdr_.coeff_probs) == sizeof(kDefaultCoeffProbs), - "coeff_probs_arrays_must_be_of_correct_size"); - memcpy(curr_entropy_hdr_.coeff_probs, kDefaultCoeffProbs, - sizeof(curr_entropy_hdr_.coeff_probs)); - - static_assert(sizeof(curr_entropy_hdr_.mv_probs) == sizeof(kDefaultMVProbs), - "mv_probs_arrays_must_be_of_correct_size"); - memcpy(curr_entropy_hdr_.mv_probs, kDefaultMVProbs, - sizeof(curr_entropy_hdr_.mv_probs)); - - static_assert( - sizeof(curr_entropy_hdr_.y_mode_probs) == sizeof(kDefaultYModeProbs), - "y_probs_arrays_must_be_of_correct_size"); - memcpy(curr_entropy_hdr_.y_mode_probs, kDefaultYModeProbs, - sizeof(curr_entropy_hdr_.y_mode_probs)); - - static_assert( - sizeof(curr_entropy_hdr_.uv_mode_probs) == sizeof(kDefaultUVModeProbs), - "uv_probs_arrays_must_be_of_correct_size"); - memcpy(curr_entropy_hdr_.uv_mode_probs, kDefaultUVModeProbs, - sizeof(curr_entropy_hdr_.uv_mode_probs)); -} - -bool Vp8Parser::ParseTokenProbs(Vp8EntropyHeader* ehdr, - bool update_curr_probs) { - for (size_t i = 0; i < kNumBlockTypes; ++i) { - for (size_t j = 0; j < kNumCoeffBands; ++j) { - for (size_t k = 0; k < kNumPrevCoeffContexts; ++k) { - for (size_t l = 0; l < kNumEntropyNodes; ++l) { - bool coeff_prob_update_flag; - BD_READ_BOOL_WITH_PROB_OR_RETURN(&coeff_prob_update_flag, - kCoeffUpdateProbs[i][j][k][l]); - if (coeff_prob_update_flag) - BD_READ_UNSIGNED_OR_RETURN(8, &ehdr->coeff_probs[i][j][k][l]); - } - } - } - } - - if (update_curr_probs) { - memcpy(curr_entropy_hdr_.coeff_probs, ehdr->coeff_probs, - sizeof(curr_entropy_hdr_.coeff_probs)); - } - - return true; -} - -bool Vp8Parser::ParseIntraProbs(Vp8EntropyHeader* ehdr, - bool update_curr_probs, - bool keyframe) { - if (keyframe) { - static_assert( - sizeof(ehdr->y_mode_probs) == sizeof(kKeyframeYModeProbs), - "y_probs_arrays_must_be_of_correct_size"); - memcpy(ehdr->y_mode_probs, kKeyframeYModeProbs, - sizeof(ehdr->y_mode_probs)); - - static_assert( - sizeof(ehdr->uv_mode_probs) == sizeof(kKeyframeUVModeProbs), - "uv_probs_arrays_must_be_of_correct_size"); - memcpy(ehdr->uv_mode_probs, kKeyframeUVModeProbs, - sizeof(ehdr->uv_mode_probs)); - } else { - bool intra_16x16_prob_update_flag; - BD_READ_BOOL_OR_RETURN(&intra_16x16_prob_update_flag); - if (intra_16x16_prob_update_flag) { - for (size_t i = 0; i < kNumYModeProbs; ++i) - BD_READ_UNSIGNED_OR_RETURN(8, &ehdr->y_mode_probs[i]); - - if (update_curr_probs) { - memcpy(curr_entropy_hdr_.y_mode_probs, ehdr->y_mode_probs, - sizeof(curr_entropy_hdr_.y_mode_probs)); - } - } - - bool intra_chroma_prob_update_flag; - BD_READ_BOOL_OR_RETURN(&intra_chroma_prob_update_flag); - if (intra_chroma_prob_update_flag) { - for (size_t i = 0; i < kNumUVModeProbs; ++i) - BD_READ_UNSIGNED_OR_RETURN(8, &ehdr->uv_mode_probs[i]); - - if (update_curr_probs) { - memcpy(curr_entropy_hdr_.uv_mode_probs, ehdr->uv_mode_probs, - sizeof(curr_entropy_hdr_.uv_mode_probs)); - } - } - } - - return true; -} - -bool Vp8Parser::ParseMVProbs(Vp8EntropyHeader* ehdr, bool update_curr_probs) { - for (size_t mv_ctx = 0; mv_ctx < kNumMVContexts; ++mv_ctx) { - for (size_t p = 0; p < kNumMVProbs; ++p) { - bool mv_prob_update_flag; - BD_READ_BOOL_WITH_PROB_OR_RETURN(&mv_prob_update_flag, - kMVUpdateProbs[mv_ctx][p]); - if (mv_prob_update_flag) { - uint8_t prob; - BD_READ_UNSIGNED_OR_RETURN(7, &prob); - ehdr->mv_probs[mv_ctx][p] = prob ? (prob << 1) : 1; - } - } - } - - if (update_curr_probs) { - memcpy(curr_entropy_hdr_.mv_probs, ehdr->mv_probs, - sizeof(curr_entropy_hdr_.mv_probs)); - } - - return true; -} - -bool Vp8Parser::ParsePartitions(Vp8FrameHeader* fhdr) { - CHECK_GE(fhdr->num_of_dct_partitions, 1u); - CHECK_LE(fhdr->num_of_dct_partitions, kMaxDCTPartitions); - - // DCT partitions start after the first partition and partition size values - // that follow it. There are num_of_dct_partitions - 1 sizes stored in the - // stream after the first partition, each 3 bytes long. The size of last - // DCT partition is not stored in the stream, but is instead calculated by - // taking the remainder of the frame size after the penultimate DCT partition. - size_t first_dct_pos = fhdr->first_part_offset + fhdr->first_part_size + - (fhdr->num_of_dct_partitions - 1) * 3; - - // Make sure we have enough data for the first partition and partition sizes. - if (fhdr->frame_size < first_dct_pos) - return false; - - // Total size of all DCT partitions. - size_t bytes_left = fhdr->frame_size - first_dct_pos; - - // Position ourselves at the beginning of partition size values. - const uint8_t* ptr = - fhdr->data + fhdr->first_part_offset + fhdr->first_part_size; - - // Read sizes from the stream (if present). - for (size_t i = 0; i < fhdr->num_of_dct_partitions - 1; ++i) { - fhdr->dct_partition_sizes[i] = (ptr[2] << 16) | (ptr[1] << 8) | ptr[0]; - - // Make sure we have enough data in the stream for ith partition and - // subtract its size from total. - if (bytes_left < fhdr->dct_partition_sizes[i]) - return false; - - bytes_left -= fhdr->dct_partition_sizes[i]; - - // Move to the position of the next partition size value. - ptr += 3; - } - - // The remainder of the data belongs to the last DCT partition. - fhdr->dct_partition_sizes[fhdr->num_of_dct_partitions - 1] = bytes_left; - - DVLOG(4) << "Control part size: " << fhdr->first_part_size; - for (size_t i = 0; i < fhdr->num_of_dct_partitions; ++i) - DVLOG(4) << "DCT part " << i << " size: " << fhdr->dct_partition_sizes[i]; - - return true; -} - -} // namespace media diff --git a/accel/vp8_parser.h b/accel/vp8_parser.h deleted file mode 100644 index c75e6cc..0000000 --- a/accel/vp8_parser.h +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of a VP8 raw stream parser, -// as defined in RFC 6386. -// Note: ported from Chromium commit head: 1323b9c - -#ifndef VP8_PARSER_H_ -#define VP8_PARSER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include "base/macros.h" -#include "vp8_bool_decoder.h" - -namespace media { - -// See spec for definitions of values/fields. -const size_t kMaxMBSegments = 4; -const size_t kNumMBFeatureTreeProbs = 3; - -// Member of Vp8FrameHeader and will be 0-initialized -// in Vp8FrameHeader's constructor. -struct Vp8SegmentationHeader { - enum SegmentFeatureMode { FEATURE_MODE_DELTA = 0, FEATURE_MODE_ABSOLUTE = 1 }; - - bool segmentation_enabled; - bool update_mb_segmentation_map; - bool update_segment_feature_data; - SegmentFeatureMode segment_feature_mode; - - int8_t quantizer_update_value[kMaxMBSegments]; - int8_t lf_update_value[kMaxMBSegments]; - static const int kDefaultSegmentProb = 255; - uint8_t segment_prob[kNumMBFeatureTreeProbs]; -}; - -const size_t kNumBlockContexts = 4; - -// Member of Vp8FrameHeader and will be 0-initialized -// in Vp8FrameHeader's constructor. -struct Vp8LoopFilterHeader { - enum Type { LOOP_FILTER_TYPE_NORMAL = 0, LOOP_FILTER_TYPE_SIMPLE = 1 }; - Type type; - uint8_t level; - uint8_t sharpness_level; - bool loop_filter_adj_enable; - bool mode_ref_lf_delta_update; - - int8_t ref_frame_delta[kNumBlockContexts]; - int8_t mb_mode_delta[kNumBlockContexts]; -}; - -// Member of Vp8FrameHeader and will be 0-initialized -// in Vp8FrameHeader's constructor. -struct Vp8QuantizationHeader { - uint8_t y_ac_qi; - int8_t y_dc_delta; - int8_t y2_dc_delta; - int8_t y2_ac_delta; - int8_t uv_dc_delta; - int8_t uv_ac_delta; -}; - -const size_t kNumBlockTypes = 4; -const size_t kNumCoeffBands = 8; -const size_t kNumPrevCoeffContexts = 3; -const size_t kNumEntropyNodes = 11; - -const size_t kNumMVContexts = 2; -const size_t kNumMVProbs = 19; - -const size_t kNumYModeProbs = 4; -const size_t kNumUVModeProbs = 3; - -// Member of Vp8FrameHeader and will be 0-initialized -// in Vp8FrameHeader's constructor. -struct Vp8EntropyHeader { - uint8_t coeff_probs[kNumBlockTypes][kNumCoeffBands][kNumPrevCoeffContexts] - [kNumEntropyNodes]; - - uint8_t y_mode_probs[kNumYModeProbs]; - uint8_t uv_mode_probs[kNumUVModeProbs]; - - uint8_t mv_probs[kNumMVContexts][kNumMVProbs]; -}; - -const size_t kMaxDCTPartitions = 8; - -struct Vp8FrameHeader { - Vp8FrameHeader(); - - enum FrameType { KEYFRAME = 0, INTERFRAME = 1 }; - bool IsKeyframe() const { return key_frame == KEYFRAME; } - - enum GoldenRefreshMode { - COPY_LAST_TO_GOLDEN = 1, - COPY_ALT_TO_GOLDEN = 2, - }; - - enum AltRefreshMode { - COPY_LAST_TO_ALT = 1, - COPY_GOLDEN_TO_ALT = 2, - }; - - FrameType key_frame; - uint8_t version; - bool is_experimental; - bool show_frame; - size_t first_part_size; - - uint16_t width; - uint8_t horizontal_scale; - uint16_t height; - uint8_t vertical_scale; - - Vp8SegmentationHeader segmentation_hdr; - Vp8LoopFilterHeader loopfilter_hdr; - Vp8QuantizationHeader quantization_hdr; - - size_t num_of_dct_partitions; - - Vp8EntropyHeader entropy_hdr; - - bool refresh_entropy_probs; - bool refresh_golden_frame; - bool refresh_alternate_frame; - GoldenRefreshMode copy_buffer_to_golden; - AltRefreshMode copy_buffer_to_alternate; - uint8_t sign_bias_golden; - uint8_t sign_bias_alternate; - bool refresh_last; - - bool mb_no_skip_coeff; - uint8_t prob_skip_false; - uint8_t prob_intra; - uint8_t prob_last; - uint8_t prob_gf; - - const uint8_t* data; - size_t frame_size; - - size_t dct_partition_sizes[kMaxDCTPartitions]; - // Offset in bytes from data. - off_t first_part_offset; - // Offset in bits from first_part_offset. - off_t macroblock_bit_offset; - - // Bool decoder state - uint8_t bool_dec_range; - uint8_t bool_dec_value; - uint8_t bool_dec_count; -}; - -// A parser for raw VP8 streams as specified in RFC 6386. -class Vp8Parser { - public: - Vp8Parser(); - ~Vp8Parser(); - - // Try to parse exactly one VP8 frame starting at |ptr| and of size |size|, - // filling the parsed data in |fhdr|. Return true on success. - // Size has to be exactly the size of the frame and coming from the caller, - // who needs to acquire it from elsewhere (normally from a container). - bool ParseFrame(const uint8_t* ptr, size_t size, Vp8FrameHeader* fhdr); - - private: - bool ParseFrameTag(Vp8FrameHeader* fhdr); - bool ParseFrameHeader(Vp8FrameHeader* fhdr); - - bool ParseSegmentationHeader(bool keyframe); - bool ParseLoopFilterHeader(bool keyframe); - bool ParseQuantizationHeader(Vp8QuantizationHeader* qhdr); - bool ParseTokenProbs(Vp8EntropyHeader* ehdr, bool update_curr_probs); - bool ParseIntraProbs(Vp8EntropyHeader* ehdr, - bool update_curr_probs, - bool keyframe); - bool ParseMVProbs(Vp8EntropyHeader* ehdr, bool update_curr_probs); - bool ParsePartitions(Vp8FrameHeader* fhdr); - void ResetProbs(); - - // These persist across calls to ParseFrame() and may be used and/or updated - // for subsequent frames if the stream instructs us to do so. - Vp8SegmentationHeader curr_segmentation_hdr_; - Vp8LoopFilterHeader curr_loopfilter_hdr_; - Vp8EntropyHeader curr_entropy_hdr_; - - const uint8_t* stream_; - size_t bytes_left_; - Vp8BoolDecoder bd_; - - DISALLOW_COPY_AND_ASSIGN(Vp8Parser); -}; - -} // namespace media - -#endif // VP8_PARSER_H_ diff --git a/accel/vp8_picture.cc b/accel/vp8_picture.cc deleted file mode 100644 index b9030ce..0000000 --- a/accel/vp8_picture.cc +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 6e70beb - -#include "vp8_picture.h" - -namespace media { - -VP8Picture::VP8Picture() {} - -VP8Picture::~VP8Picture() {} - -V4L2VP8Picture* VP8Picture::AsV4L2VP8Picture() { - return nullptr; -} - -} // namespace media diff --git a/accel/vp8_picture.h b/accel/vp8_picture.h deleted file mode 100644 index bd04ec7..0000000 --- a/accel/vp8_picture.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 70340ce - -#ifndef VP8_PICTURE_H_ -#define VP8_PICTURE_H_ - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "rect.h" - -namespace media { - -class V4L2VP8Picture; - -class VP8Picture : public base::RefCountedThreadSafe<VP8Picture> { - public: - VP8Picture(); - - virtual V4L2VP8Picture* AsV4L2VP8Picture(); - - // The visible size of picture. - Rect visible_rect; - - protected: - friend class base::RefCountedThreadSafe<VP8Picture>; - virtual ~VP8Picture(); - - DISALLOW_COPY_AND_ASSIGN(VP8Picture); -}; - -} // namespace media - -#endif // VP8_PICTURE_H_ diff --git a/accel/vp9_bool_decoder.cc b/accel/vp9_bool_decoder.cc deleted file mode 100644 index 1d2b6f4..0000000 --- a/accel/vp9_bool_decoder.cc +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 1323b9c - -#include "vp9_bool_decoder.h" - -#include <algorithm> - -#include "base/logging.h" -#include "bit_reader.h" - -namespace media { - -namespace { - -// This is an optimization lookup table for the loop in spec 9.2.2. -// while BoolRange <= 128: -// read 1 bit -// BoolRange *= 2 -// This table indicates how many iterations to run for a given BoolRange. So -// the loop could be reduced to -// read (kCountToShiftTo128[BoolRange]) bits -const int kCountToShiftTo128[256] = { - 0, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; -} // namespace - -Vp9BoolDecoder::Vp9BoolDecoder() = default; - -Vp9BoolDecoder::~Vp9BoolDecoder() = default; - -// 9.2.1 Initialization process for Boolean decoder -bool Vp9BoolDecoder::Initialize(const uint8_t* data, size_t size) { - DCHECK(data); - if (size < 1) { - DVLOG(1) << "input size of bool decoder shall be at least 1"; - valid_ = false; - return false; - } - - reader_.reset(new BitReader(data, size)); - valid_ = true; - - bool_value_ = 0; - count_to_fill_ = 8; - bool_range_ = 255; - if (ReadLiteral(1) != 0) { - DVLOG(1) << "marker bit should be 0"; - valid_ = false; - return false; - } - return true; -} - -// Fill at least |count_to_fill_| bits and prefill remain bits of |bool_value_| -// if data is enough. -bool Vp9BoolDecoder::Fill() { - DCHECK_GE(count_to_fill_, 0); - - int bits_left = reader_->bits_available(); - if (bits_left < count_to_fill_) { - valid_ = false; - DVLOG(1) << "Vp9BoolDecoder reads beyond the end of stream"; - return false; - } - - DCHECK_LE(count_to_fill_, kBoolSize); - int max_bits_to_read = kBigBoolBitSize - kBoolSize + count_to_fill_; - int bits_to_read = std::min(max_bits_to_read, bits_left); - - BigBool data; - reader_->ReadBits(bits_to_read, &data); - bool_value_ |= data << (max_bits_to_read - bits_to_read); - count_to_fill_ -= bits_to_read; - - return true; -} - -// 9.2.2 Boolean decoding process -bool Vp9BoolDecoder::ReadBool(int prob) { - DCHECK(reader_); - - if (count_to_fill_ > 0) { - if (!Fill()) - return false; - } - - unsigned int split = (bool_range_ * prob + (256 - prob)) >> kBoolSize; - BigBool big_split = static_cast<BigBool>(split) - << (kBigBoolBitSize - kBoolSize); - - bool bit; - if (bool_value_ < big_split) { - bool_range_ = split; - bit = false; - } else { - bool_range_ -= split; - bool_value_ -= big_split; - bit = true; - } - - // Need to fill |count| bits next time in order to make |bool_range_| >= - // 128. - DCHECK_LT(bool_range_, arraysize(kCountToShiftTo128)); - DCHECK_GT(bool_range_, 0u); - int count = kCountToShiftTo128[bool_range_]; - bool_range_ <<= count; - bool_value_ <<= count; - count_to_fill_ += count; - - return bit; -} - -// 9.2.4 Parsing process for read_literal -uint8_t Vp9BoolDecoder::ReadLiteral(int bits) { - DCHECK_LT(static_cast<size_t>(bits), sizeof(uint8_t) * 8); - DCHECK(reader_); - - uint8_t x = 0; - for (int i = 0; i < bits; i++) - x = 2 * x + ReadBool(128); - - return x; -} - -bool Vp9BoolDecoder::ConsumePaddingBits() { - DCHECK(reader_); - - if (count_to_fill_ > reader_->bits_available()) { - // 9.2.2 Boolean decoding process - // Although we actually don't used the value, spec says the bitstream - // should have enough bits to fill bool range, this should never happen. - DVLOG(2) << "not enough bits in bitstream to fill bool range"; - return false; - } - - if (bool_value_ != 0) { - DVLOG(1) << "prefilled padding bits are not zero"; - return false; - } - while (reader_->bits_available() > 0) { - int data; - int size_to_read = - std::min(reader_->bits_available(), static_cast<int>(sizeof(data) * 8)); - reader_->ReadBits(size_to_read, &data); - if (data != 0) { - DVLOG(1) << "padding bits are not zero"; - return false; - } - } - return true; -} - -} // namespace media diff --git a/accel/vp9_bool_decoder.h b/accel/vp9_bool_decoder.h deleted file mode 100644 index 50c386f..0000000 --- a/accel/vp9_bool_decoder.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: e5a9a62 - -#ifndef VP9_BOOL_DECODER_H_ -#define VP9_BOOL_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> - -#include "base/macros.h" - -namespace media { - -class BitReader; - -class Vp9BoolDecoder { - public: - Vp9BoolDecoder(); - ~Vp9BoolDecoder(); - - // |data| is the input buffer with |size| bytes. - // Returns true if read first marker bit successfully. - bool Initialize(const uint8_t* data, size_t size); - - // Returns true if none of the reads since the last Initialize() call has - // gone beyond the end of available data. - bool IsValid() const { return valid_; } - - // Reads one bit. B(p). - // If the read goes beyond the end of buffer, the return value is undefined. - bool ReadBool(int prob); - - // Reads a literal. L(n). - // If the read goes beyond the end of buffer, the return value is undefined. - uint8_t ReadLiteral(int bits); - - // Consumes padding bits up to end of data. Returns true if no - // padding bits or they are all zero. - bool ConsumePaddingBits(); - - private: - // The highest 8 bits of BigBool is actual "bool value". The remain bits - // are optimization of prefill buffer. - using BigBool = size_t; - // The size of "bool value" used for boolean decoding defined in spec. - const int kBoolSize = 8; - const int kBigBoolBitSize = sizeof(BigBool) * 8; - - bool Fill(); - - std::unique_ptr<BitReader> reader_; - - // Indicates if none of the reads since the last Initialize() call has gone - // beyond the end of available data. - bool valid_ = true; - - BigBool bool_value_ = 0; - - // Need to fill at least |count_to_fill_| bits. Negative value means extra - // bits pre-filled. - int count_to_fill_ = 0; - unsigned int bool_range_ = 0; - - DISALLOW_COPY_AND_ASSIGN(Vp9BoolDecoder); -}; - -} // namespace media - -#endif // VP9_BOOL_DECODER_H_ diff --git a/accel/vp9_compressed_header_parser.cc b/accel/vp9_compressed_header_parser.cc deleted file mode 100644 index 524472f..0000000 --- a/accel/vp9_compressed_header_parser.cc +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "vp9_compressed_header_parser.h" - -#include "base/logging.h" - -namespace media { - -namespace { - -// 6.3.6 Inv recenter noneg syntax, inv_recenter_nonneg(). -int InvRecenterNonneg(int v, int m) { - DCHECK_LE(m, kVp9MaxProb / 2); - if (v > 2 * m) - return v; - - if (v & 1) - return m - ((v + 1) >> 1); - return m + (v >> 1); -} - -// 6.3.5 Inv remap prob syntax, inv_remap_prob(). -Vp9Prob InvRemapProb(uint8_t delta_prob, uint8_t prob) { - static uint8_t inv_map_table[kVp9MaxProb] = { - 7, 20, 33, 46, 59, 72, 85, 98, 111, 124, 137, 150, 163, 176, - 189, 202, 215, 228, 241, 254, 1, 2, 3, 4, 5, 6, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, - 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 123, 125, 126, 127, 128, 129, 130, - 131, 132, 133, 134, 135, 136, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 162, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, - 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 204, 205, 206, - 207, 208, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, - 237, 238, 239, 240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 253}; - uint8_t m = prob; - uint8_t v = delta_prob; - DCHECK_GE(m, 1); - DCHECK_LE(m, kVp9MaxProb); - DCHECK_LT(v, arraysize(inv_map_table)); - v = inv_map_table[v]; - m--; - if ((m << 1) <= kVp9MaxProb) { - return 1 + InvRecenterNonneg(v, m); - } else { - return kVp9MaxProb - InvRecenterNonneg(v, kVp9MaxProb - 1 - m); - } -} - -} // namespace - -Vp9CompressedHeaderParser::Vp9CompressedHeaderParser() = default; - -// 6.3.1 Tx mode syntax -void Vp9CompressedHeaderParser::ReadTxMode(Vp9FrameHeader* fhdr) { - int tx_mode; - if (fhdr->quant_params.IsLossless()) { - tx_mode = Vp9CompressedHeader::ONLY_4X4; - } else { - tx_mode = reader_.ReadLiteral(2); - if (tx_mode == Vp9CompressedHeader::ALLOW_32X32) - tx_mode += reader_.ReadLiteral(1); - } - fhdr->compressed_header.tx_mode = - static_cast<Vp9CompressedHeader::Vp9TxMode>(tx_mode); -} - -// 6.3.4 Decode term subexp syntax -uint8_t Vp9CompressedHeaderParser::DecodeTermSubexp() { - if (reader_.ReadLiteral(1) == 0) - return reader_.ReadLiteral(4); - if (reader_.ReadLiteral(1) == 0) - return reader_.ReadLiteral(4) + 16; - if (reader_.ReadLiteral(1) == 0) - return reader_.ReadLiteral(5) + 32; - uint8_t v = reader_.ReadLiteral(7); - if (v < 65) - return v + 64; - return (v << 1) - 1 + reader_.ReadLiteral(1); -} - -// 6.3.3 Diff update prob syntax -void Vp9CompressedHeaderParser::DiffUpdateProb(Vp9Prob* prob) { - const Vp9Prob kUpdateProb = 252; - if (reader_.ReadBool(kUpdateProb)) { - uint8_t delta_prob = DecodeTermSubexp(); - *prob = InvRemapProb(delta_prob, *prob); - } -} - -// Helper function to DiffUpdateProb an array of probs. -template <int N> -void Vp9CompressedHeaderParser::DiffUpdateProbArray(Vp9Prob (&prob_array)[N]) { - for (auto& x : prob_array) { - DiffUpdateProb(&x); - } -} - -// 6.3.2 Tx mode probs syntax -void Vp9CompressedHeaderParser::ReadTxModeProbs( - Vp9FrameContext* frame_context) { - for (auto& a : frame_context->tx_probs_8x8) { - DiffUpdateProbArray(a); - } - for (auto& a : frame_context->tx_probs_16x16) { - DiffUpdateProbArray(a); - } - for (auto& a : frame_context->tx_probs_32x32) { - DiffUpdateProbArray(a); - } -} - -// 6.3.7 Coef probs syntax -void Vp9CompressedHeaderParser::ReadCoefProbs(Vp9FrameHeader* fhdr) { - const int tx_mode_to_biggest_tx_size[Vp9CompressedHeader::TX_MODES] = { - 0, 1, 2, 3, 3, - }; - const int max_tx_size = - tx_mode_to_biggest_tx_size[fhdr->compressed_header.tx_mode]; - for (int tx_size = 0; tx_size <= max_tx_size; tx_size++) { - if (reader_.ReadLiteral(1) == 0) - continue; - - for (auto& ai : fhdr->frame_context.coef_probs[tx_size]) { - for (auto& aj : ai) { - for (auto& ak : aj) { - int max_l = (ak == aj[0]) ? 3 : 6; - for (int l = 0; l < max_l; l++) { - DiffUpdateProbArray(ak[l]); - } - } - } - } - } -} - -// 6.3.8 Skip probs syntax -void Vp9CompressedHeaderParser::ReadSkipProb(Vp9FrameContext* frame_context) { - DiffUpdateProbArray(frame_context->skip_prob); -} - -// 6.3.9 Inter mode probs syntax -void Vp9CompressedHeaderParser::ReadInterModeProbs( - Vp9FrameContext* frame_context) { - for (auto& a : frame_context->inter_mode_probs) - DiffUpdateProbArray(a); -} - -// 6.3.10 Interp filter probs syntax -void Vp9CompressedHeaderParser::ReadInterpFilterProbs( - Vp9FrameContext* frame_context) { - for (auto& a : frame_context->interp_filter_probs) - DiffUpdateProbArray(a); -} - -// 6.3.11 Intra inter probs syntax -void Vp9CompressedHeaderParser::ReadIsInterProbs( - Vp9FrameContext* frame_context) { - DiffUpdateProbArray(frame_context->is_inter_prob); -} - -// 6.3.12 Frame reference mode syntax -void Vp9CompressedHeaderParser::ReadFrameReferenceMode(Vp9FrameHeader* fhdr) { - bool compound_reference_allowed = false; - for (int i = VP9_FRAME_LAST + 1; i < VP9_FRAME_MAX; i++) - if (fhdr->ref_frame_sign_bias[i] != fhdr->ref_frame_sign_bias[1]) - compound_reference_allowed = true; - - if (compound_reference_allowed && reader_.ReadLiteral(1)) { - fhdr->compressed_header.reference_mode = - reader_.ReadLiteral(1) ? REFERENCE_MODE_SELECT : COMPOUND_REFERENCE; - } else { - fhdr->compressed_header.reference_mode = SINGLE_REFERENCE; - } -} - -// 6.3.13 Frame reference mode probs syntax -void Vp9CompressedHeaderParser::ReadFrameReferenceModeProbs( - Vp9FrameHeader* fhdr) { - Vp9FrameContext* frame_context = &fhdr->frame_context; - if (fhdr->compressed_header.reference_mode == REFERENCE_MODE_SELECT) - DiffUpdateProbArray(frame_context->comp_mode_prob); - - if (fhdr->compressed_header.reference_mode != COMPOUND_REFERENCE) - for (auto& a : frame_context->single_ref_prob) - DiffUpdateProbArray(a); - - if (fhdr->compressed_header.reference_mode != SINGLE_REFERENCE) - DiffUpdateProbArray(frame_context->comp_ref_prob); -} - -// 6.3.14 Y mode probs syntax -void Vp9CompressedHeaderParser::ReadYModeProbs(Vp9FrameContext* frame_context) { - for (auto& a : frame_context->y_mode_probs) - DiffUpdateProbArray(a); -} - -// 6.3.15 Partition probs syntax -void Vp9CompressedHeaderParser::ReadPartitionProbs( - Vp9FrameContext* frame_context) { - for (auto& a : frame_context->partition_probs) - DiffUpdateProbArray(a); -} - -// 6.3.16 MV probs syntax -void Vp9CompressedHeaderParser::ReadMvProbs(bool allow_high_precision_mv, - Vp9FrameContext* frame_context) { - UpdateMvProbArray(frame_context->mv_joint_probs); - - for (int i = 0; i < 2; i++) { - UpdateMvProb(&frame_context->mv_sign_prob[i]); - UpdateMvProbArray(frame_context->mv_class_probs[i]); - UpdateMvProb(&frame_context->mv_class0_bit_prob[i]); - UpdateMvProbArray(frame_context->mv_bits_prob[i]); - } - - for (int i = 0; i < 2; i++) { - for (auto& a : frame_context->mv_class0_fr_probs[i]) - UpdateMvProbArray(a); - UpdateMvProbArray(frame_context->mv_fr_probs[i]); - } - - if (allow_high_precision_mv) { - for (int i = 0; i < 2; i++) { - UpdateMvProb(&frame_context->mv_class0_hp_prob[i]); - UpdateMvProb(&frame_context->mv_hp_prob[i]); - } - } -} - -// 6.3.17 Update mv prob syntax -void Vp9CompressedHeaderParser::UpdateMvProb(Vp9Prob* prob) { - if (reader_.ReadBool(252)) - *prob = reader_.ReadLiteral(7) << 1 | 1; -} - -// Helper function to UpdateMvProb an array of probs. -template <int N> -void Vp9CompressedHeaderParser::UpdateMvProbArray(Vp9Prob (&prob_array)[N]) { - for (auto& x : prob_array) { - UpdateMvProb(&x); - } -} - -// 6.3 Compressed header syntax -bool Vp9CompressedHeaderParser::Parse(const uint8_t* stream, - off_t frame_size, - Vp9FrameHeader* fhdr) { - DVLOG(2) << "Vp9CompressedHeaderParser::Parse"; - if (!reader_.Initialize(stream, frame_size)) - return false; - - ReadTxMode(fhdr); - if (fhdr->compressed_header.tx_mode == Vp9CompressedHeader::TX_MODE_SELECT) - ReadTxModeProbs(&fhdr->frame_context); - - ReadCoefProbs(fhdr); - ReadSkipProb(&fhdr->frame_context); - - if (!fhdr->IsIntra()) { - ReadInterModeProbs(&fhdr->frame_context); - if (fhdr->interpolation_filter == SWITCHABLE) - ReadInterpFilterProbs(&fhdr->frame_context); - ReadIsInterProbs(&fhdr->frame_context); - ReadFrameReferenceMode(fhdr); - ReadFrameReferenceModeProbs(fhdr); - ReadYModeProbs(&fhdr->frame_context); - ReadPartitionProbs(&fhdr->frame_context); - ReadMvProbs(fhdr->allow_high_precision_mv, &fhdr->frame_context); - } - - if (!reader_.IsValid()) { - DVLOG(1) << "parser reads beyond the end of buffer"; - return false; - } - if (!reader_.ConsumePaddingBits()) { - DVLOG(1) << "padding bits are not zero"; - return false; - } - return true; -} - -} // namespace media diff --git a/accel/vp9_compressed_header_parser.h b/accel/vp9_compressed_header_parser.h deleted file mode 100644 index 5f5ff56..0000000 --- a/accel/vp9_compressed_header_parser.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: e5a9a62 - -#ifndef VP9_COMPRESSED_HEADER_PARSER_H_ -#define VP9_COMPRESSED_HEADER_PARSER_H_ - -#include "vp9_bool_decoder.h" -#include "vp9_parser.h" - -namespace media { - -class Vp9CompressedHeaderParser { - public: - Vp9CompressedHeaderParser(); - - // Parses VP9 compressed header in |stream| with |frame_size| into |fhdr|. - // Returns true if no error. - bool Parse(const uint8_t* stream, off_t frame_size, Vp9FrameHeader* fhdr); - - private: - void ReadTxMode(Vp9FrameHeader* fhdr); - uint8_t DecodeTermSubexp(); - void DiffUpdateProb(Vp9Prob* prob); - template <int N> - void DiffUpdateProbArray(Vp9Prob (&prob_array)[N]); - void ReadTxModeProbs(Vp9FrameContext* frame_context); - void ReadCoefProbs(Vp9FrameHeader* fhdr); - void ReadSkipProb(Vp9FrameContext* frame_context); - void ReadInterModeProbs(Vp9FrameContext* frame_context); - void ReadInterpFilterProbs(Vp9FrameContext* frame_context); - void ReadIsInterProbs(Vp9FrameContext* frame_context); - void ReadFrameReferenceMode(Vp9FrameHeader* fhdr); - void ReadFrameReferenceModeProbs(Vp9FrameHeader* fhdr); - void ReadYModeProbs(Vp9FrameContext* frame_context); - void ReadPartitionProbs(Vp9FrameContext* frame_context); - void ReadMvProbs(bool allow_high_precision_mv, - Vp9FrameContext* frame_context); - void UpdateMvProb(Vp9Prob* prob); - template <int N> - void UpdateMvProbArray(Vp9Prob (&prob_array)[N]); - - // Bool decoder for compressed frame header. - Vp9BoolDecoder reader_; - - DISALLOW_COPY_AND_ASSIGN(Vp9CompressedHeaderParser); -}; - -} // namespace media - -#endif // VP9_COMPRESSED_HEADER_PARSER_H_ diff --git a/accel/vp9_decoder.cc b/accel/vp9_decoder.cc deleted file mode 100644 index d8af03d..0000000 --- a/accel/vp9_decoder.cc +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 7441087 - -#include "rect.h" -#include "vp9_decoder.h" - -#include <memory> - -#include "base/bind.h" -#include "base/logging.h" - -namespace media { - -VP9Decoder::VP9Accelerator::VP9Accelerator() {} - -VP9Decoder::VP9Accelerator::~VP9Accelerator() {} - -VP9Decoder::VP9Decoder(VP9Accelerator* accelerator) - : state_(kNeedStreamMetadata), - accelerator_(accelerator), - parser_(accelerator->IsFrameContextRequired()) { - ref_frames_.resize(kVp9NumRefFrames); -} - -VP9Decoder::~VP9Decoder() {} - -void VP9Decoder::SetStream(const uint8_t* ptr, size_t size) { - DCHECK(ptr); - DCHECK(size); - - DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size; - parser_.SetStream(ptr, size); -} - -bool VP9Decoder::Flush() { - DVLOG(2) << "Decoder flush"; - Reset(); - return true; -} - -void VP9Decoder::Reset() { - curr_frame_hdr_ = nullptr; - for (auto& ref_frame : ref_frames_) - ref_frame = nullptr; - - parser_.Reset(); - - if (state_ == kDecoding) - state_ = kAfterReset; -} - -VP9Decoder::DecodeResult VP9Decoder::Decode() { - while (1) { - // Read a new frame header if one is not awaiting decoding already. - if (!curr_frame_hdr_) { - std::unique_ptr<Vp9FrameHeader> hdr(new Vp9FrameHeader()); - Vp9Parser::Result res = parser_.ParseNextFrame(hdr.get()); - switch (res) { - case Vp9Parser::kOk: - curr_frame_hdr_ = std::move(hdr); - break; - - case Vp9Parser::kEOStream: - return kRanOutOfStreamData; - - case Vp9Parser::kInvalidStream: - DVLOG(1) << "Error parsing stream"; - SetError(); - return kDecodeError; - - case Vp9Parser::kAwaitingRefresh: - DVLOG(4) << "Awaiting context update"; - return kNeedContextUpdate; - } - } - - if (state_ != kDecoding) { - // Not kDecoding, so we need a resume point (a keyframe), as we are after - // reset or at the beginning of the stream. Drop anything that is not - // a keyframe in such case, and continue looking for a keyframe. - if (curr_frame_hdr_->IsKeyframe()) { - state_ = kDecoding; - } else { - curr_frame_hdr_.reset(); - continue; - } - } - - if (curr_frame_hdr_->show_existing_frame) { - // This frame header only instructs us to display one of the - // previously-decoded frames, but has no frame data otherwise. Display - // and continue decoding subsequent frames. - size_t frame_to_show = curr_frame_hdr_->frame_to_show_map_idx; - if (frame_to_show >= ref_frames_.size() || !ref_frames_[frame_to_show]) { - DVLOG(1) << "Request to show an invalid frame"; - SetError(); - return kDecodeError; - } - - if (!accelerator_->OutputPicture(ref_frames_[frame_to_show])) { - SetError(); - return kDecodeError; - } - - curr_frame_hdr_.reset(); - continue; - } - - Size new_pic_size(curr_frame_hdr_->frame_width, - curr_frame_hdr_->frame_height); - DCHECK(!new_pic_size.IsEmpty()); - - if (new_pic_size != pic_size_) { - DVLOG(1) << "New resolution: " << new_pic_size.ToString(); - - if (!curr_frame_hdr_->IsKeyframe()) { - // TODO(posciak): This is doable, but requires a few modifications to - // VDA implementations to allow multiple picture buffer sets in flight. - DVLOG(1) << "Resolution change currently supported for keyframes only"; - SetError(); - return kDecodeError; - } - - // TODO(posciak): This requires us to be on a keyframe (see above) and is - // required, because VDA clients expect all surfaces to be returned before - // they can cycle surface sets after receiving kAllocateNewSurfaces. - // This is only an implementation detail of VDAs and can be improved. - for (auto& ref_frame : ref_frames_) - ref_frame = nullptr; - - pic_size_ = new_pic_size; - return kAllocateNewSurfaces; - } - - scoped_refptr<VP9Picture> pic = accelerator_->CreateVP9Picture(); - if (!pic) - return kRanOutOfSurfaces; - - Rect new_render_rect(curr_frame_hdr_->render_width, - curr_frame_hdr_->render_height); - // For safety, check the validity of render size or leave it as (0, 0). - if (!Rect(pic_size_).Contains(new_render_rect)) { - DVLOG(1) << "Render size exceeds picture size. render size: " - << new_render_rect.ToString() - << ", picture size: " << pic_size_.ToString(); - new_render_rect = Rect(); - } - DVLOG(2) << "Render resolution: " << new_render_rect.ToString(); - - pic->visible_rect = new_render_rect; - pic->frame_hdr.reset(curr_frame_hdr_.release()); - - if (!DecodeAndOutputPicture(pic)) { - SetError(); - return kDecodeError; - } - } -} - -void VP9Decoder::RefreshReferenceFrames(const scoped_refptr<VP9Picture>& pic) { - for (size_t i = 0; i < kVp9NumRefFrames; ++i) { - DCHECK(!pic->frame_hdr->IsKeyframe() || pic->frame_hdr->RefreshFlag(i)); - if (pic->frame_hdr->RefreshFlag(i)) - ref_frames_[i] = pic; - } -} - -void VP9Decoder::UpdateFrameContext( - const scoped_refptr<VP9Picture>& pic, - const base::Callback<void(const Vp9FrameContext&)>& context_refresh_cb) { - DCHECK(!context_refresh_cb.is_null()); - Vp9FrameContext frame_ctx; - memset(&frame_ctx, 0, sizeof(frame_ctx)); - - if (!accelerator_->GetFrameContext(pic, &frame_ctx)) { - SetError(); - return; - } - - context_refresh_cb.Run(frame_ctx); -} - -bool VP9Decoder::DecodeAndOutputPicture(scoped_refptr<VP9Picture> pic) { - DCHECK(!pic_size_.IsEmpty()); - DCHECK(pic->frame_hdr); - - base::Closure done_cb; - const auto& context_refresh_cb = - parser_.GetContextRefreshCb(pic->frame_hdr->frame_context_idx); - if (!context_refresh_cb.is_null()) - done_cb = base::Bind(&VP9Decoder::UpdateFrameContext, - base::Unretained(this), pic, context_refresh_cb); - - const Vp9Parser::Context& context = parser_.context(); - if (!accelerator_->SubmitDecode(pic, context.segmentation(), - context.loop_filter(), ref_frames_, done_cb)) - return false; - - if (pic->frame_hdr->show_frame) { - if (!accelerator_->OutputPicture(pic)) - return false; - } - - RefreshReferenceFrames(pic); - return true; -} - -void VP9Decoder::SetError() { - Reset(); - state_ = kError; -} - -Size VP9Decoder::GetPicSize() const { - return pic_size_; -} - -size_t VP9Decoder::GetRequiredNumOfPictures() const { - // kMaxVideoFrames to keep higher level media pipeline populated, +2 for the - // pictures being parsed and decoded currently. - // TODO(johnylin): see if we could get rid of kMaxVideoFrames. - const size_t kMaxVideoFrames = 4; - return kMaxVideoFrames + kVp9NumRefFrames + 2; -} - -} // namespace media diff --git a/accel/vp9_decoder.h b/accel/vp9_decoder.h deleted file mode 100644 index cdbcd69..0000000 --- a/accel/vp9_decoder.h +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 77118c9 - -#ifndef VP9_DECODER_H_ -#define VP9_DECODER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> -#include <vector> - -#include "base/callback_forward.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "accelerated_video_decoder.h" -#include "vp9_parser.h" -#include "vp9_picture.h" - -namespace media { - -// This class implements an AcceleratedVideoDecoder for VP9 decoding. -// Clients of this class are expected to pass raw VP9 stream and are expected -// to provide an implementation of VP9Accelerator for offloading final steps -// of the decoding process. -// -// This class must be created, called and destroyed on a single thread, and -// does nothing internally on any other thread. -class VP9Decoder : public AcceleratedVideoDecoder { - public: - class VP9Accelerator { - public: - VP9Accelerator(); - virtual ~VP9Accelerator(); - - // Create a new VP9Picture that the decoder client can use for initial - // stages of the decoding process and pass back to this accelerator for - // final, accelerated stages of it, or for reference when decoding other - // pictures. - // - // When a picture is no longer needed by the decoder, it will just drop - // its reference to it, and it may do so at any time. - // - // Note that this may return nullptr if the accelerator is not able to - // provide any new pictures at the given time. The decoder must handle this - // case and treat it as normal, returning kRanOutOfSurfaces from Decode(). - virtual scoped_refptr<VP9Picture> CreateVP9Picture() = 0; - - // Submit decode for |pic| to be run in accelerator, taking as arguments - // information contained in it, as well as current segmentation and loop - // filter state in |segm_params| and |lf_params|, respectively, and using - // pictures in |ref_pictures| for reference. - // If done_cb_ is not null, it will be run once decode is done in hardware. - // - // Note that returning from this method does not mean that the decode - // process is finished, but the caller may drop its references to |pic| - // and |ref_pictures| immediately, and the data in |segm_params| and - // |lf_params| does not need to remain valid after this method returns. - // - // Return true when successful, false otherwise. - virtual bool SubmitDecode( - const scoped_refptr<VP9Picture>& pic, - const Vp9SegmentationParams& segm_params, - const Vp9LoopFilterParams& lf_params, - const std::vector<scoped_refptr<VP9Picture>>& ref_pictures, - const base::Closure& done_cb) = 0; - - // Schedule output (display) of |pic|. - // - // Note that returning from this method does not mean that |pic| has already - // been outputted (displayed), but guarantees that all pictures will be - // outputted in the same order as this method was called for them, and that - // they are decoded before outputting (assuming SubmitDecode() has been - // called for them beforehand). Decoder may drop its references to |pic| - // immediately after calling this method. - // - // Return true when successful, false otherwise. - virtual bool OutputPicture(const scoped_refptr<VP9Picture>& pic) = 0; - - // Return true if the accelerator requires the client to provide frame - // context in order to decode. If so, the Vp9FrameHeader provided by the - // client must contain a valid compressed header and frame context data. - virtual bool IsFrameContextRequired() const = 0; - - // Set |frame_ctx| to the state after decoding |pic|, returning true on - // success, false otherwise. - virtual bool GetFrameContext(const scoped_refptr<VP9Picture>& pic, - Vp9FrameContext* frame_ctx) = 0; - - private: - DISALLOW_COPY_AND_ASSIGN(VP9Accelerator); - }; - - explicit VP9Decoder(VP9Accelerator* accelerator); - ~VP9Decoder() override; - - // AcceleratedVideoDecoder implementation. - void SetStream(const uint8_t* ptr, size_t size) override; - bool Flush() override WARN_UNUSED_RESULT; - void Reset() override; - DecodeResult Decode() override WARN_UNUSED_RESULT; - Size GetPicSize() const override; - size_t GetRequiredNumOfPictures() const override; - - private: - // Update ref_frames_ based on the information in current frame header. - void RefreshReferenceFrames(const scoped_refptr<VP9Picture>& pic); - - // Decode and possibly output |pic| (if the picture is to be shown). - // Return true on success, false otherwise. - bool DecodeAndOutputPicture(scoped_refptr<VP9Picture> pic); - - // Get frame context state after decoding |pic| from the accelerator, and call - // |context_refresh_cb| with the acquired state. - void UpdateFrameContext( - const scoped_refptr<VP9Picture>& pic, - const base::Callback<void(const Vp9FrameContext&)>& context_refresh_cb); - - // Called on error, when decoding cannot continue. Sets state_ to kError and - // releases current state. - void SetError(); - - enum State { - kNeedStreamMetadata, // After initialization, need a keyframe. - kDecoding, // Ready to decode from any point. - kAfterReset, // After Reset(), need a resume point. - kError, // Error in decode, can't continue. - }; - - // Current decoder state. - State state_; - - // Current frame header to be used in decoding the next picture. - std::unique_ptr<Vp9FrameHeader> curr_frame_hdr_; - - // Reference frames currently in use. - std::vector<scoped_refptr<VP9Picture>> ref_frames_; - - // Current coded resolution. - Size pic_size_; - - // VP9Accelerator instance owned by the client. - VP9Accelerator* accelerator_; - - Vp9Parser parser_; - - DISALLOW_COPY_AND_ASSIGN(VP9Decoder); -}; - -} // namespace media - -#endif // VP9_DECODER_H_ diff --git a/accel/vp9_parser.cc b/accel/vp9_parser.cc deleted file mode 100644 index bbd90b9..0000000 --- a/accel/vp9_parser.cc +++ /dev/null @@ -1,676 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of a VP9 bitstream parser. -// -// VERBOSE level: -// 1 something wrong in bitstream -// 2 parsing steps -// 3 parsed values (selected) -// Note: ported from Chromium commit head: 2de6929 - -#include "vp9_parser.h" - -#include <algorithm> - -#include "base/bind.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/numerics/safe_conversions.h" -#include "vp9_compressed_header_parser.h" -#include "vp9_uncompressed_header_parser.h" - -namespace media { - -namespace { - -// Coefficients extracted verbatim from "VP9 Bitstream & Decoding Process -// Specification" Version 0.6, Sec 8.6.1 Dequantization functions, see: -// https://www.webmproject.org/vp9/#draft-vp9-bitstream-and-decoding-process-specification -constexpr size_t kQIndexRange = 256; -// clang-format off -// libva is the only user of high bit depth VP9 formats and only supports -// 10 bits per component, see https://github.com/01org/libva/issues/137. -// TODO(mcasas): Add the 12 bit versions of these tables. -const int16_t kDcQLookup[][kQIndexRange] = { - { - 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, - 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, - 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, - 38, 38, 39, 40, 41, 42, 43, 43, 44, 45, 46, 47, - 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, - 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, - 67, 68, 69, 70, 70, 71, 72, 73, 74, 74, 75, 76, - 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, - 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, - 105, 107, 108, 110, 111, 113, 114, 116, 117, 118, 120, 121, - 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, - 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, - 177, 180, 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, - 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, - 250, 253, 257, 261, 265, 269, 272, 276, 280, 284, 288, 292, - 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, - 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, - 411, 417, 423, 429, 435, 441, 447, 454, 461, 467, 475, 482, - 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, - 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, 775, - 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, - 1184, 1232, 1282, 1336, - }, - { - 4, 9, 10, 13, 15, 17, 20, 22, 25, 28, 31, 34, - 37, 40, 43, 47, 50, 53, 57, 60, 64, 68, 71, 75, - 78, 82, 86, 90, 93, 97, 101, 105, 109, 113, 116, 120, - 124, 128, 132, 136, 140, 143, 147, 151, 155, 159, 163, 166, - 170, 174, 178, 182, 185, 189, 193, 197, 200, 204, 208, 212, - 215, 219, 223, 226, 230, 233, 237, 241, 244, 248, 251, 255, - 259, 262, 266, 269, 273, 276, 280, 283, 287, 290, 293, 297, - 300, 304, 307, 310, 314, 317, 321, 324, 327, 331, 334, 337, - 343, 350, 356, 362, 369, 375, 381, 387, 394, 400, 406, 412, - 418, 424, 430, 436, 442, 448, 454, 460, 466, 472, 478, 484, - 490, 499, 507, 516, 525, 533, 542, 550, 559, 567, 576, 584, - 592, 601, 609, 617, 625, 634, 644, 655, 666, 676, 687, 698, - 708, 718, 729, 739, 749, 759, 770, 782, 795, 807, 819, 831, - 844, 856, 868, 880, 891, 906, 920, 933, 947, 961, 975, 988, - 1001, 1015, 1030, 1045, 1061, 1076, 1090, 1105, 1120, 1137, 1153, 1170, - 1186, 1202, 1218, 1236, 1253, 1271, 1288, 1306, 1323, 1342, 1361, 1379, - 1398, 1416, 1436, 1456, 1476, 1496, 1516, 1537, 1559, 1580, 1601, 1624, - 1647, 1670, 1692, 1717, 1741, 1766, 1791, 1817, 1844, 1871, 1900, 1929, - 1958, 1990, 2021, 2054, 2088, 2123, 2159, 2197, 2236, 2276, 2319, 2363, - 2410, 2458, 2508, 2561, 2616, 2675, 2737, 2802, 2871, 2944, 3020, 3102, - 3188, 3280, 3375, 3478, 3586, 3702, 3823, 3953, 4089, 4236, 4394, 4559, - 4737, 4929, 5130, 5347 - } -}; - -const int16_t kAcQLookup[][kQIndexRange] = { - { - 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, - 128, 130, 132, 134, 136, 138, 140, 142, 144, 146, 148, 150, - 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, - 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, 227, - 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, - 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, - 353, 359, 366, 373, 380, 387, 394, 401, 408, 416, 424, 432, - 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, - 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, - 689, 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, - 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, - 1087, 1108, 1129, 1151, 1173, 1196, 1219, 1243, 1267, 1292, 1317, 1343, - 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, - 1725, 1759, 1793, 1828, - }, - { - 4, 9, 11, 13, 16, 18, 21, 24, 27, 30, 33, 37, - 40, 44, 48, 51, 55, 59, 63, 67, 71, 75, 79, 83, - 88, 92, 96, 100, 105, 109, 114, 118, 122, 127, 131, 136, - 140, 145, 149, 154, 158, 163, 168, 172, 177, 181, 186, 190, - 195, 199, 204, 208, 213, 217, 222, 226, 231, 235, 240, 244, - 249, 253, 258, 262, 267, 271, 275, 280, 284, 289, 293, 297, - 302, 306, 311, 315, 319, 324, 328, 332, 337, 341, 345, 349, - 354, 358, 362, 367, 371, 375, 379, 384, 388, 392, 396, 401, - 409, 417, 425, 433, 441, 449, 458, 466, 474, 482, 490, 498, - 506, 514, 523, 531, 539, 547, 555, 563, 571, 579, 588, 596, - 604, 616, 628, 640, 652, 664, 676, 688, 700, 713, 725, 737, - 749, 761, 773, 785, 797, 809, 825, 841, 857, 873, 889, 905, - 922, 938, 954, 970, 986, 1002, 1018, 1038, 1058, 1078, 1098, 1118, - 1138, 1158, 1178, 1198, 1218, 1242, 1266, 1290, 1314, 1338, 1362, 1386, - 1411, 1435, 1463, 1491, 1519, 1547, 1575, 1603, 1631, 1663, 1695, 1727, - 1759, 1791, 1823, 1859, 1895, 1931, 1967, 2003, 2039, 2079, 2119, 2159, - 2199, 2239, 2283, 2327, 2371, 2415, 2459, 2507, 2555, 2603, 2651, 2703, - 2755, 2807, 2859, 2915, 2971, 3027, 3083, 3143, 3203, 3263, 3327, 3391, - 3455, 3523, 3591, 3659, 3731, 3803, 3876, 3952, 4028, 4104, 4184, 4264, - 4348, 4432, 4516, 4604, 4692, 4784, 4876, 4972, 5068, 5168, 5268, 5372, - 5476, 5584, 5692, 5804, 5916, 6032, 6148, 6268, 6388, 6512, 6640, 6768, - 6900, 7036, 7172, 7312 - } -}; -// clang-format on - -static_assert(arraysize(kDcQLookup[0]) == arraysize(kAcQLookup[0]), - "quantizer lookup arrays of incorrect size"); - -size_t ClampQ(size_t q) { - return std::min(q, kQIndexRange - 1); -} - -int ClampLf(int lf) { - const int kMaxLoopFilterLevel = 63; - return std::min(std::max(0, lf), kMaxLoopFilterLevel); -} - -} // namespace - -bool Vp9FrameHeader::IsKeyframe() const { - // When show_existing_frame is true, the frame header does not precede an - // actual frame to be decoded, so frame_type does not apply (and is not read - // from the stream). - return !show_existing_frame && frame_type == KEYFRAME; -} - -bool Vp9FrameHeader::IsIntra() const { - return !show_existing_frame && (frame_type == KEYFRAME || intra_only); -} - -Vp9Parser::FrameInfo::FrameInfo(const uint8_t* ptr, off_t size) - : ptr(ptr), size(size) {} - -bool Vp9FrameContext::IsValid() const { - // probs should be in [1, 255] range. - static_assert(sizeof(Vp9Prob) == 1, - "following checks assuming Vp9Prob is single byte"); - if (memchr(tx_probs_8x8, 0, sizeof(tx_probs_8x8))) - return false; - if (memchr(tx_probs_16x16, 0, sizeof(tx_probs_16x16))) - return false; - if (memchr(tx_probs_32x32, 0, sizeof(tx_probs_32x32))) - return false; - - for (auto& a : coef_probs) { - for (auto& ai : a) { - for (auto& aj : ai) { - for (auto& ak : aj) { - int max_l = (ak == aj[0]) ? 3 : 6; - for (int l = 0; l < max_l; l++) { - for (auto& x : ak[l]) { - if (x == 0) - return false; - } - } - } - } - } - } - if (memchr(skip_prob, 0, sizeof(skip_prob))) - return false; - if (memchr(inter_mode_probs, 0, sizeof(inter_mode_probs))) - return false; - if (memchr(interp_filter_probs, 0, sizeof(interp_filter_probs))) - return false; - if (memchr(is_inter_prob, 0, sizeof(is_inter_prob))) - return false; - if (memchr(comp_mode_prob, 0, sizeof(comp_mode_prob))) - return false; - if (memchr(single_ref_prob, 0, sizeof(single_ref_prob))) - return false; - if (memchr(comp_ref_prob, 0, sizeof(comp_ref_prob))) - return false; - if (memchr(y_mode_probs, 0, sizeof(y_mode_probs))) - return false; - if (memchr(uv_mode_probs, 0, sizeof(uv_mode_probs))) - return false; - if (memchr(partition_probs, 0, sizeof(partition_probs))) - return false; - if (memchr(mv_joint_probs, 0, sizeof(mv_joint_probs))) - return false; - if (memchr(mv_sign_prob, 0, sizeof(mv_sign_prob))) - return false; - if (memchr(mv_class_probs, 0, sizeof(mv_class_probs))) - return false; - if (memchr(mv_class0_bit_prob, 0, sizeof(mv_class0_bit_prob))) - return false; - if (memchr(mv_bits_prob, 0, sizeof(mv_bits_prob))) - return false; - if (memchr(mv_class0_fr_probs, 0, sizeof(mv_class0_fr_probs))) - return false; - if (memchr(mv_fr_probs, 0, sizeof(mv_fr_probs))) - return false; - if (memchr(mv_class0_hp_prob, 0, sizeof(mv_class0_hp_prob))) - return false; - if (memchr(mv_hp_prob, 0, sizeof(mv_hp_prob))) - return false; - - return true; -} - -Vp9Parser::Context::Vp9FrameContextManager::Vp9FrameContextManager() - : weak_ptr_factory_(this) {} - -Vp9Parser::Context::Vp9FrameContextManager::~Vp9FrameContextManager() = default; - -const Vp9FrameContext& -Vp9Parser::Context::Vp9FrameContextManager::frame_context() const { - DCHECK(initialized_); - DCHECK(!needs_client_update_); - return frame_context_; -} - -void Vp9Parser::Context::Vp9FrameContextManager::Reset() { - initialized_ = false; - needs_client_update_ = false; - weak_ptr_factory_.InvalidateWeakPtrs(); -} - -void Vp9Parser::Context::Vp9FrameContextManager::SetNeedsClientUpdate() { - DCHECK(!needs_client_update_); - initialized_ = true; - needs_client_update_ = true; -} - -Vp9Parser::ContextRefreshCallback -Vp9Parser::Context::Vp9FrameContextManager::GetUpdateCb() { - if (needs_client_update_) - return base::Bind(&Vp9FrameContextManager::UpdateFromClient, - weak_ptr_factory_.GetWeakPtr()); - else - return Vp9Parser::ContextRefreshCallback(); -} - -void Vp9Parser::Context::Vp9FrameContextManager::Update( - const Vp9FrameContext& frame_context) { - // DCHECK because we can trust values from our parser. - DCHECK(frame_context.IsValid()); - initialized_ = true; - frame_context_ = frame_context; - - // For frame context we are updating, it may be still awaiting previous - // ContextRefreshCallback. Because we overwrite the value of context here and - // previous ContextRefreshCallback no longer matters, invalidate the weak ptr - // to prevent previous ContextRefreshCallback run. - // With this optimization, we may be able to parse more frames while previous - // are still decoding. - weak_ptr_factory_.InvalidateWeakPtrs(); - needs_client_update_ = false; -} - -void Vp9Parser::Context::Vp9FrameContextManager::UpdateFromClient( - const Vp9FrameContext& frame_context) { - DVLOG(2) << "Got external frame_context update"; - DCHECK(needs_client_update_); - if (!frame_context.IsValid()) { - DLOG(ERROR) << "Invalid prob value in frame_context"; - return; - } - needs_client_update_ = false; - initialized_ = true; - frame_context_ = frame_context; -} - -void Vp9Parser::Context::Reset() { - memset(&segmentation_, 0, sizeof(segmentation_)); - memset(&loop_filter_, 0, sizeof(loop_filter_)); - memset(&ref_slots_, 0, sizeof(ref_slots_)); - for (auto& manager : frame_context_managers_) - manager.Reset(); -} - -void Vp9Parser::Context::MarkFrameContextForUpdate(size_t frame_context_idx) { - DCHECK_LT(frame_context_idx, arraysize(frame_context_managers_)); - frame_context_managers_[frame_context_idx].SetNeedsClientUpdate(); -} - -void Vp9Parser::Context::UpdateFrameContext( - size_t frame_context_idx, - const Vp9FrameContext& frame_context) { - DCHECK_LT(frame_context_idx, arraysize(frame_context_managers_)); - frame_context_managers_[frame_context_idx].Update(frame_context); -} - -const Vp9Parser::ReferenceSlot& Vp9Parser::Context::GetRefSlot( - size_t ref_type) const { - DCHECK_LT(ref_type, arraysize(ref_slots_)); - return ref_slots_[ref_type]; -} - -void Vp9Parser::Context::UpdateRefSlot( - size_t ref_type, - const Vp9Parser::ReferenceSlot& ref_slot) { - DCHECK_LT(ref_type, arraysize(ref_slots_)); - ref_slots_[ref_type] = ref_slot; -} - -Vp9Parser::Vp9Parser(bool parsing_compressed_header) - : parsing_compressed_header_(parsing_compressed_header) { - Reset(); -} - -Vp9Parser::~Vp9Parser() = default; - -void Vp9Parser::SetStream(const uint8_t* stream, off_t stream_size) { - DCHECK(stream); - stream_ = stream; - bytes_left_ = stream_size; - frames_.clear(); -} - -void Vp9Parser::Reset() { - stream_ = nullptr; - bytes_left_ = 0; - frames_.clear(); - curr_frame_info_.Reset(); - - context_.Reset(); -} - -bool Vp9Parser::ParseUncompressedHeader(const FrameInfo& frame_info, - Vp9FrameHeader* fhdr, - Result* result) { - memset(&curr_frame_header_, 0, sizeof(curr_frame_header_)); - *result = kInvalidStream; - - Vp9UncompressedHeaderParser uncompressed_parser(&context_); - if (!uncompressed_parser.Parse(frame_info.ptr, frame_info.size, - &curr_frame_header_)) { - *result = kInvalidStream; - return true; - } - - if (curr_frame_header_.header_size_in_bytes == 0) { - // Verify padding bits are zero. - for (off_t i = curr_frame_header_.uncompressed_header_size; - i < frame_info.size; i++) { - if (frame_info.ptr[i] != 0) { - DVLOG(1) << "Padding bits are not zeros."; - *result = kInvalidStream; - return true; - } - } - *fhdr = curr_frame_header_; - *result = kOk; - return true; - } - if (curr_frame_header_.uncompressed_header_size + - curr_frame_header_.header_size_in_bytes > - base::checked_cast<size_t>(frame_info.size)) { - DVLOG(1) << "header_size_in_bytes=" - << curr_frame_header_.header_size_in_bytes - << " is larger than bytes left in buffer: " - << frame_info.size - curr_frame_header_.uncompressed_header_size; - *result = kInvalidStream; - return true; - } - - return false; -} - -bool Vp9Parser::ParseCompressedHeader(const FrameInfo& frame_info, - Result* result) { - *result = kInvalidStream; - size_t frame_context_idx = curr_frame_header_.frame_context_idx; - const Context::Vp9FrameContextManager& context_to_load = - context_.frame_context_managers_[frame_context_idx]; - if (!context_to_load.initialized()) { - // 8.2 Frame order constraints - // must load an initialized set of probabilities. - DVLOG(1) << "loading uninitialized frame context, index=" - << frame_context_idx; - *result = kInvalidStream; - return true; - } - if (context_to_load.needs_client_update()) { - DVLOG(3) << "waiting frame_context_idx=" << frame_context_idx - << " to update"; - curr_frame_info_ = frame_info; - *result = kAwaitingRefresh; - return true; - } - curr_frame_header_.initial_frame_context = curr_frame_header_.frame_context = - context_to_load.frame_context(); - - Vp9CompressedHeaderParser compressed_parser; - if (!compressed_parser.Parse( - frame_info.ptr + curr_frame_header_.uncompressed_header_size, - curr_frame_header_.header_size_in_bytes, &curr_frame_header_)) { - *result = kInvalidStream; - return true; - } - - if (curr_frame_header_.refresh_frame_context) { - // In frame parallel mode, we can refresh the context without decoding - // tile data. - if (curr_frame_header_.frame_parallel_decoding_mode) { - context_.UpdateFrameContext(frame_context_idx, - curr_frame_header_.frame_context); - } else { - context_.MarkFrameContextForUpdate(frame_context_idx); - } - } - return false; -} - -Vp9Parser::Result Vp9Parser::ParseNextFrame(Vp9FrameHeader* fhdr) { - DCHECK(fhdr); - DVLOG(2) << "ParseNextFrame"; - FrameInfo frame_info; - Result result; - - // If |curr_frame_info_| is valid, uncompressed header was parsed into - // |curr_frame_header_| and we are awaiting context update to proceed with - // compressed header parsing. - if (curr_frame_info_.IsValid()) { - DCHECK(parsing_compressed_header_); - frame_info = curr_frame_info_; - curr_frame_info_.Reset(); - } else { - if (frames_.empty()) { - // No frames to be decoded, if there is no more stream, request more. - if (!stream_) - return kEOStream; - - // New stream to be parsed, parse it and fill frames_. - frames_ = ParseSuperframe(); - if (frames_.empty()) { - DVLOG(1) << "Failed parsing superframes"; - return kInvalidStream; - } - } - - frame_info = frames_.front(); - frames_.pop_front(); - - if (ParseUncompressedHeader(frame_info, fhdr, &result)) - return result; - } - - if (parsing_compressed_header_) { - if (ParseCompressedHeader(frame_info, &result)) { - DCHECK(result != kAwaitingRefresh || curr_frame_info_.IsValid()); - return result; - } - } - - if (!SetupSegmentationDequant()) - return kInvalidStream; - SetupLoopFilter(); - UpdateSlots(); - - *fhdr = curr_frame_header_; - return kOk; -} - -Vp9Parser::ContextRefreshCallback Vp9Parser::GetContextRefreshCb( - size_t frame_context_idx) { - DCHECK_LT(frame_context_idx, arraysize(context_.frame_context_managers_)); - auto& frame_context_manager = - context_.frame_context_managers_[frame_context_idx]; - - return frame_context_manager.GetUpdateCb(); -} - -// Annex B Superframes -std::deque<Vp9Parser::FrameInfo> Vp9Parser::ParseSuperframe() { - const uint8_t* stream = stream_; - off_t bytes_left = bytes_left_; - - // Make sure we don't parse stream_ more than once. - stream_ = nullptr; - bytes_left_ = 0; - - if (bytes_left < 1) - return std::deque<FrameInfo>(); - - // If this is a superframe, the last byte in the stream will contain the - // superframe marker. If not, the whole buffer contains a single frame. - uint8_t marker = *(stream + bytes_left - 1); - if ((marker & 0xe0) != 0xc0) { - return {FrameInfo(stream, bytes_left)}; - } - - DVLOG(1) << "Parsing a superframe"; - - // The bytes immediately before the superframe marker constitute superframe - // index, which stores information about sizes of each frame in it. - // Calculate its size and set index_ptr to the beginning of it. - size_t num_frames = (marker & 0x7) + 1; - size_t mag = ((marker >> 3) & 0x3) + 1; - off_t index_size = 2 + mag * num_frames; - - if (bytes_left < index_size) - return std::deque<FrameInfo>(); - - const uint8_t* index_ptr = stream + bytes_left - index_size; - if (marker != *index_ptr) - return std::deque<FrameInfo>(); - - ++index_ptr; - bytes_left -= index_size; - - // Parse frame information contained in the index and add a pointer to and - // size of each frame to frames. - std::deque<FrameInfo> frames; - for (size_t i = 0; i < num_frames; ++i) { - uint32_t size = 0; - for (size_t j = 0; j < mag; ++j) { - size |= *index_ptr << (j * 8); - ++index_ptr; - } - - if (base::checked_cast<off_t>(size) > bytes_left) { - DVLOG(1) << "Not enough data in the buffer for frame " << i; - return std::deque<FrameInfo>(); - } - - frames.push_back(FrameInfo(stream, size)); - stream += size; - bytes_left -= size; - - DVLOG(1) << "Frame " << i << ", size: " << size; - } - - return frames; -} - -// 8.6.1 Dequantization functions -size_t Vp9Parser::GetQIndex(const Vp9QuantizationParams& quant, - size_t segid) const { - const Vp9SegmentationParams& segmentation = context_.segmentation(); - - if (segmentation.FeatureEnabled(segid, - Vp9SegmentationParams::SEG_LVL_ALT_Q)) { - int16_t feature_data = - segmentation.FeatureData(segid, Vp9SegmentationParams::SEG_LVL_ALT_Q); - size_t q_index = segmentation.abs_or_delta_update - ? feature_data - : quant.base_q_idx + feature_data; - return ClampQ(q_index); - } - - return quant.base_q_idx; -} - -// 8.6.1 Dequantization functions -bool Vp9Parser::SetupSegmentationDequant() { - const Vp9QuantizationParams& quant = curr_frame_header_.quant_params; - Vp9SegmentationParams& segmentation = context_.segmentation_; - - if (curr_frame_header_.bit_depth > 10) { - DLOG(ERROR) << "bit_depth > 10 is not supported yet, kDcQLookup and " - "kAcQLookup need to be extended"; - return false; - } - const size_t bit_depth_index = (curr_frame_header_.bit_depth == 8) ? 0 : 1; - - if (segmentation.enabled) { - for (size_t i = 0; i < Vp9SegmentationParams::kNumSegments; ++i) { - const size_t q_index = GetQIndex(quant, i); - segmentation.y_dequant[i][0] = - kDcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_y_dc)]; - segmentation.y_dequant[i][1] = - kAcQLookup[bit_depth_index][ClampQ(q_index)]; - segmentation.uv_dequant[i][0] = - kDcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_uv_dc)]; - segmentation.uv_dequant[i][1] = - kAcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_uv_ac)]; - } - } else { - const size_t q_index = quant.base_q_idx; - segmentation.y_dequant[0][0] = - kDcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_y_dc)]; - segmentation.y_dequant[0][1] = kAcQLookup[bit_depth_index][ClampQ(q_index)]; - segmentation.uv_dequant[0][0] = - kDcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_uv_dc)]; - segmentation.uv_dequant[0][1] = - kAcQLookup[bit_depth_index][ClampQ(q_index + quant.delta_q_uv_ac)]; - } - return true; -} - -// 8.8.1 Loop filter frame init process -void Vp9Parser::SetupLoopFilter() { - Vp9LoopFilterParams& loop_filter = context_.loop_filter_; - if (!loop_filter.level) - return; - - int scale = loop_filter.level < 32 ? 1 : 2; - - for (size_t i = 0; i < Vp9SegmentationParams::kNumSegments; ++i) { - int level = loop_filter.level; - const Vp9SegmentationParams& segmentation = context_.segmentation(); - - if (segmentation.FeatureEnabled(i, Vp9SegmentationParams::SEG_LVL_ALT_LF)) { - int feature_data = - segmentation.FeatureData(i, Vp9SegmentationParams::SEG_LVL_ALT_LF); - level = ClampLf(segmentation.abs_or_delta_update ? feature_data - : level + feature_data); - } - - if (!loop_filter.delta_enabled) { - memset(loop_filter.lvl[i], level, sizeof(loop_filter.lvl[i])); - } else { - loop_filter.lvl[i][Vp9RefType::VP9_FRAME_INTRA][0] = ClampLf( - level + loop_filter.ref_deltas[Vp9RefType::VP9_FRAME_INTRA] * scale); - loop_filter.lvl[i][Vp9RefType::VP9_FRAME_INTRA][1] = 0; - - for (size_t type = Vp9RefType::VP9_FRAME_LAST; - type < Vp9RefType::VP9_FRAME_MAX; ++type) { - for (size_t mode = 0; mode < Vp9LoopFilterParams::kNumModeDeltas; - ++mode) { - loop_filter.lvl[i][type][mode] = - ClampLf(level + loop_filter.ref_deltas[type] * scale + - loop_filter.mode_deltas[mode] * scale); - } - } - } - } -} - -void Vp9Parser::UpdateSlots() { - // 8.10 Reference frame update process - for (size_t i = 0; i < kVp9NumRefFrames; i++) { - if (curr_frame_header_.RefreshFlag(i)) { - ReferenceSlot ref_slot; - ref_slot.initialized = true; - - ref_slot.frame_width = curr_frame_header_.frame_width; - ref_slot.frame_height = curr_frame_header_.frame_height; - ref_slot.subsampling_x = curr_frame_header_.subsampling_x; - ref_slot.subsampling_y = curr_frame_header_.subsampling_y; - ref_slot.bit_depth = curr_frame_header_.bit_depth; - - ref_slot.profile = curr_frame_header_.profile; - ref_slot.color_space = curr_frame_header_.color_space; - context_.UpdateRefSlot(i, ref_slot); - } - } -} - -} // namespace media diff --git a/accel/vp9_parser.h b/accel/vp9_parser.h deleted file mode 100644 index ab1fa57..0000000 --- a/accel/vp9_parser.h +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// This file contains an implementation of a VP9 bitstream parser. The main -// purpose of this parser is to support hardware decode acceleration. Some -// accelerators, e.g. libva which implements VA-API, require the caller -// (chrome) to feed them parsed VP9 frame header. -// -// See media::VP9Decoder for example usage. -// -// Note: ported from Chromium commit head: ec6c6e0 -#ifndef VP9_PARSER_H_ -#define VP9_PARSER_H_ - -#include <stddef.h> -#include <stdint.h> -#include <sys/types.h> - -#include <deque> -#include <memory> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/weak_ptr.h" - -namespace media { - -const int kVp9MaxProfile = 4; -const int kVp9NumRefFramesLog2 = 3; -const size_t kVp9NumRefFrames = 1 << kVp9NumRefFramesLog2; -const uint8_t kVp9MaxProb = 255; -const size_t kVp9NumRefsPerFrame = 3; -const size_t kVp9NumFrameContextsLog2 = 2; -const size_t kVp9NumFrameContexts = 1 << kVp9NumFrameContextsLog2; - -using Vp9Prob = uint8_t; - -enum class Vp9ColorSpace { - UNKNOWN = 0, - BT_601 = 1, - BT_709 = 2, - SMPTE_170 = 3, - SMPTE_240 = 4, - BT_2020 = 5, - RESERVED = 6, - SRGB = 7, -}; - -enum Vp9InterpolationFilter { - EIGHTTAP = 0, - EIGHTTAP_SMOOTH = 1, - EIGHTTAP_SHARP = 2, - BILINEAR = 3, - SWITCHABLE = 4, -}; - -enum Vp9RefType { - VP9_FRAME_INTRA = 0, - VP9_FRAME_LAST = 1, - VP9_FRAME_GOLDEN = 2, - VP9_FRAME_ALTREF = 3, - VP9_FRAME_MAX = 4, -}; - -enum Vp9ReferenceMode { - SINGLE_REFERENCE = 0, - COMPOUND_REFERENCE = 1, - REFERENCE_MODE_SELECT = 2, -}; - -struct Vp9SegmentationParams { - static const size_t kNumSegments = 8; - static const size_t kNumTreeProbs = kNumSegments - 1; - static const size_t kNumPredictionProbs = 3; - enum SegmentLevelFeature { - SEG_LVL_ALT_Q = 0, - SEG_LVL_ALT_LF = 1, - SEG_LVL_REF_FRAME = 2, - SEG_LVL_SKIP = 3, - SEG_LVL_MAX - }; - - bool enabled; - - bool update_map; - uint8_t tree_probs[kNumTreeProbs]; - bool temporal_update; - uint8_t pred_probs[kNumPredictionProbs]; - - bool update_data; - bool abs_or_delta_update; - bool feature_enabled[kNumSegments][SEG_LVL_MAX]; - int16_t feature_data[kNumSegments][SEG_LVL_MAX]; - - int16_t y_dequant[kNumSegments][2]; - int16_t uv_dequant[kNumSegments][2]; - - bool FeatureEnabled(size_t seg_id, SegmentLevelFeature feature) const { - return feature_enabled[seg_id][feature]; - } - - int16_t FeatureData(size_t seg_id, SegmentLevelFeature feature) const { - return feature_data[seg_id][feature]; - } -}; - -struct Vp9LoopFilterParams { - static const size_t kNumModeDeltas = 2; - - uint8_t level; - uint8_t sharpness; - - bool delta_enabled; - bool delta_update; - bool update_ref_deltas[VP9_FRAME_MAX]; - int8_t ref_deltas[VP9_FRAME_MAX]; - bool update_mode_deltas[kNumModeDeltas]; - int8_t mode_deltas[kNumModeDeltas]; - - // Calculated from above fields. - uint8_t lvl[Vp9SegmentationParams::kNumSegments][VP9_FRAME_MAX] - [kNumModeDeltas]; -}; - -// Members of Vp9FrameHeader will be 0-initialized by Vp9Parser::ParseNextFrame. -struct Vp9QuantizationParams { - bool IsLossless() const { - return base_q_idx == 0 && delta_q_y_dc == 0 && delta_q_uv_dc == 0 && - delta_q_uv_ac == 0; - } - - uint8_t base_q_idx; - int8_t delta_q_y_dc; - int8_t delta_q_uv_dc; - int8_t delta_q_uv_ac; -}; - -// Entropy context for frame parsing -struct Vp9FrameContext { - bool IsValid() const; - - Vp9Prob tx_probs_8x8[2][1]; - Vp9Prob tx_probs_16x16[2][2]; - Vp9Prob tx_probs_32x32[2][3]; - - Vp9Prob coef_probs[4][2][2][6][6][3]; - Vp9Prob skip_prob[3]; - Vp9Prob inter_mode_probs[7][3]; - Vp9Prob interp_filter_probs[4][2]; - Vp9Prob is_inter_prob[4]; - - Vp9Prob comp_mode_prob[5]; - Vp9Prob single_ref_prob[5][2]; - Vp9Prob comp_ref_prob[5]; - - Vp9Prob y_mode_probs[4][9]; - Vp9Prob uv_mode_probs[10][9]; - Vp9Prob partition_probs[16][3]; - - Vp9Prob mv_joint_probs[3]; - Vp9Prob mv_sign_prob[2]; - Vp9Prob mv_class_probs[2][10]; - Vp9Prob mv_class0_bit_prob[2]; - Vp9Prob mv_bits_prob[2][10]; - Vp9Prob mv_class0_fr_probs[2][2][3]; - Vp9Prob mv_fr_probs[2][3]; - Vp9Prob mv_class0_hp_prob[2]; - Vp9Prob mv_hp_prob[2]; -}; - -struct Vp9CompressedHeader { - enum Vp9TxMode { - ONLY_4X4 = 0, - ALLOW_8X8 = 1, - ALLOW_16X16 = 2, - ALLOW_32X32 = 3, - TX_MODE_SELECT = 4, - TX_MODES = 5, - }; - - Vp9TxMode tx_mode; - Vp9ReferenceMode reference_mode; -}; - -// VP9 frame header. -struct Vp9FrameHeader { - enum FrameType { - KEYFRAME = 0, - INTERFRAME = 1, - }; - - bool IsKeyframe() const; - bool IsIntra() const; - bool RefreshFlag(size_t i) const { - return !!(refresh_frame_flags & (1u << i)); - } - - uint8_t profile; - - bool show_existing_frame; - uint8_t frame_to_show_map_idx; - - FrameType frame_type; - - bool show_frame; - bool error_resilient_mode; - - uint8_t bit_depth; - Vp9ColorSpace color_space; - bool color_range; - uint8_t subsampling_x; - uint8_t subsampling_y; - - // The range of frame_width and frame_height is 1..2^16. - uint32_t frame_width; - uint32_t frame_height; - uint32_t render_width; - uint32_t render_height; - - bool intra_only; - uint8_t reset_frame_context; - uint8_t refresh_frame_flags; - uint8_t ref_frame_idx[kVp9NumRefsPerFrame]; - bool ref_frame_sign_bias[Vp9RefType::VP9_FRAME_MAX]; - bool allow_high_precision_mv; - Vp9InterpolationFilter interpolation_filter; - - bool refresh_frame_context; - bool frame_parallel_decoding_mode; - uint8_t frame_context_idx; - // |frame_context_idx_to_save_probs| is to be used by save_probs() only, and - // |frame_context_idx| otherwise. - uint8_t frame_context_idx_to_save_probs; - - Vp9QuantizationParams quant_params; - - uint8_t tile_cols_log2; - uint8_t tile_rows_log2; - - // Pointer to the beginning of frame data. It is a responsibility of the - // client of the Vp9Parser to maintain validity of this data while it is - // being used outside of that class. - const uint8_t* data; - - // Size of |data| in bytes. - size_t frame_size; - - // Size of compressed header in bytes. - size_t header_size_in_bytes; - - // Size of uncompressed header in bytes. - size_t uncompressed_header_size; - - Vp9CompressedHeader compressed_header; - // Initial frame entropy context after load_probs2(frame_context_idx). - Vp9FrameContext initial_frame_context; - // Current frame entropy context after header parsing. - Vp9FrameContext frame_context; -}; - -// A parser for VP9 bitstream. -class Vp9Parser { - public: - // If context update is needed after decoding a frame, the client must - // execute this callback, passing the updated context state. - using ContextRefreshCallback = base::Callback<void(const Vp9FrameContext&)>; - - // ParseNextFrame() return values. See documentation for ParseNextFrame(). - enum Result { - kOk, - kInvalidStream, - kEOStream, - kAwaitingRefresh, - }; - - // The parsing context to keep track of references. - struct ReferenceSlot { - bool initialized; - uint32_t frame_width; - uint32_t frame_height; - uint8_t subsampling_x; - uint8_t subsampling_y; - uint8_t bit_depth; - - // More fields for consistency checking. - uint8_t profile; - Vp9ColorSpace color_space; - }; - - // The parsing context that persists across frames. - class Context { - public: - class Vp9FrameContextManager { - public: - Vp9FrameContextManager(); - ~Vp9FrameContextManager(); - bool initialized() const { return initialized_; } - bool needs_client_update() const { return needs_client_update_; } - const Vp9FrameContext& frame_context() const; - - // Resets to uninitialized state. - void Reset(); - - // Marks this context as requiring an update from parser's client. - void SetNeedsClientUpdate(); - - // Updates frame context. - void Update(const Vp9FrameContext& frame_context); - - // Returns a callback to update frame context at a later time with. - ContextRefreshCallback GetUpdateCb(); - - private: - // Updates frame context from parser's client. - void UpdateFromClient(const Vp9FrameContext& frame_context); - - bool initialized_ = false; - bool needs_client_update_ = false; - Vp9FrameContext frame_context_; - - base::WeakPtrFactory<Vp9FrameContextManager> weak_ptr_factory_; - }; - - void Reset(); - - // Mark |frame_context_idx| as requiring update from the client. - void MarkFrameContextForUpdate(size_t frame_context_idx); - - // Update frame context at |frame_context_idx| with the contents of - // |frame_context|. - void UpdateFrameContext(size_t frame_context_idx, - const Vp9FrameContext& frame_context); - - // Return ReferenceSlot for frame at |ref_idx|. - const ReferenceSlot& GetRefSlot(size_t ref_idx) const; - - // Update contents of ReferenceSlot at |ref_idx| with the contents of - // |ref_slot|. - void UpdateRefSlot(size_t ref_idx, const ReferenceSlot& ref_slot); - - const Vp9SegmentationParams& segmentation() const { return segmentation_; } - - const Vp9LoopFilterParams& loop_filter() const { return loop_filter_; } - - private: - friend class Vp9UncompressedHeaderParser; - friend class Vp9Parser; - - // Segmentation and loop filter state. - Vp9SegmentationParams segmentation_; - Vp9LoopFilterParams loop_filter_; - - // Frame references. - ReferenceSlot ref_slots_[kVp9NumRefFrames]; - - Vp9FrameContextManager frame_context_managers_[kVp9NumFrameContexts]; - }; - - // The constructor. See ParseNextFrame() for comments for - // |parsing_compressed_header|. - explicit Vp9Parser(bool parsing_compressed_header); - ~Vp9Parser(); - - // Set a new stream buffer to read from, starting at |stream| and of size - // |stream_size| in bytes. |stream| must point to the beginning of a single - // frame or a single superframe, is owned by caller and must remain valid - // until the next call to SetStream(). - void SetStream(const uint8_t* stream, off_t stream_size); - - // Parse the next frame in the current stream buffer, filling |fhdr| with - // the parsed frame header and updating current segmentation and loop filter - // state. - // Return kOk if a frame has successfully been parsed, - // kEOStream if there is no more data in the current stream buffer, - // kAwaitingRefresh if this frame awaiting frame context update, or - // kInvalidStream on error. - Result ParseNextFrame(Vp9FrameHeader* fhdr); - - // Return current parsing context. - const Context& context() const { return context_; } - - // Return a ContextRefreshCallback, which, if not null, has to be called with - // the new context state after the frame associated with |frame_context_idx| - // is decoded. - ContextRefreshCallback GetContextRefreshCb(size_t frame_context_idx); - - // Clear parser state and return to an initialized state. - void Reset(); - - private: - // Stores start pointer and size of each frame within the current superframe. - struct FrameInfo { - FrameInfo() = default; - FrameInfo(const uint8_t* ptr, off_t size); - bool IsValid() const { return ptr != nullptr; } - void Reset() { ptr = nullptr; } - - // Starting address of the frame. - const uint8_t* ptr = nullptr; - - // Size of the frame in bytes. - off_t size = 0; - }; - - std::deque<FrameInfo> ParseSuperframe(); - - // Returns true and populates |result| with the parsing result if parsing of - // current frame is finished (possibly unsuccessfully). |fhdr| will only be - // populated and valid if |result| is kOk. Otherwise return false, indicating - // that the compressed header must be parsed next. - bool ParseUncompressedHeader(const FrameInfo& frame_info, - Vp9FrameHeader* fhdr, - Result* result); - - // Returns true if parsing of current frame is finished and |result| will be - // populated with value of parsing result. Otherwise, needs to continue setup - // current frame. - bool ParseCompressedHeader(const FrameInfo& frame_info, Result* result); - - size_t GetQIndex(const Vp9QuantizationParams& quant, size_t segid) const; - // Returns true if the setup succeeded. - bool SetupSegmentationDequant(); - void SetupLoopFilter(); - void UpdateSlots(); - - // Current address in the bitstream buffer. - const uint8_t* stream_; - - // Remaining bytes in stream_. - off_t bytes_left_; - - const bool parsing_compressed_header_; - - // FrameInfo for the remaining frames in the current superframe to be parsed. - std::deque<FrameInfo> frames_; - - Context context_; - - FrameInfo curr_frame_info_; - Vp9FrameHeader curr_frame_header_; - - DISALLOW_COPY_AND_ASSIGN(Vp9Parser); -}; - -} // namespace media - -#endif // VP9_PARSER_H_ diff --git a/accel/vp9_picture.cc b/accel/vp9_picture.cc deleted file mode 100644 index df2c3b0..0000000 --- a/accel/vp9_picture.cc +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 6e70beb - -#include "vp9_picture.h" - -namespace media { - -VP9Picture::VP9Picture() {} - -VP9Picture::~VP9Picture() {} - -V4L2VP9Picture* VP9Picture::AsV4L2VP9Picture() { - return nullptr; -} - -} // namespace media diff --git a/accel/vp9_picture.h b/accel/vp9_picture.h deleted file mode 100644 index efff37b..0000000 --- a/accel/vp9_picture.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 70340ce - -#ifndef VP9_PICTURE_H_ -#define VP9_PICTURE_H_ - -#include <memory> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "rect.h" -#include "vp9_parser.h" - -namespace media { - -class V4L2VP9Picture; - -class VP9Picture : public base::RefCountedThreadSafe<VP9Picture> { - public: - VP9Picture(); - - virtual V4L2VP9Picture* AsV4L2VP9Picture(); - - std::unique_ptr<Vp9FrameHeader> frame_hdr; - - // The visible size of picture. This could be either parsed from frame - // header, or set to Rect(0, 0) for indicating invalid values or - // not available. - Rect visible_rect; - - protected: - friend class base::RefCountedThreadSafe<VP9Picture>; - virtual ~VP9Picture(); - - DISALLOW_COPY_AND_ASSIGN(VP9Picture); -}; - -} // namespace media - -#endif // VP9_PICTURE_H_ diff --git a/accel/vp9_raw_bits_reader.cc b/accel/vp9_raw_bits_reader.cc deleted file mode 100644 index dea06e0..0000000 --- a/accel/vp9_raw_bits_reader.cc +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: 2de6929 - -#include "vp9_raw_bits_reader.h" - -#include <limits.h> - -#include "base/logging.h" -#include "bit_reader.h" - -namespace media { - -Vp9RawBitsReader::Vp9RawBitsReader() : valid_(true) {} - -Vp9RawBitsReader::~Vp9RawBitsReader() = default; - -void Vp9RawBitsReader::Initialize(const uint8_t* data, size_t size) { - DCHECK(data); - reader_.reset(new BitReader(data, size)); - valid_ = true; -} - -bool Vp9RawBitsReader::ReadBool() { - DCHECK(reader_); - if (!valid_) - return false; - - int value = 0; - valid_ = reader_->ReadBits(1, &value); - return valid_ ? value == 1 : false; -} - -int Vp9RawBitsReader::ReadLiteral(int bits) { - DCHECK(reader_); - if (!valid_) - return 0; - - int value = 0; - DCHECK_LT(static_cast<size_t>(bits), sizeof(value) * 8); - valid_ = reader_->ReadBits(bits, &value); - return valid_ ? value : 0; -} - -int Vp9RawBitsReader::ReadSignedLiteral(int bits) { - int value = ReadLiteral(bits); - return ReadBool() ? -value : value; -} - -size_t Vp9RawBitsReader::GetBytesRead() const { - DCHECK(reader_); - return (reader_->bits_read() + 7) / 8; -} - -bool Vp9RawBitsReader::ConsumeTrailingBits() { - DCHECK(reader_); - int bits_left = GetBytesRead() * 8 - reader_->bits_read(); - return ReadLiteral(bits_left) == 0; -} - -} // namespace media diff --git a/accel/vp9_raw_bits_reader.h b/accel/vp9_raw_bits_reader.h deleted file mode 100644 index 04ad413..0000000 --- a/accel/vp9_raw_bits_reader.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: e5a9a62 - -#ifndef VP9_RAW_BITS_READER_H_ -#define VP9_RAW_BITS_READER_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <memory> - -#include "base/macros.h" - -namespace media { - -class BitReader; - -// A class to read raw bits stream. See VP9 spec, "RAW-BITS DECODING" section -// for detail. -class Vp9RawBitsReader { - public: - Vp9RawBitsReader(); - ~Vp9RawBitsReader(); - - // |data| is the input buffer with |size| bytes. - void Initialize(const uint8_t* data, size_t size); - - // Returns true if none of the reads since the last Initialize() call has - // gone beyond the end of available data. - bool IsValid() const { return valid_; } - - // Returns how many bytes were read since the last Initialize() call. - // Partial bytes will be counted as one byte. For example, it will return 1 - // if 3 bits were read. - size_t GetBytesRead() const; - - // Reads one bit. - // If the read goes beyond the end of buffer, the return value is undefined. - bool ReadBool(); - - // Reads a literal with |bits| bits. - // If the read goes beyond the end of buffer, the return value is undefined. - int ReadLiteral(int bits); - - // Reads a signed literal with |bits| bits (not including the sign bit). - // If the read goes beyond the end of buffer, the return value is undefined. - int ReadSignedLiteral(int bits); - - // Consumes trailing bits up to next byte boundary. Returns true if no - // trailing bits or they are all zero. - bool ConsumeTrailingBits(); - - private: - std::unique_ptr<BitReader> reader_; - - // Indicates if none of the reads since the last Initialize() call has gone - // beyond the end of available data. - bool valid_; - - DISALLOW_COPY_AND_ASSIGN(Vp9RawBitsReader); -}; - -} // namespace media - -#endif // VP9_RAW_BITS_READER_H_ diff --git a/accel/vp9_uncompressed_header_parser.cc b/accel/vp9_uncompressed_header_parser.cc deleted file mode 100644 index f6dc2eb..0000000 --- a/accel/vp9_uncompressed_header_parser.cc +++ /dev/null @@ -1,1103 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: f06caa0 - -#include "vp9_uncompressed_header_parser.h" - -#include "base/logging.h" - -namespace media { - -namespace { - -// 10.5 Default probability tables -Vp9FrameContext kVp9DefaultFrameContext = { - // tx_probs_8x8 - {{100}, {66}}, - // tx_probs_16x16 - {{20, 152}, {15, 101}}, - // tx_probs_32x32 - {{3, 136, 37}, {5, 52, 13}}, - // coef_probs - {// 4x4 - {{{{{195, 29, 183}, {84, 49, 136}, {8, 42, 71}}, - {{31, 107, 169}, - {35, 99, 159}, - {17, 82, 140}, - {8, 66, 114}, - {2, 44, 76}, - {1, 19, 32}}, - {{40, 132, 201}, - {29, 114, 187}, - {13, 91, 157}, - {7, 75, 127}, - {3, 58, 95}, - {1, 28, 47}}, - {{69, 142, 221}, - {42, 122, 201}, - {15, 91, 159}, - {6, 67, 121}, - {1, 42, 77}, - {1, 17, 31}}, - {{102, 148, 228}, - {67, 117, 204}, - {17, 82, 154}, - {6, 59, 114}, - {2, 39, 75}, - {1, 15, 29}}, - {{156, 57, 233}, - {119, 57, 212}, - {58, 48, 163}, - {29, 40, 124}, - {12, 30, 81}, - {3, 12, 31}}}, - {{{191, 107, 226}, {124, 117, 204}, {25, 99, 155}}, - {{29, 148, 210}, - {37, 126, 194}, - {8, 93, 157}, - {2, 68, 118}, - {1, 39, 69}, - {1, 17, 33}}, - {{41, 151, 213}, - {27, 123, 193}, - {3, 82, 144}, - {1, 58, 105}, - {1, 32, 60}, - {1, 13, 26}}, - {{59, 159, 220}, - {23, 126, 198}, - {4, 88, 151}, - {1, 66, 114}, - {1, 38, 71}, - {1, 18, 34}}, - {{114, 136, 232}, - {51, 114, 207}, - {11, 83, 155}, - {3, 56, 105}, - {1, 33, 65}, - {1, 17, 34}}, - {{149, 65, 234}, - {121, 57, 215}, - {61, 49, 166}, - {28, 36, 114}, - {12, 25, 76}, - {3, 16, 42}}}}, - {{{{214, 49, 220}, {132, 63, 188}, {42, 65, 137}}, - {{85, 137, 221}, - {104, 131, 216}, - {49, 111, 192}, - {21, 87, 155}, - {2, 49, 87}, - {1, 16, 28}}, - {{89, 163, 230}, - {90, 137, 220}, - {29, 100, 183}, - {10, 70, 135}, - {2, 42, 81}, - {1, 17, 33}}, - {{108, 167, 237}, - {55, 133, 222}, - {15, 97, 179}, - {4, 72, 135}, - {1, 45, 85}, - {1, 19, 38}}, - {{124, 146, 240}, - {66, 124, 224}, - {17, 88, 175}, - {4, 58, 122}, - {1, 36, 75}, - {1, 18, 37}}, - {{141, 79, 241}, - {126, 70, 227}, - {66, 58, 182}, - {30, 44, 136}, - {12, 34, 96}, - {2, 20, 47}}}, - {{{229, 99, 249}, {143, 111, 235}, {46, 109, 192}}, - {{82, 158, 236}, - {94, 146, 224}, - {25, 117, 191}, - {9, 87, 149}, - {3, 56, 99}, - {1, 33, 57}}, - {{83, 167, 237}, - {68, 145, 222}, - {10, 103, 177}, - {2, 72, 131}, - {1, 41, 79}, - {1, 20, 39}}, - {{99, 167, 239}, - {47, 141, 224}, - {10, 104, 178}, - {2, 73, 133}, - {1, 44, 85}, - {1, 22, 47}}, - {{127, 145, 243}, - {71, 129, 228}, - {17, 93, 177}, - {3, 61, 124}, - {1, 41, 84}, - {1, 21, 52}}, - {{157, 78, 244}, - {140, 72, 231}, - {69, 58, 184}, - {31, 44, 137}, - {14, 38, 105}, - {8, 23, 61}}}}}, - // 8x8 - {{{{{125, 34, 187}, {52, 41, 133}, {6, 31, 56}}, - {{37, 109, 153}, - {51, 102, 147}, - {23, 87, 128}, - {8, 67, 101}, - {1, 41, 63}, - {1, 19, 29}}, - {{31, 154, 185}, - {17, 127, 175}, - {6, 96, 145}, - {2, 73, 114}, - {1, 51, 82}, - {1, 28, 45}}, - {{23, 163, 200}, - {10, 131, 185}, - {2, 93, 148}, - {1, 67, 111}, - {1, 41, 69}, - {1, 14, 24}}, - {{29, 176, 217}, - {12, 145, 201}, - {3, 101, 156}, - {1, 69, 111}, - {1, 39, 63}, - {1, 14, 23}}, - {{57, 192, 233}, - {25, 154, 215}, - {6, 109, 167}, - {3, 78, 118}, - {1, 48, 69}, - {1, 21, 29}}}, - {{{202, 105, 245}, {108, 106, 216}, {18, 90, 144}}, - {{33, 172, 219}, - {64, 149, 206}, - {14, 117, 177}, - {5, 90, 141}, - {2, 61, 95}, - {1, 37, 57}}, - {{33, 179, 220}, - {11, 140, 198}, - {1, 89, 148}, - {1, 60, 104}, - {1, 33, 57}, - {1, 12, 21}}, - {{30, 181, 221}, - {8, 141, 198}, - {1, 87, 145}, - {1, 58, 100}, - {1, 31, 55}, - {1, 12, 20}}, - {{32, 186, 224}, - {7, 142, 198}, - {1, 86, 143}, - {1, 58, 100}, - {1, 31, 55}, - {1, 12, 22}}, - {{57, 192, 227}, - {20, 143, 204}, - {3, 96, 154}, - {1, 68, 112}, - {1, 42, 69}, - {1, 19, 32}}}}, - {{{{212, 35, 215}, {113, 47, 169}, {29, 48, 105}}, - {{74, 129, 203}, - {106, 120, 203}, - {49, 107, 178}, - {19, 84, 144}, - {4, 50, 84}, - {1, 15, 25}}, - {{71, 172, 217}, - {44, 141, 209}, - {15, 102, 173}, - {6, 76, 133}, - {2, 51, 89}, - {1, 24, 42}}, - {{64, 185, 231}, - {31, 148, 216}, - {8, 103, 175}, - {3, 74, 131}, - {1, 46, 81}, - {1, 18, 30}}, - {{65, 196, 235}, - {25, 157, 221}, - {5, 105, 174}, - {1, 67, 120}, - {1, 38, 69}, - {1, 15, 30}}, - {{65, 204, 238}, - {30, 156, 224}, - {7, 107, 177}, - {2, 70, 124}, - {1, 42, 73}, - {1, 18, 34}}}, - {{{225, 86, 251}, {144, 104, 235}, {42, 99, 181}}, - {{85, 175, 239}, - {112, 165, 229}, - {29, 136, 200}, - {12, 103, 162}, - {6, 77, 123}, - {2, 53, 84}}, - {{75, 183, 239}, - {30, 155, 221}, - {3, 106, 171}, - {1, 74, 128}, - {1, 44, 76}, - {1, 17, 28}}, - {{73, 185, 240}, - {27, 159, 222}, - {2, 107, 172}, - {1, 75, 127}, - {1, 42, 73}, - {1, 17, 29}}, - {{62, 190, 238}, - {21, 159, 222}, - {2, 107, 172}, - {1, 72, 122}, - {1, 40, 71}, - {1, 18, 32}}, - {{61, 199, 240}, - {27, 161, 226}, - {4, 113, 180}, - {1, 76, 129}, - {1, 46, 80}, - {1, 23, 41}}}}}, - // 16x16 - {{{{{7, 27, 153}, {5, 30, 95}, {1, 16, 30}}, - {{50, 75, 127}, - {57, 75, 124}, - {27, 67, 108}, - {10, 54, 86}, - {1, 33, 52}, - {1, 12, 18}}, - {{43, 125, 151}, - {26, 108, 148}, - {7, 83, 122}, - {2, 59, 89}, - {1, 38, 60}, - {1, 17, 27}}, - {{23, 144, 163}, - {13, 112, 154}, - {2, 75, 117}, - {1, 50, 81}, - {1, 31, 51}, - {1, 14, 23}}, - {{18, 162, 185}, - {6, 123, 171}, - {1, 78, 125}, - {1, 51, 86}, - {1, 31, 54}, - {1, 14, 23}}, - {{15, 199, 227}, - {3, 150, 204}, - {1, 91, 146}, - {1, 55, 95}, - {1, 30, 53}, - {1, 11, 20}}}, - {{{19, 55, 240}, {19, 59, 196}, {3, 52, 105}}, - {{41, 166, 207}, - {104, 153, 199}, - {31, 123, 181}, - {14, 101, 152}, - {5, 72, 106}, - {1, 36, 52}}, - {{35, 176, 211}, - {12, 131, 190}, - {2, 88, 144}, - {1, 60, 101}, - {1, 36, 60}, - {1, 16, 28}}, - {{28, 183, 213}, - {8, 134, 191}, - {1, 86, 142}, - {1, 56, 96}, - {1, 30, 53}, - {1, 12, 20}}, - {{20, 190, 215}, - {4, 135, 192}, - {1, 84, 139}, - {1, 53, 91}, - {1, 28, 49}, - {1, 11, 20}}, - {{13, 196, 216}, - {2, 137, 192}, - {1, 86, 143}, - {1, 57, 99}, - {1, 32, 56}, - {1, 13, 24}}}}, - {{{{211, 29, 217}, {96, 47, 156}, {22, 43, 87}}, - {{78, 120, 193}, - {111, 116, 186}, - {46, 102, 164}, - {15, 80, 128}, - {2, 49, 76}, - {1, 18, 28}}, - {{71, 161, 203}, - {42, 132, 192}, - {10, 98, 150}, - {3, 69, 109}, - {1, 44, 70}, - {1, 18, 29}}, - {{57, 186, 211}, - {30, 140, 196}, - {4, 93, 146}, - {1, 62, 102}, - {1, 38, 65}, - {1, 16, 27}}, - {{47, 199, 217}, - {14, 145, 196}, - {1, 88, 142}, - {1, 57, 98}, - {1, 36, 62}, - {1, 15, 26}}, - {{26, 219, 229}, - {5, 155, 207}, - {1, 94, 151}, - {1, 60, 104}, - {1, 36, 62}, - {1, 16, 28}}}, - {{{233, 29, 248}, {146, 47, 220}, {43, 52, 140}}, - {{100, 163, 232}, - {179, 161, 222}, - {63, 142, 204}, - {37, 113, 174}, - {26, 89, 137}, - {18, 68, 97}}, - {{85, 181, 230}, - {32, 146, 209}, - {7, 100, 164}, - {3, 71, 121}, - {1, 45, 77}, - {1, 18, 30}}, - {{65, 187, 230}, - {20, 148, 207}, - {2, 97, 159}, - {1, 68, 116}, - {1, 40, 70}, - {1, 14, 29}}, - {{40, 194, 227}, - {8, 147, 204}, - {1, 94, 155}, - {1, 65, 112}, - {1, 39, 66}, - {1, 14, 26}}, - {{16, 208, 228}, - {3, 151, 207}, - {1, 98, 160}, - {1, 67, 117}, - {1, 41, 74}, - {1, 17, 31}}}}}, - // 32x32 - {{{{{17, 38, 140}, {7, 34, 80}, {1, 17, 29}}, - {{37, 75, 128}, - {41, 76, 128}, - {26, 66, 116}, - {12, 52, 94}, - {2, 32, 55}, - {1, 10, 16}}, - {{50, 127, 154}, - {37, 109, 152}, - {16, 82, 121}, - {5, 59, 85}, - {1, 35, 54}, - {1, 13, 20}}, - {{40, 142, 167}, - {17, 110, 157}, - {2, 71, 112}, - {1, 44, 72}, - {1, 27, 45}, - {1, 11, 17}}, - {{30, 175, 188}, - {9, 124, 169}, - {1, 74, 116}, - {1, 48, 78}, - {1, 30, 49}, - {1, 11, 18}}, - {{10, 222, 223}, - {2, 150, 194}, - {1, 83, 128}, - {1, 48, 79}, - {1, 27, 45}, - {1, 11, 17}}}, - {{{36, 41, 235}, {29, 36, 193}, {10, 27, 111}}, - {{85, 165, 222}, - {177, 162, 215}, - {110, 135, 195}, - {57, 113, 168}, - {23, 83, 120}, - {10, 49, 61}}, - {{85, 190, 223}, - {36, 139, 200}, - {5, 90, 146}, - {1, 60, 103}, - {1, 38, 65}, - {1, 18, 30}}, - {{72, 202, 223}, - {23, 141, 199}, - {2, 86, 140}, - {1, 56, 97}, - {1, 36, 61}, - {1, 16, 27}}, - {{55, 218, 225}, - {13, 145, 200}, - {1, 86, 141}, - {1, 57, 99}, - {1, 35, 61}, - {1, 13, 22}}, - {{15, 235, 212}, - {1, 132, 184}, - {1, 84, 139}, - {1, 57, 97}, - {1, 34, 56}, - {1, 14, 23}}}}, - {{{{181, 21, 201}, {61, 37, 123}, {10, 38, 71}}, - {{47, 106, 172}, - {95, 104, 173}, - {42, 93, 159}, - {18, 77, 131}, - {4, 50, 81}, - {1, 17, 23}}, - {{62, 147, 199}, - {44, 130, 189}, - {28, 102, 154}, - {18, 75, 115}, - {2, 44, 65}, - {1, 12, 19}}, - {{55, 153, 210}, - {24, 130, 194}, - {3, 93, 146}, - {1, 61, 97}, - {1, 31, 50}, - {1, 10, 16}}, - {{49, 186, 223}, - {17, 148, 204}, - {1, 96, 142}, - {1, 53, 83}, - {1, 26, 44}, - {1, 11, 17}}, - {{13, 217, 212}, - {2, 136, 180}, - {1, 78, 124}, - {1, 50, 83}, - {1, 29, 49}, - {1, 14, 23}}}, - {{{197, 13, 247}, {82, 17, 222}, {25, 17, 162}}, - {{126, 186, 247}, - {234, 191, 243}, - {176, 177, 234}, - {104, 158, 220}, - {66, 128, 186}, - {55, 90, 137}}, - {{111, 197, 242}, - {46, 158, 219}, - {9, 104, 171}, - {2, 65, 125}, - {1, 44, 80}, - {1, 17, 91}}, - {{104, 208, 245}, - {39, 168, 224}, - {3, 109, 162}, - {1, 79, 124}, - {1, 50, 102}, - {1, 43, 102}}, - {{84, 220, 246}, - {31, 177, 231}, - {2, 115, 180}, - {1, 79, 134}, - {1, 55, 77}, - {1, 60, 79}}, - {{43, 243, 240}, - {8, 180, 217}, - {1, 115, 166}, - {1, 84, 121}, - {1, 51, 67}, - {1, 16, 6}}}}}}, - // skip_prob - {192, 128, 64}, - // inter_mode_probs - {{2, 173, 34}, - {7, 145, 85}, - {7, 166, 63}, - {7, 94, 66}, - {8, 64, 46}, - {17, 81, 31}, - {25, 29, 30}}, - // interp_filter_probs - {{235, 162}, {36, 255}, {34, 3}, {149, 144}}, - // is_inter_prob - {9, 102, 187, 225}, - // comp_mode_prob - {239, 183, 119, 96, 41}, - // single_ref_prob - {{33, 16}, {77, 74}, {142, 142}, {172, 170}, {238, 247}}, - // comp_ref_prob - {50, 126, 123, 221, 226}, - // y_mode_probs - {{65, 32, 18, 144, 162, 194, 41, 51, 98}, - {132, 68, 18, 165, 217, 196, 45, 40, 78}, - {173, 80, 19, 176, 240, 193, 64, 35, 46}, - {221, 135, 38, 194, 248, 121, 96, 85, 29}}, - // uv_mode_probs - {{120, 7, 76, 176, 208, 126, 28, 54, 103}, - {48, 12, 154, 155, 139, 90, 34, 117, 119}, - {67, 6, 25, 204, 243, 158, 13, 21, 96}, - {97, 5, 44, 131, 176, 139, 48, 68, 97}, - {83, 5, 42, 156, 111, 152, 26, 49, 152}, - {80, 5, 58, 178, 74, 83, 33, 62, 145}, - {86, 5, 32, 154, 192, 168, 14, 22, 163}, - {85, 5, 32, 156, 216, 148, 19, 29, 73}, - {77, 7, 64, 116, 132, 122, 37, 126, 120}, - {101, 21, 107, 181, 192, 103, 19, 67, 125}}, - // partition_probs - {{199, 122, 141}, - {147, 63, 159}, - {148, 133, 118}, - {121, 104, 114}, - {174, 73, 87}, - {92, 41, 83}, - {82, 99, 50}, - {53, 39, 39}, - {177, 58, 59}, - {68, 26, 63}, - {52, 79, 25}, - {17, 14, 12}, - {222, 34, 30}, - {72, 16, 44}, - {58, 32, 12}, - {10, 7, 6}}, - // mv_joint_probs - {32, 64, 96}, - // mv_sign_prob - {128, 128}, - // mv_class_probs - {{224, 144, 192, 168, 192, 176, 192, 198, 198, 245}, - {216, 128, 176, 160, 176, 176, 192, 198, 198, 208}}, - // mv_class0_bit_prob - {216, 208}, - // mv_bits_prob - {{136, 140, 148, 160, 176, 192, 224, 234, 234, 240}, - {136, 140, 148, 160, 176, 192, 224, 234, 234, 240}}, - // mv_class0_fr_probs - {{{128, 128, 64}, {96, 112, 64}}, {{128, 128, 64}, {96, 112, 64}}}, - // mv_fr_probs - {{64, 96, 64}, {64, 96, 64}}, - // mv_class0_hp_prob - {160, 160}, - // mv_hp_prob - {128, 128}, -}; - -// Helper function for Vp9Parser::ReadTileInfo. Defined as -// calc_min_log2_tile_cols in spec 6.2.14 Tile size calculation. -int GetMinLog2TileCols(int sb64_cols) { - const int kMaxTileWidthB64 = 64; - int min_log2 = 0; - while ((kMaxTileWidthB64 << min_log2) < sb64_cols) - min_log2++; - return min_log2; -} - -// Helper function for Vp9Parser::ReadTileInfo. Defined as -// calc_max_log2_tile_cols in spec 6.2.14 Tile size calculation. -int GetMaxLog2TileCols(int sb64_cols) { - const int kMinTileWidthB64 = 4; - int max_log2 = 1; - while ((sb64_cols >> max_log2) >= kMinTileWidthB64) - max_log2++; - return max_log2 - 1; -} - -} // namespace - -Vp9UncompressedHeaderParser::Vp9UncompressedHeaderParser( - Vp9Parser::Context* context) - : context_(context) {} - -uint8_t Vp9UncompressedHeaderParser::ReadProfile() { - uint8_t profile = 0; - - // LSB first. - if (reader_.ReadBool()) - profile |= 1; - if (reader_.ReadBool()) - profile |= 2; - if (profile > 2 && reader_.ReadBool()) - profile += 1; - return profile; -} - -// 6.2.1 Frame sync syntax -bool Vp9UncompressedHeaderParser::VerifySyncCode() { - const int kSyncCode = 0x498342; - if (reader_.ReadLiteral(8 * 3) != kSyncCode) { - DVLOG(1) << "Invalid frame sync code"; - return false; - } - return true; -} - -// 6.2.2 Color config syntax -bool Vp9UncompressedHeaderParser::ReadColorConfig(Vp9FrameHeader* fhdr) { - if (fhdr->profile == 2 || fhdr->profile == 3) { - fhdr->bit_depth = reader_.ReadBool() ? 12 : 10; - } else { - fhdr->bit_depth = 8; - } - - fhdr->color_space = static_cast<Vp9ColorSpace>(reader_.ReadLiteral(3)); - if (fhdr->color_space != Vp9ColorSpace::SRGB) { - fhdr->color_range = reader_.ReadBool(); - if (fhdr->profile == 1 || fhdr->profile == 3) { - fhdr->subsampling_x = reader_.ReadBool() ? 1 : 0; - fhdr->subsampling_y = reader_.ReadBool() ? 1 : 0; - if (fhdr->subsampling_x == 1 && fhdr->subsampling_y == 1) { - DVLOG(1) << "4:2:0 color not supported in profile 1 or 3"; - return false; - } - bool reserved = reader_.ReadBool(); - if (reserved) { - DVLOG(1) << "reserved bit set"; - return false; - } - } else { - fhdr->subsampling_x = fhdr->subsampling_y = 1; - } - } else { - fhdr->color_range = true; - if (fhdr->profile == 1 || fhdr->profile == 3) { - fhdr->subsampling_x = fhdr->subsampling_y = 0; - - bool reserved = reader_.ReadBool(); - if (reserved) { - DVLOG(1) << "reserved bit set"; - return false; - } - } else { - DVLOG(1) << "4:4:4 color not supported in profile 0 or 2"; - return false; - } - } - - return true; -} - -// 6.2.3 Frame size syntax -void Vp9UncompressedHeaderParser::ReadFrameSize(Vp9FrameHeader* fhdr) { - fhdr->frame_width = reader_.ReadLiteral(16) + 1; - fhdr->frame_height = reader_.ReadLiteral(16) + 1; -} - -// 6.2.4 Render size syntax -void Vp9UncompressedHeaderParser::ReadRenderSize(Vp9FrameHeader* fhdr) { - if (reader_.ReadBool()) { - fhdr->render_width = reader_.ReadLiteral(16) + 1; - fhdr->render_height = reader_.ReadLiteral(16) + 1; - } else { - fhdr->render_width = fhdr->frame_width; - fhdr->render_height = fhdr->frame_height; - } -} - -// 6.2.5 Frame size with refs syntax -bool Vp9UncompressedHeaderParser::ReadFrameSizeFromRefs(Vp9FrameHeader* fhdr) { - bool found_ref = false; - for (const auto& idx : fhdr->ref_frame_idx) { - found_ref = reader_.ReadBool(); - if (found_ref) { - const Vp9Parser::ReferenceSlot& ref = context_->GetRefSlot(idx); - DCHECK(ref.initialized); - fhdr->frame_width = ref.frame_width; - fhdr->frame_height = ref.frame_height; - - const unsigned kMaxDimension = 1u << 16; - DCHECK_LE(fhdr->frame_width, kMaxDimension); - DCHECK_LE(fhdr->frame_height, kMaxDimension); - break; - } - } - - if (!found_ref) - ReadFrameSize(fhdr); - - // 7.2.5 Frame size with refs semantics - bool has_valid_ref_frame = false; - for (const auto& idx : fhdr->ref_frame_idx) { - const Vp9Parser::ReferenceSlot& ref = context_->GetRefSlot(idx); - if (2 * fhdr->frame_width >= ref.frame_width && - 2 * fhdr->frame_height >= ref.frame_height && - fhdr->frame_width <= 16 * ref.frame_width && - fhdr->frame_height <= 16 * ref.frame_height) { - has_valid_ref_frame = true; - break; - } - } - if (!has_valid_ref_frame) { - DVLOG(1) << "There should be at least one reference frame meeting " - << "size conditions."; - return false; - } - - ReadRenderSize(fhdr); - return true; -} - -// 6.2.7 Interpolation filter syntax -Vp9InterpolationFilter Vp9UncompressedHeaderParser::ReadInterpolationFilter() { - if (reader_.ReadBool()) - return Vp9InterpolationFilter::SWITCHABLE; - - // The mapping table for next two bits. - const Vp9InterpolationFilter table[] = { - Vp9InterpolationFilter::EIGHTTAP_SMOOTH, Vp9InterpolationFilter::EIGHTTAP, - Vp9InterpolationFilter::EIGHTTAP_SHARP, Vp9InterpolationFilter::BILINEAR, - }; - return table[reader_.ReadLiteral(2)]; -} - -void Vp9UncompressedHeaderParser::SetupPastIndependence(Vp9FrameHeader* fhdr) { - memset(&context_->segmentation_, 0, sizeof(context_->segmentation_)); - ResetLoopfilter(); - fhdr->frame_context = kVp9DefaultFrameContext; - DCHECK(fhdr->frame_context.IsValid()); -} - -// 6.2.8 Loop filter params syntax -void Vp9UncompressedHeaderParser::ReadLoopFilterParams() { - Vp9LoopFilterParams& loop_filter = context_->loop_filter_; - - loop_filter.level = reader_.ReadLiteral(6); - loop_filter.sharpness = reader_.ReadLiteral(3); - loop_filter.delta_update = false; - - loop_filter.delta_enabled = reader_.ReadBool(); - if (loop_filter.delta_enabled) { - loop_filter.delta_update = reader_.ReadBool(); - if (loop_filter.delta_update) { - for (size_t i = 0; i < Vp9RefType::VP9_FRAME_MAX; i++) { - loop_filter.update_ref_deltas[i] = reader_.ReadBool(); - if (loop_filter.update_ref_deltas[i]) - loop_filter.ref_deltas[i] = reader_.ReadSignedLiteral(6); - } - - for (size_t i = 0; i < Vp9LoopFilterParams::kNumModeDeltas; i++) { - loop_filter.update_mode_deltas[i] = reader_.ReadBool(); - if (loop_filter.update_mode_deltas[i]) - loop_filter.mode_deltas[i] = reader_.ReadSignedLiteral(6); - } - } - } -} - -// 6.2.9 Quantization params syntax -void Vp9UncompressedHeaderParser::ReadQuantizationParams( - Vp9QuantizationParams* quants) { - quants->base_q_idx = reader_.ReadLiteral(8); - - quants->delta_q_y_dc = ReadDeltaQ(); - quants->delta_q_uv_dc = ReadDeltaQ(); - quants->delta_q_uv_ac = ReadDeltaQ(); -} - -// 6.2.10 Delta quantizer syntax -int8_t Vp9UncompressedHeaderParser::ReadDeltaQ() { - if (reader_.ReadBool()) - return reader_.ReadSignedLiteral(4); - return 0; -} - -// 6.2.11 Segmentation params syntax -bool Vp9UncompressedHeaderParser::ReadSegmentationParams() { - Vp9SegmentationParams& segmentation = context_->segmentation_; - segmentation.update_map = false; - segmentation.update_data = false; - - segmentation.enabled = reader_.ReadBool(); - if (!segmentation.enabled) - return true; - - segmentation.update_map = reader_.ReadBool(); - if (segmentation.update_map) { - for (auto& tree_prob : segmentation.tree_probs) { - tree_prob = ReadProb(); - } - - segmentation.temporal_update = reader_.ReadBool(); - for (auto& pred_prob : segmentation.pred_probs) { - pred_prob = segmentation.temporal_update ? ReadProb() : kVp9MaxProb; - } - } - - segmentation.update_data = reader_.ReadBool(); - if (segmentation.update_data) { - segmentation.abs_or_delta_update = reader_.ReadBool(); - - const int kFeatureDataBits[] = {8, 6, 2, 0}; - const bool kFeatureDataSigned[] = {true, true, false, false}; - - for (size_t i = 0; i < Vp9SegmentationParams::kNumSegments; i++) { - for (size_t j = 0; j < Vp9SegmentationParams::SEG_LVL_MAX; j++) { - int16_t data = 0; - segmentation.feature_enabled[i][j] = reader_.ReadBool(); - if (segmentation.feature_enabled[i][j]) { - data = reader_.ReadLiteral(kFeatureDataBits[j]); - if (kFeatureDataSigned[j]) - if (reader_.ReadBool()) { - // 7.2.9 - if (segmentation.abs_or_delta_update) { - DVLOG(1) << "feature_sign should be 0" - << " if abs_or_delta_update is 1"; - return false; - } - data = -data; - } - } - segmentation.feature_data[i][j] = data; - } - } - } - return true; -} - -// 6.2.12 Probability syntax -uint8_t Vp9UncompressedHeaderParser::ReadProb() { - return reader_.ReadBool() ? reader_.ReadLiteral(8) : kVp9MaxProb; -} - -// 6.2.13 Tile info syntax -bool Vp9UncompressedHeaderParser::ReadTileInfo(Vp9FrameHeader* fhdr) { - int sb64_cols = (fhdr->frame_width + 63) / 64; - - int min_log2_tile_cols = GetMinLog2TileCols(sb64_cols); - int max_log2_tile_cols = GetMaxLog2TileCols(sb64_cols); - - int max_ones = max_log2_tile_cols - min_log2_tile_cols; - fhdr->tile_cols_log2 = min_log2_tile_cols; - while (max_ones-- && reader_.ReadBool()) - fhdr->tile_cols_log2++; - - fhdr->tile_rows_log2 = reader_.ReadBool() ? 1 : 0; - if (fhdr->tile_rows_log2 > 0 && reader_.ReadBool()) - fhdr->tile_rows_log2++; - - // 7.2.11 Tile info semantics - if (fhdr->tile_cols_log2 > 6) { - DVLOG(1) << "tile_cols_log2 should be <= 6"; - return false; - } - - return true; -} - -void Vp9UncompressedHeaderParser::ResetLoopfilter() { - Vp9LoopFilterParams& loop_filter = context_->loop_filter_; - - loop_filter.delta_enabled = true; - loop_filter.delta_update = true; - - loop_filter.ref_deltas[VP9_FRAME_INTRA] = 1; - loop_filter.ref_deltas[VP9_FRAME_LAST] = 0; - loop_filter.ref_deltas[VP9_FRAME_GOLDEN] = -1; - loop_filter.ref_deltas[VP9_FRAME_ALTREF] = -1; - - memset(loop_filter.mode_deltas, 0, sizeof(loop_filter.mode_deltas)); -} - -// 6.2 Uncompressed header syntax -bool Vp9UncompressedHeaderParser::Parse(const uint8_t* stream, - off_t frame_size, - Vp9FrameHeader* fhdr) { - DVLOG(2) << "Vp9UncompressedHeaderParser::Parse"; - reader_.Initialize(stream, frame_size); - - fhdr->data = stream; - fhdr->frame_size = frame_size; - - // frame marker - if (reader_.ReadLiteral(2) != 0x2) { - DVLOG(1) << "frame marker shall be equal to 2"; - return false; - } - - fhdr->profile = ReadProfile(); - if (fhdr->profile >= kVp9MaxProfile) { - DVLOG(1) << "Unsupported bitstream profile"; - return false; - } - - fhdr->show_existing_frame = reader_.ReadBool(); - if (fhdr->show_existing_frame) { - fhdr->frame_to_show_map_idx = reader_.ReadLiteral(3); - fhdr->show_frame = true; - - if (!reader_.ConsumeTrailingBits()) { - DVLOG(1) << "trailing bits are not zero"; - return false; - } - if (!reader_.IsValid()) { - DVLOG(1) << "parser reads beyond the end of buffer"; - return false; - } - fhdr->uncompressed_header_size = reader_.GetBytesRead(); - fhdr->header_size_in_bytes = 0; - return true; - } - - fhdr->frame_type = static_cast<Vp9FrameHeader::FrameType>(reader_.ReadBool()); - fhdr->show_frame = reader_.ReadBool(); - fhdr->error_resilient_mode = reader_.ReadBool(); - - if (fhdr->IsKeyframe()) { - if (!VerifySyncCode()) - return false; - - if (!ReadColorConfig(fhdr)) - return false; - - ReadFrameSize(fhdr); - ReadRenderSize(fhdr); - fhdr->refresh_frame_flags = 0xff; - } else { - if (!fhdr->show_frame) - fhdr->intra_only = reader_.ReadBool(); - - if (!fhdr->error_resilient_mode) - fhdr->reset_frame_context = reader_.ReadLiteral(2); - - if (fhdr->intra_only) { - if (!VerifySyncCode()) - return false; - - if (fhdr->profile > 0) { - if (!ReadColorConfig(fhdr)) - return false; - } else { - fhdr->bit_depth = 8; - fhdr->color_space = Vp9ColorSpace::BT_601; - fhdr->subsampling_x = fhdr->subsampling_y = 1; - } - - fhdr->refresh_frame_flags = reader_.ReadLiteral(8); - - ReadFrameSize(fhdr); - ReadRenderSize(fhdr); - } else { - fhdr->refresh_frame_flags = reader_.ReadLiteral(8); - - static_assert(arraysize(fhdr->ref_frame_sign_bias) >= - Vp9RefType::VP9_FRAME_LAST + kVp9NumRefsPerFrame, - "ref_frame_sign_bias is not big enough"); - for (size_t i = 0; i < kVp9NumRefsPerFrame; i++) { - fhdr->ref_frame_idx[i] = reader_.ReadLiteral(kVp9NumRefFramesLog2); - fhdr->ref_frame_sign_bias[Vp9RefType::VP9_FRAME_LAST + i] = - reader_.ReadBool(); - - // 8.2 Frame order constraints - // ref_frame_idx[i] refers to an earlier decoded frame. - const Vp9Parser::ReferenceSlot& ref = - context_->GetRefSlot(fhdr->ref_frame_idx[i]); - if (!ref.initialized) { - DVLOG(1) << "ref_frame_idx[" << i - << "]=" << static_cast<int>(fhdr->ref_frame_idx[i]) - << " refers to unused frame"; - return false; - } - - // 7.2 Uncompressed header semantics - // the selected reference frames match the current frame in bit depth, - // profile, chroma subsampling, and color space. - if (ref.profile != fhdr->profile) { - DVLOG(1) << "profile of referenced frame mismatch"; - return false; - } - if (i == 0) { - // Below fields are not specified for inter-frame in header, so copy - // them from referenced frame. - fhdr->bit_depth = ref.bit_depth; - fhdr->color_space = ref.color_space; - fhdr->subsampling_x = ref.subsampling_x; - fhdr->subsampling_y = ref.subsampling_y; - } else { - if (fhdr->bit_depth != ref.bit_depth) { - DVLOG(1) << "bit_depth of referenced frame mismatch"; - return false; - } - if (fhdr->color_space != ref.color_space) { - DVLOG(1) << "color_space of referenced frame mismatch"; - return false; - } - if (fhdr->subsampling_x != ref.subsampling_x || - fhdr->subsampling_y != ref.subsampling_y) { - DVLOG(1) << "chroma subsampling of referenced frame mismatch"; - return false; - } - } - } - - if (!ReadFrameSizeFromRefs(fhdr)) - return false; - - fhdr->allow_high_precision_mv = reader_.ReadBool(); - fhdr->interpolation_filter = ReadInterpolationFilter(); - } - } - - if (fhdr->error_resilient_mode) { - fhdr->refresh_frame_context = false; - fhdr->frame_parallel_decoding_mode = true; - } else { - fhdr->refresh_frame_context = reader_.ReadBool(); - fhdr->frame_parallel_decoding_mode = reader_.ReadBool(); - } - - fhdr->frame_context_idx_to_save_probs = fhdr->frame_context_idx = - reader_.ReadLiteral(kVp9NumFrameContextsLog2); - - if (fhdr->IsIntra()) { - SetupPastIndependence(fhdr); - if (fhdr->IsKeyframe() || fhdr->error_resilient_mode || - fhdr->reset_frame_context == 3) { - for (size_t i = 0; i < kVp9NumFrameContexts; ++i) - context_->UpdateFrameContext(i, fhdr->frame_context); - } else if (fhdr->reset_frame_context == 2) { - context_->UpdateFrameContext(fhdr->frame_context_idx, - fhdr->frame_context); - } - fhdr->frame_context_idx = 0; - } - - ReadLoopFilterParams(); - ReadQuantizationParams(&fhdr->quant_params); - if (!ReadSegmentationParams()) - return false; - - if (!ReadTileInfo(fhdr)) - return false; - - fhdr->header_size_in_bytes = reader_.ReadLiteral(16); - if (fhdr->header_size_in_bytes == 0) { - DVLOG(1) << "invalid header size"; - return false; - } - - if (!reader_.ConsumeTrailingBits()) { - DVLOG(1) << "trailing bits are not zero"; - return false; - } - if (!reader_.IsValid()) { - DVLOG(1) << "parser reads beyond the end of buffer"; - return false; - } - fhdr->uncompressed_header_size = reader_.GetBytesRead(); - - return true; -} - -} // namespace media diff --git a/accel/vp9_uncompressed_header_parser.h b/accel/vp9_uncompressed_header_parser.h deleted file mode 100644 index 6780d38..0000000 --- a/accel/vp9_uncompressed_header_parser.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// Note: ported from Chromium commit head: e5a9a62 - -#ifndef VP9_UNCOMPRESSED_HEADER_PARSER_H_ -#define VP9_UNCOMPRESSED_HEADER_PARSER_H_ - -#include "vp9_parser.h" -#include "vp9_raw_bits_reader.h" - -namespace media { - -class Vp9UncompressedHeaderParser { - public: - Vp9UncompressedHeaderParser(Vp9Parser::Context* context); - - // Parses VP9 uncompressed header in |stream| with |frame_size| into |fhdr|. - // Returns true if no error. - bool Parse(const uint8_t* stream, off_t frame_size, Vp9FrameHeader* fhdr); - - private: - uint8_t ReadProfile(); - bool VerifySyncCode(); - bool ReadColorConfig(Vp9FrameHeader* fhdr); - void ReadFrameSize(Vp9FrameHeader* fhdr); - bool ReadFrameSizeFromRefs(Vp9FrameHeader* fhdr); - void ReadRenderSize(Vp9FrameHeader* fhdr); - Vp9InterpolationFilter ReadInterpolationFilter(); - void ResetLoopfilter(); - void SetupPastIndependence(Vp9FrameHeader* fhdr); - void ReadLoopFilterParams(); - void ReadQuantizationParams(Vp9QuantizationParams* quants); - int8_t ReadDeltaQ(); - uint8_t ReadProb(); - bool ReadSegmentationParams(); - bool ReadTileInfo(Vp9FrameHeader* fhdr); - - // Raw bits reader for uncompressed frame header. - Vp9RawBitsReader reader_; - - Vp9Parser::Context* context_; - - DISALLOW_COPY_AND_ASSIGN(Vp9UncompressedHeaderParser); -}; - -} // namespace media - -#endif // VP9_UNCOMPRESSED_HEADER_PARSER_H_ diff --git a/common/Android.bp b/common/Android.bp index 45334d0..e9c5fe3 100644 --- a/common/Android.bp +++ b/common/Android.bp @@ -1,3 +1,12 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_v4l2_codec2_license" + // to get the below license kinds: + // SPDX-license-identifier-BSD + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + cc_library { name: "libv4l2_codec2_common", vendor: true, @@ -7,10 +16,16 @@ cc_library { ], srcs: [ + "Common.cpp", "EncodeHelpers.cpp", "FormatConverter.cpp", + "Fourcc.cpp", + "NalParser.cpp", "V4L2ComponentCommon.cpp", "VideoTypes.cpp", + "V4L2Device.cpp", + "V4L2DevicePoller.cpp", + "VideoPixelFormat.cpp", ], export_include_dirs: [ @@ -21,9 +36,9 @@ cc_library { "libchrome", "libcutils", "liblog", + "libstagefright_foundation", "libui", "libutils", - "libv4l2_codec2_accel" ], static_libs: [ diff --git a/common/Common.cpp b/common/Common.cpp new file mode 100644 index 0000000..79243ec --- /dev/null +++ b/common/Common.cpp @@ -0,0 +1,35 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <v4l2_codec2/common/Common.h> + +#include <base/numerics/safe_math.h> + +namespace android { + +bool contains(const Rect& rect1, const Rect& rect2) { + return (rect2.left >= rect1.left && rect2.right <= rect1.right && rect2.top >= rect1.top && + rect2.bottom <= rect1.bottom); +} + +std::string toString(const Rect& rect) { + return std::string("(") + std::to_string(rect.left) + "," + std::to_string(rect.top) + ") " + + std::to_string(rect.width()) + "x" + std::to_string(rect.height()); +} + +std::optional<int> getArea(const ui::Size& size) { + base::CheckedNumeric<int> checked_area = size.width; + checked_area *= size.height; + return checked_area.IsValid() ? std::optional<int>(checked_area.ValueOrDie()) : std::nullopt; +} + +bool isEmpty(const ui::Size& size) { + return !size.width || !size.height; +} + +std::string toString(const ui::Size& size) { + return std::to_string(size.width) + "x" + std::to_string(size.height); +} + +} // namespace android diff --git a/common/EncodeHelpers.cpp b/common/EncodeHelpers.cpp index 757d064..4575197 100644 --- a/common/EncodeHelpers.cpp +++ b/common/EncodeHelpers.cpp @@ -7,81 +7,51 @@ #include <v4l2_codec2/common/EncodeHelpers.h> -#include <string.h> +#include <linux/v4l2-controls.h> #include <C2AllocatorGralloc.h> #include <cutils/native_handle.h> #include <ui/GraphicBuffer.h> #include <utils/Log.h> -namespace android { +#include <v4l2_codec2/common/NalParser.h> -media::VideoCodecProfile c2ProfileToVideoCodecProfile(C2Config::profile_t profile) { - switch (profile) { - case C2Config::PROFILE_AVC_BASELINE: - return media::VideoCodecProfile::H264PROFILE_BASELINE; - case C2Config::PROFILE_AVC_MAIN: - return media::VideoCodecProfile::H264PROFILE_MAIN; - case C2Config::PROFILE_AVC_EXTENDED: - return media::VideoCodecProfile::H264PROFILE_EXTENDED; - case C2Config::PROFILE_AVC_HIGH: - return media::VideoCodecProfile::H264PROFILE_HIGH; - case C2Config::PROFILE_AVC_HIGH_10: - return media::VideoCodecProfile::H264PROFILE_HIGH10PROFILE; - case C2Config::PROFILE_AVC_HIGH_422: - return media::VideoCodecProfile::H264PROFILE_HIGH422PROFILE; - case C2Config::PROFILE_AVC_HIGH_444_PREDICTIVE: - return media::VideoCodecProfile::H264PROFILE_HIGH444PREDICTIVEPROFILE; - case C2Config::PROFILE_AVC_SCALABLE_BASELINE: - return media::VideoCodecProfile::H264PROFILE_SCALABLEBASELINE; - case C2Config::PROFILE_AVC_SCALABLE_HIGH: - return media::VideoCodecProfile::H264PROFILE_SCALABLEHIGH; - case C2Config::PROFILE_AVC_STEREO_HIGH: - return media::VideoCodecProfile::H264PROFILE_STEREOHIGH; - case C2Config::PROFILE_AVC_MULTIVIEW_HIGH: - return media::VideoCodecProfile::H264PROFILE_MULTIVIEWHIGH; - default: - ALOGE("Unrecognizable C2 profile (value = 0x%x)...", profile); - return media::VideoCodecProfile::VIDEO_CODEC_PROFILE_UNKNOWN; - } -} +namespace android { -uint8_t c2LevelToLevelIDC(C2Config::level_t level) { +uint8_t c2LevelToV4L2Level(C2Config::level_t level) { switch (level) { case C2Config::LEVEL_AVC_1: - return 10; + return V4L2_MPEG_VIDEO_H264_LEVEL_1_0; case C2Config::LEVEL_AVC_1B: - return 9; + return V4L2_MPEG_VIDEO_H264_LEVEL_1B; case C2Config::LEVEL_AVC_1_1: - return 11; + return V4L2_MPEG_VIDEO_H264_LEVEL_1_1; case C2Config::LEVEL_AVC_1_2: - return 12; + return V4L2_MPEG_VIDEO_H264_LEVEL_1_2; case C2Config::LEVEL_AVC_1_3: - return 13; + return V4L2_MPEG_VIDEO_H264_LEVEL_1_3; case C2Config::LEVEL_AVC_2: - return 20; + return V4L2_MPEG_VIDEO_H264_LEVEL_2_0; case C2Config::LEVEL_AVC_2_1: - return 21; + return V4L2_MPEG_VIDEO_H264_LEVEL_2_1; case C2Config::LEVEL_AVC_2_2: - return 22; + return V4L2_MPEG_VIDEO_H264_LEVEL_2_2; case C2Config::LEVEL_AVC_3: - return 30; + return V4L2_MPEG_VIDEO_H264_LEVEL_3_0; case C2Config::LEVEL_AVC_3_1: - return 31; + return V4L2_MPEG_VIDEO_H264_LEVEL_3_1; case C2Config::LEVEL_AVC_3_2: - return 32; + return V4L2_MPEG_VIDEO_H264_LEVEL_3_2; case C2Config::LEVEL_AVC_4: - return 40; + return V4L2_MPEG_VIDEO_H264_LEVEL_4_0; case C2Config::LEVEL_AVC_4_1: - return 41; + return V4L2_MPEG_VIDEO_H264_LEVEL_4_1; case C2Config::LEVEL_AVC_4_2: - return 42; + return V4L2_MPEG_VIDEO_H264_LEVEL_4_2; case C2Config::LEVEL_AVC_5: - return 50; + return V4L2_MPEG_VIDEO_H264_LEVEL_5_0; case C2Config::LEVEL_AVC_5_1: - return 51; - case C2Config::LEVEL_AVC_5_2: - return 52; + return V4L2_MPEG_VIDEO_H264_LEVEL_5_1; default: ALOGE("Unrecognizable C2 level (value = 0x%x)...", level); return 0; @@ -98,20 +68,24 @@ android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block) { height, format, 1, usage, stride); native_handle_delete(grallocHandle); + // Pass SW flag so that ARCVM returns the guest buffer dimensions instead + // of the host buffer dimensions. This means we will have to convert the + // return value from ptrs to buffer offsets ourselves. android_ycbcr ycbcr = {}; - // Usage flag without SW_READ/WRITE bits. - constexpr uint32_t kNonSWLockUsage = 0; - int32_t status = buf->lockYCbCr(kNonSWLockUsage, &ycbcr); + int32_t status = buf->lockYCbCr(GRALLOC_USAGE_SW_READ_OFTEN, &ycbcr); if (status != OK) ALOGE("lockYCbCr is failed: %d", (int)status); buf->unlock(); + + uintptr_t y = reinterpret_cast<uintptr_t>(ycbcr.y); + ycbcr.y = nullptr; + ycbcr.cb = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(ycbcr.cb) - y); + ycbcr.cr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(ycbcr.cr) - y); + return ycbcr; } void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, size_t length) { - constexpr uint8_t kTypeSeqParamSet = 7; - constexpr uint8_t kTypePicParamSet = 8; - // Android frameworks needs 4 bytes start code. constexpr uint8_t kStartCode[] = {0x00, 0x00, 0x00, 0x01}; constexpr int kStartCodeLength = 4; @@ -127,9 +101,9 @@ void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, co NalParser parser(data, length); while (parser.locateNextNal()) { if (parser.length() == 0) continue; - uint8_t nalType = *parser.data() & 0x1f; + uint8_t nalType = parser.type(); ALOGV("find next NAL: type=%d, length=%zu", nalType, parser.length()); - if (nalType != kTypeSeqParamSet && nalType != kTypePicParamSet) continue; + if (nalType != NalParser::kSPSType && nalType != NalParser::kPPSType) continue; if (tmpOutput + kStartCodeLength + parser.length() > tmpConfigDataEnd) { ALOGE("Buffer overflow on extracting codec config data (length=%zu)", length); @@ -147,32 +121,4 @@ void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, co std::memcpy((*csd)->m.value, tmpConfigData.get(), configDataLength); } -NalParser::NalParser(const uint8_t* data, size_t length) - : mCurrNalDataPos(data), mDataEnd(data + length) { - mNextNalStartCodePos = findNextStartCodePos(); -} - -bool NalParser::locateNextNal() { - if (mNextNalStartCodePos == mDataEnd) return false; - mCurrNalDataPos = mNextNalStartCodePos + kNalStartCodeLength; // skip start code. - mNextNalStartCodePos = findNextStartCodePos(); - return true; -} - -const uint8_t* NalParser::data() const { - return mCurrNalDataPos; -} - -size_t NalParser::length() const { - if (mNextNalStartCodePos == mDataEnd) return mDataEnd - mCurrNalDataPos; - size_t length = mNextNalStartCodePos - mCurrNalDataPos; - // The start code could be 3 or 4 bytes, i.e., 0x000001 or 0x00000001. - return *(mNextNalStartCodePos - 1) == 0x00 ? length - 1 : length; -} - -const uint8_t* NalParser::findNextStartCodePos() const { - return std::search(mCurrNalDataPos, mDataEnd, kNalStartCode, - kNalStartCode + kNalStartCodeLength); -} - } // namespace android diff --git a/common/FormatConverter.cpp b/common/FormatConverter.cpp index 9ab9161..d694bd1 100644 --- a/common/FormatConverter.cpp +++ b/common/FormatConverter.cpp @@ -29,9 +29,8 @@ namespace android { namespace { // The constant expression of mapping the pixel format conversion pair (src, dst) to a unique // integer. -constexpr int convertMap(media::VideoPixelFormat src, media::VideoPixelFormat dst) { - return static_cast<int>(src) * - (static_cast<int>(media::VideoPixelFormat::PIXEL_FORMAT_MAX) + 1) + +constexpr int convertMap(VideoPixelFormat src, VideoPixelFormat dst) { + return static_cast<int>(src) * (static_cast<int>(VideoPixelFormat::UNKNOWN) + 1) + static_cast<int>(dst); } @@ -93,12 +92,11 @@ std::unique_ptr<ImplDefinedToRGBXMap> ImplDefinedToRGBXMap::Create( } // static -std::unique_ptr<FormatConverter> FormatConverter::Create(media::VideoPixelFormat outFormat, - const media::Size& visibleSize, +std::unique_ptr<FormatConverter> FormatConverter::Create(VideoPixelFormat outFormat, + const ui::Size& visibleSize, uint32_t inputCount, - const media::Size& codedSize) { - if (outFormat != media::VideoPixelFormat::PIXEL_FORMAT_I420 && - outFormat != media::VideoPixelFormat::PIXEL_FORMAT_NV12) { + const ui::Size& codedSize) { + if (outFormat != VideoPixelFormat::I420 && outFormat != VideoPixelFormat::NV12) { ALOGE("Unsupported output format: %d", static_cast<int32_t>(outFormat)); return nullptr; } @@ -111,12 +109,11 @@ std::unique_ptr<FormatConverter> FormatConverter::Create(media::VideoPixelFormat return converter; } -c2_status_t FormatConverter::initialize(media::VideoPixelFormat outFormat, - const media::Size& visibleSize, uint32_t inputCount, - const media::Size& codedSize) { +c2_status_t FormatConverter::initialize(VideoPixelFormat outFormat, const ui::Size& visibleSize, + uint32_t inputCount, const ui::Size& codedSize) { ALOGV("initialize(out_format=%s, visible_size=%dx%d, input_count=%u, coded_size=%dx%d)", - media::VideoPixelFormatToString(outFormat).c_str(), visibleSize.width(), - visibleSize.height(), inputCount, codedSize.width(), codedSize.height()); + videoPixelFormatToString(outFormat).c_str(), visibleSize.width, visibleSize.height, + inputCount, codedSize.width, codedSize.height); std::shared_ptr<C2BlockPool> pool; c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool); @@ -126,7 +123,7 @@ c2_status_t FormatConverter::initialize(media::VideoPixelFormat outFormat, } HalPixelFormat halFormat; - if (outFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) { + if (outFormat == VideoPixelFormat::I420) { // Android HAL format doesn't have I420, we use YV12 instead and swap U and V data while // conversion to perform I420. halFormat = HalPixelFormat::YV12; @@ -137,7 +134,7 @@ c2_status_t FormatConverter::initialize(media::VideoPixelFormat outFormat, uint32_t bufferCount = std::max(inputCount, kMinInputBufferCount); for (uint32_t i = 0; i < bufferCount; i++) { std::shared_ptr<C2GraphicBlock> block; - status = pool->fetchGraphicBlock(codedSize.width(), codedSize.height(), + status = pool->fetchGraphicBlock(codedSize.width, codedSize.height, static_cast<uint32_t>(halFormat), {(C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE), static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)}, @@ -153,10 +150,10 @@ c2_status_t FormatConverter::initialize(media::VideoPixelFormat outFormat, mOutFormat = outFormat; mVisibleSize = visibleSize; - mTempPlaneU = std::unique_ptr<uint8_t[]>( - new uint8_t[mVisibleSize.width() * mVisibleSize.height() / 4]); - mTempPlaneV = std::unique_ptr<uint8_t[]>( - new uint8_t[mVisibleSize.width() * mVisibleSize.height() / 4]); + mTempPlaneU = + std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]); + mTempPlaneV = + std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]); return C2_OK; } @@ -201,7 +198,7 @@ C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex, const int dstStrideV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for I420 const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12 - media::VideoPixelFormat inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_UNKNOWN; + VideoPixelFormat inputFormat = VideoPixelFormat::UNKNOWN; *status = C2_OK; if (inputLayout.type == C2PlanarLayout::TYPE_YUV) { const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y]; @@ -211,10 +208,9 @@ C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex, const int srcStrideU = inputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc; if (inputLayout.rootPlanes == 3) { - inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_YV12; + inputFormat = VideoPixelFormat::YV12; } else if (inputLayout.rootPlanes == 2) { - inputFormat = (srcV > srcU) ? media::VideoPixelFormat::PIXEL_FORMAT_NV12 - : media::VideoPixelFormat::PIXEL_FORMAT_NV21; + inputFormat = (srcV > srcU) ? VideoPixelFormat::NV12 : VideoPixelFormat::NV21; } if (inputFormat == mOutFormat) { @@ -224,79 +220,73 @@ C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex, } switch (convertMap(inputFormat, mOutFormat)) { - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12, - media::VideoPixelFormat::PIXEL_FORMAT_I420): + case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::I420): libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, - dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width(), - mVisibleSize.height()); + dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width, + mVisibleSize.height); break; - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12, - media::VideoPixelFormat::PIXEL_FORMAT_NV12): + case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::NV12): libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, - dstStrideY, dstUV, dstStrideUV, mVisibleSize.width(), - mVisibleSize.height()); + dstStrideY, dstUV, dstStrideUV, mVisibleSize.width, + mVisibleSize.height); break; - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV12, - media::VideoPixelFormat::PIXEL_FORMAT_I420): + case convertMap(VideoPixelFormat::NV12, VideoPixelFormat::I420): libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU, - dstStrideU, dstV, dstStrideV, mVisibleSize.width(), - mVisibleSize.height()); + dstStrideU, dstV, dstStrideV, mVisibleSize.width, + mVisibleSize.height); break; - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21, - media::VideoPixelFormat::PIXEL_FORMAT_I420): + case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::I420): libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU, - dstStrideU, dstV, dstStrideV, mVisibleSize.width(), - mVisibleSize.height()); + dstStrideU, dstV, dstStrideV, mVisibleSize.width, + mVisibleSize.height); break; - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21, - media::VideoPixelFormat::PIXEL_FORMAT_NV12): - libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width(), - mVisibleSize.height()); - copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width() / 2, - mVisibleSize.height() / 2); - copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2, - mVisibleSize.width() / 2, mVisibleSize.height() / 2); + case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::NV12): + ALOGV("%s(): Converting PIXEL_FORMAT_NV21 -> PIXEL_FORMAT_NV12", __func__); + libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width, + mVisibleSize.height); + copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width / 2, + mVisibleSize.height / 2); + copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2, mVisibleSize.width / 2, + mVisibleSize.height / 2); break; default: ALOGE("Unsupported pixel format conversion from %s to %s", - media::VideoPixelFormatToString(inputFormat).c_str(), - media::VideoPixelFormatToString(mOutFormat).c_str()); + videoPixelFormatToString(inputFormat).c_str(), + videoPixelFormatToString(mOutFormat).c_str()); *status = C2_CORRUPTED; return inputBlock; // This is actually redundant and should not be used. } } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) { // There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe // BGRA_8888 is not used now? - inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_ABGR; + inputFormat = VideoPixelFormat::ABGR; const uint8_t* srcRGB = (idMap) ? idMap->addr() : inputView.data()[C2PlanarLayout::PLANE_R]; const int srcStrideRGB = (idMap) ? idMap->rowInc() : inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc; switch (convertMap(inputFormat, mOutFormat)) { - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR, - media::VideoPixelFormat::PIXEL_FORMAT_I420): + case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::I420): libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV, - dstStrideV, mVisibleSize.width(), mVisibleSize.height()); + dstStrideV, mVisibleSize.width, mVisibleSize.height); break; - case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR, - media::VideoPixelFormat::PIXEL_FORMAT_NV12): { + case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::NV12): { // There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to // I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from // temporary planes to dst-UV interleavedly. - const int tempStride = mVisibleSize.width() / 2; + const int tempStride = mVisibleSize.width / 2; libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, mTempPlaneU.get(), - tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width(), - mVisibleSize.height()); + tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width, + mVisibleSize.height); libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride, - dstUV, dstStrideUV, mVisibleSize.width() / 2, - mVisibleSize.height() / 2); + dstUV, dstStrideUV, mVisibleSize.width / 2, + mVisibleSize.height / 2); break; } default: ALOGE("Unsupported pixel format conversion from %s to %s", - media::VideoPixelFormatToString(inputFormat).c_str(), - media::VideoPixelFormatToString(mOutFormat).c_str()); + videoPixelFormatToString(inputFormat).c_str(), + videoPixelFormatToString(mOutFormat).c_str()); *status = C2_CORRUPTED; return inputBlock; // This is actually redundant and should not be used. } @@ -307,10 +297,10 @@ C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex, } ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex, - media::VideoPixelFormatToString(inputFormat).c_str()); + videoPixelFormatToString(inputFormat).c_str()); entry->mAssociatedFrameIndex = frameIndex; mAvailableQueue.pop(); - return outputBlock->share(C2Rect(mVisibleSize.width(), mVisibleSize.height()), C2Fence()); + return outputBlock->share(C2Rect(mVisibleSize.width, mVisibleSize.height), C2Fence()); } c2_status_t FormatConverter::returnBlock(uint64_t frameIndex) { diff --git a/common/Fourcc.cpp b/common/Fourcc.cpp new file mode 100644 index 0000000..f7d3efd --- /dev/null +++ b/common/Fourcc.cpp @@ -0,0 +1,280 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <v4l2_codec2/common/Fourcc.h> + +#include <linux/videodev2.h> + +#include <utils/Log.h> + +namespace android { + +Fourcc::Fourcc(Fourcc::Value fourcc) : mValue(fourcc) {} +Fourcc::~Fourcc() = default; +Fourcc& Fourcc::operator=(const Fourcc& other) = default; + +// static +std::optional<Fourcc> Fourcc::fromUint32(uint32_t fourcc) { + switch (fourcc) { + case AR24: + case AB24: + case XR24: + case XB24: + case RGB4: + case YU12: + case YV12: + case YM12: + case YM21: + case YUYV: + case NV12: + case NV21: + case NM12: + case NM21: + case YM16: + case MT21: + case MM21: + return Fourcc(static_cast<Value>(fourcc)); + } + ALOGE("Unmapped fourcc: %s", fourccToString(fourcc).c_str()); + return std::nullopt; +} + +// static +std::optional<Fourcc> Fourcc::fromVideoPixelFormat(VideoPixelFormat pixelFormat, + bool singlePlanar) { + if (singlePlanar) { + switch (pixelFormat) { + case VideoPixelFormat::ARGB: + return Fourcc(AR24); + case VideoPixelFormat::ABGR: + return Fourcc(AB24); + case VideoPixelFormat::XRGB: + return Fourcc(XR24); + case VideoPixelFormat::XBGR: + return Fourcc(XB24); + case VideoPixelFormat::BGRA: + return Fourcc(RGB4); + case VideoPixelFormat::I420: + return Fourcc(YU12); + case VideoPixelFormat::YV12: + return Fourcc(YV12); + case VideoPixelFormat::YUY2: + return Fourcc(YUYV); + case VideoPixelFormat::NV12: + return Fourcc(NV12); + case VideoPixelFormat::NV21: + return Fourcc(NV21); + case VideoPixelFormat::I422: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I444: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + case VideoPixelFormat::Y16: + case VideoPixelFormat::P016LE: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + case VideoPixelFormat::UNKNOWN: + break; + } + } else { + switch (pixelFormat) { + case VideoPixelFormat::I420: + return Fourcc(YM12); + case VideoPixelFormat::YV12: + return Fourcc(YM21); + case VideoPixelFormat::NV12: + return Fourcc(NM12); + case VideoPixelFormat::I422: + return Fourcc(YM16); + case VideoPixelFormat::NV21: + return Fourcc(NM21); + case VideoPixelFormat::I420A: + case VideoPixelFormat::I444: + case VideoPixelFormat::YUY2: + case VideoPixelFormat::ARGB: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + case VideoPixelFormat::Y16: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::P016LE: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::UNKNOWN: + break; + } + } + ALOGE("Unmapped %s for %s", videoPixelFormatToString(pixelFormat).c_str(), + singlePlanar ? "single-planar" : "multi-planar"); + return std::nullopt; +} + +VideoPixelFormat Fourcc::toVideoPixelFormat() const { + switch (mValue) { + case AR24: + return VideoPixelFormat::ARGB; + case AB24: + return VideoPixelFormat::ABGR; + case XR24: + return VideoPixelFormat::XRGB; + case XB24: + return VideoPixelFormat::XBGR; + case RGB4: + return VideoPixelFormat::BGRA; + case YU12: + case YM12: + return VideoPixelFormat::I420; + case YV12: + case YM21: + return VideoPixelFormat::YV12; + case YUYV: + return VideoPixelFormat::YUY2; + case NV12: + case NM12: + return VideoPixelFormat::NV12; + case NV21: + case NM21: + return VideoPixelFormat::NV21; + case YM16: + return VideoPixelFormat::I422; + // V4L2_PIX_FMT_MT21C is only used for MT8173 hardware video decoder output + // and should be converted by MT8173 image processor for compositor to + // render. Since it is an intermediate format for video decoder, + // VideoPixelFormat shall not have its mapping. However, we need to create a + // VideoFrameLayout for the format to process the intermediate frame. Hence + // we map V4L2_PIX_FMT_MT21C to PIXEL_FORMAT_NV12 as their layout are the + // same. + case MT21: + // V4L2_PIX_FMT_MM21 is used for MT8183 hardware video decoder. It is + // similar to V4L2_PIX_FMT_MT21C but is not compressed ; thus it can also + // be mapped to PIXEL_FORMAT_NV12. + case MM21: + return VideoPixelFormat::NV12; + } + + ALOGE("Unmapped Fourcc: %s", toString().c_str()); + return VideoPixelFormat::UNKNOWN; +} + +// static +std::optional<Fourcc> Fourcc::fromV4L2PixFmt(uint32_t v4l2PixFmt) { + // We can do that because we adopt the same internal definition of Fourcc as + // V4L2. + return fromUint32(v4l2PixFmt); +} + +uint32_t Fourcc::toV4L2PixFmt() const { + // Note that we can do that because we adopt the same internal definition of + // Fourcc as V4L2. + return static_cast<uint32_t>(mValue); +} + +std::optional<Fourcc> Fourcc::toSinglePlanar() const { + switch (mValue) { + case AR24: + case AB24: + case XR24: + case XB24: + case RGB4: + case YU12: + case YV12: + case YUYV: + case NV12: + case NV21: + return Fourcc(mValue); + case YM12: + return Fourcc(YU12); + case YM21: + return Fourcc(YV12); + case NM12: + return Fourcc(NV12); + case NM21: + return Fourcc(NV21); + case YM16: + case MT21: + case MM21: + return std::nullopt; + } +} + +bool operator!=(const Fourcc& lhs, const Fourcc& rhs) { + return !(lhs == rhs); +} + +bool Fourcc::isMultiPlanar() const { + switch (mValue) { + case AR24: + case AB24: + case XR24: + case XB24: + case RGB4: + case YU12: + case YV12: + case YUYV: + case NV12: + case NV21: + return false; + case YM12: + case YM21: + case NM12: + case NM21: + case YM16: + case MT21: + case MM21: + return true; + } +} + +std::string Fourcc::toString() const { + return fourccToString(static_cast<uint32_t>(mValue)); +} + +static_assert(Fourcc::AR24 == V4L2_PIX_FMT_ABGR32, "Mismatch Fourcc"); +#ifdef V4L2_PIX_FMT_RGBA32 +// V4L2_PIX_FMT_RGBA32 is defined since v5.2 +static_assert(Fourcc::AB24 == V4L2_PIX_FMT_RGBA32, "Mismatch Fourcc"); +#endif // V4L2_PIX_FMT_RGBA32 +static_assert(Fourcc::XR24 == V4L2_PIX_FMT_XBGR32, "Mismatch Fourcc"); +#ifdef V4L2_PIX_FMT_RGBX32 +// V4L2_PIX_FMT_RGBX32 is defined since v5.2 +static_assert(Fourcc::XB24 == V4L2_PIX_FMT_RGBX32, "Mismatch Fourcc"); +#endif // V4L2_PIX_FMT_RGBX32 +static_assert(Fourcc::RGB4 == V4L2_PIX_FMT_RGB32, "Mismatch Fourcc"); +static_assert(Fourcc::YU12 == V4L2_PIX_FMT_YUV420, "Mismatch Fourcc"); +static_assert(Fourcc::YV12 == V4L2_PIX_FMT_YVU420, "Mismatch Fourcc"); +static_assert(Fourcc::YM12 == V4L2_PIX_FMT_YUV420M, "Mismatch Fourcc"); +static_assert(Fourcc::YM21 == V4L2_PIX_FMT_YVU420M, "Mismatch Fourcc"); +static_assert(Fourcc::YUYV == V4L2_PIX_FMT_YUYV, "Mismatch Fourcc"); +static_assert(Fourcc::NV12 == V4L2_PIX_FMT_NV12, "Mismatch Fourcc"); +static_assert(Fourcc::NV21 == V4L2_PIX_FMT_NV21, "Mismatch Fourcc"); +static_assert(Fourcc::NM12 == V4L2_PIX_FMT_NV12M, "Mismatch Fourcc"); +static_assert(Fourcc::NM21 == V4L2_PIX_FMT_NV21M, "Mismatch Fourcc"); +static_assert(Fourcc::YM16 == V4L2_PIX_FMT_YUV422M, "Mismatch Fourcc"); +static_assert(Fourcc::MT21 == V4L2_PIX_FMT_MT21C, "Mismatch Fourcc"); +#ifdef V4L2_PIX_FMT_MM21 +// V4L2_PIX_FMT_MM21 is not yet upstreamed. +static_assert(Fourcc::MM21 == V4L2_PIX_FMT_MM21, "Mismatch Fourcc"); +#endif // V4L2_PIX_FMT_MM21 + +} // namespace android diff --git a/common/NalParser.cpp b/common/NalParser.cpp new file mode 100644 index 0000000..3216574 --- /dev/null +++ b/common/NalParser.cpp @@ -0,0 +1,217 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NalParser" + +#include <v4l2_codec2/common/NalParser.h> + +#include <algorithm> + +#include <media/stagefright/foundation/ABitReader.h> +#include <utils/Log.h> + +namespace android { + +namespace { + +enum H264ProfileIDC { + kProfileIDCAVLC444 = 44, + kProfileIDScalableBaseline = 83, + kProfileIDScalableHigh = 86, + kProfileIDCHigh = 100, + kProfileIDHigh10 = 110, + kProfileIDSMultiviewHigh = 118, + kProfileIDHigh422 = 122, + kProfileIDStereoHigh = 128, + kProfileIDHigh444Predictive = 244, +}; + +constexpr uint32_t kYUV444Idc = 3; + +// Read unsigned int encoded with exponential-golomb. +uint32_t parseUE(ABitReader* br) { + uint32_t numZeroes = 0; + while (br->getBits(1) == 0) { + ++numZeroes; + } + uint32_t val = br->getBits(numZeroes); + return val + (1u << numZeroes) - 1; +} + +// Read signed int encoded with exponential-golomb. +int32_t parseSE(ABitReader* br) { + uint32_t codeNum = parseUE(br); + return (codeNum & 1) ? (codeNum + 1) >> 1 : -static_cast<int32_t>(codeNum >> 1); +} + +// Skip a H.264 sequence scaling list in the specified bitstream. +void skipScalingList(ABitReader* br, size_t scalingListSize) { + size_t nextScale = 8; + size_t lastScale = 8; + for (size_t j = 0; j < scalingListSize; ++j) { + if (nextScale != 0) { + int32_t deltaScale = parseSE(br); // delta_sl + if (deltaScale < -128) { + ALOGW("delta scale (%d) is below range, capping to -128", deltaScale); + deltaScale = -128; + } else if (deltaScale > 127) { + ALOGW("delta scale (%d) is above range, capping to 127", deltaScale); + deltaScale = 127; + } + nextScale = (lastScale + (deltaScale + 256)) % 256; + } + lastScale = (nextScale == 0) ? lastScale : nextScale; + } +} + +// Skip the H.264 sequence scaling matrix in the specified bitstream. +void skipScalingMatrix(ABitReader* br, size_t numScalingLists) { + for (size_t i = 0; i < numScalingLists; ++i) { + if (br->getBits(1)) { // seq_scaling_list_present_flag + if (i < 6) { + skipScalingList(br, 16); + } else { + skipScalingList(br, 64); + } + } + } +} + +} // namespace + +NalParser::NalParser(const uint8_t* data, size_t length) + : mCurrNalDataPos(data), mDataEnd(data + length) { + mNextNalStartCodePos = findNextStartCodePos(); +} + +bool NalParser::locateNextNal() { + if (mNextNalStartCodePos == mDataEnd) return false; + mCurrNalDataPos = mNextNalStartCodePos + kNalStartCodeLength; // skip start code. + mNextNalStartCodePos = findNextStartCodePos(); + return true; +} + +bool NalParser::locateSPS() { + while (locateNextNal()) { + if (length() == 0) continue; + if (type() != kSPSType) continue; + return true; + } + + return false; +} + +const uint8_t* NalParser::data() const { + return mCurrNalDataPos; +} + +size_t NalParser::length() const { + if (mNextNalStartCodePos == mDataEnd) return mDataEnd - mCurrNalDataPos; + size_t length = mNextNalStartCodePos - mCurrNalDataPos; + // The start code could be 3 or 4 bytes, i.e., 0x000001 or 0x00000001. + return *(mNextNalStartCodePos - 1) == 0x00 ? length - 1 : length; +} + +uint8_t NalParser::type() const { + // First byte is forbidden_zero_bit (1) + nal_ref_idc (2) + nal_unit_type (5) + constexpr uint8_t kNALTypeMask = 0x1f; + return *mCurrNalDataPos & kNALTypeMask; +} + +const uint8_t* NalParser::findNextStartCodePos() const { + return std::search(mCurrNalDataPos, mDataEnd, kNalStartCode, + kNalStartCode + kNalStartCodeLength); +} + +bool NalParser::findCodedColorAspects(ColorAspects* colorAspects) { + ALOG_ASSERT(colorAspects); + ALOG_ASSERT(type() == kSPSType); + + // Unfortunately we can't directly jump to the Video Usability Information (VUI) parameters that + // contain the color aspects. We need to parse the entire SPS header up until the values we + // need. + + // Skip first byte containing type. + ABitReader br(mCurrNalDataPos + 1, length() - 1); + + uint32_t profileIDC = br.getBits(8); // profile_idc + br.skipBits(16); // constraint flags + reserved bits + level_idc + parseUE(&br); // seq_parameter_set_id + + if (profileIDC == kProfileIDCHigh || profileIDC == kProfileIDHigh10 || + profileIDC == kProfileIDHigh422 || profileIDC == kProfileIDHigh444Predictive || + profileIDC == kProfileIDCAVLC444 || profileIDC == kProfileIDScalableBaseline || + profileIDC == kProfileIDScalableHigh || profileIDC == kProfileIDSMultiviewHigh || + profileIDC == kProfileIDStereoHigh) { + uint32_t chromaFormatIDC = parseUE(&br); + if (chromaFormatIDC == kYUV444Idc) { // chroma_format_idc + br.skipBits(1); // separate_colour_plane_flag + } + parseUE(&br); // bit_depth_luma_minus8 + parseUE(&br); // bit_depth_chroma_minus8 + br.skipBits(1); // lossless_qpprime_y_zero_flag + + if (br.getBits(1)) { // seq_scaling_matrix_present_flag + const size_t numScalingLists = (chromaFormatIDC != kYUV444Idc) ? 8 : 12; + skipScalingMatrix(&br, numScalingLists); + } + } + + parseUE(&br); // log2_max_frame_num_minus4 + uint32_t pictureOrderCountType = parseUE(&br); // pic_order_cnt_type + if (pictureOrderCountType == 0) { + parseUE(&br); // log2_max_pic_order_cnt_lsb_minus4 + } else if (pictureOrderCountType == 1) { + br.skipBits(1); // delta_pic_order_always_zero_flag + parseSE(&br); // offset_for_non_ref_pic + parseSE(&br); // offset_for_top_to_bottom_field + uint32_t numReferenceFrames = parseUE(&br); // num_ref_frames_in_pic_order_cnt_cycle + for (uint32_t i = 0; i < numReferenceFrames; ++i) { + parseUE(&br); // offset_for_ref_frame + } + } + + parseUE(&br); // num_ref_frames + br.skipBits(1); // gaps_in_frame_num_value_allowed_flag + parseUE(&br); // pic_width_in_mbs_minus1 + parseUE(&br); // pic_height_in_map_units_minus1 + if (!br.getBits(1)) { // frame_mbs_only_flag + br.skipBits(1); // mb_adaptive_frame_field_flag + } + br.skipBits(1); // direct_8x8_inference_flag + + if (br.getBits(1)) { // frame_cropping_flag + parseUE(&br); // frame_cropping_rect_left_offset + parseUE(&br); // frame_cropping_rect_right_offset + parseUE(&br); // frame_cropping_rect_top_offset + parseUE(&br); // frame_cropping_rect_bottom_offset + } + + if (br.getBits(1)) { // vui_parameters_present_flag + if (br.getBits(1)) { // VUI aspect_ratio_info_present_flag + if (br.getBits(8) == 255) { // VUI aspect_ratio_idc == extended sample aspect ratio + br.skipBits(32); // VUI sar_width + sar_height + } + } + + if (br.getBits(1)) { // VUI overscan_info_present_flag + br.skipBits(1); // VUI overscan_appropriate_flag + } + if (br.getBits(1)) { // VUI video_signal_type_present_flag + br.skipBits(3); // VUI video_format + colorAspects->fullRange = br.getBits(1); // VUI video_full_range_flag + if (br.getBits(1)) { // VUI color_description_present_flag + colorAspects->primaries = br.getBits(8); // VUI colour_primaries + colorAspects->transfer = br.getBits(8); // VUI transfer_characteristics + colorAspects->coeffs = br.getBits(8); // VUI matrix_coefficients + return !br.overRead(); + } + } + } + + return false; +} + +} // namespace android diff --git a/common/V4L2ComponentCommon.cpp b/common/V4L2ComponentCommon.cpp index 86448a8..518b489 100644 --- a/common/V4L2ComponentCommon.cpp +++ b/common/V4L2ComponentCommon.cpp @@ -12,6 +12,8 @@ namespace android { const std::string V4L2ComponentName::kH264Encoder = "c2.v4l2.avc.encoder"; +const std::string V4L2ComponentName::kVP8Encoder = "c2.v4l2.vp8.encoder"; +const std::string V4L2ComponentName::kVP9Encoder = "c2.v4l2.vp9.encoder"; const std::string V4L2ComponentName::kH264Decoder = "c2.v4l2.avc.decoder"; const std::string V4L2ComponentName::kVP8Decoder = "c2.v4l2.vp8.decoder"; @@ -22,16 +24,16 @@ const std::string V4L2ComponentName::kVP9SecureDecoder = "c2.v4l2.vp9.decoder.se // static bool V4L2ComponentName::isValid(const char* name) { - return name == kH264Encoder || name == kH264Decoder || name == kVP8Decoder || - name == kVP9Decoder || name == kH264SecureDecoder || name == kVP8SecureDecoder || - name == kVP9SecureDecoder; + return name == kH264Encoder || name == kVP8Encoder || name == kVP9Encoder || + name == kH264Decoder || name == kVP8Decoder || name == kVP9Decoder || + name == kH264SecureDecoder || name == kVP8SecureDecoder || name == kVP9SecureDecoder; } // static bool V4L2ComponentName::isEncoder(const char* name) { ALOG_ASSERT(isValid(name)); - return name == kH264Encoder; + return name == kH264Encoder || name == kVP8Encoder || name == kVP9Encoder; } } // namespace android diff --git a/common/V4L2Device.cpp b/common/V4L2Device.cpp new file mode 100644 index 0000000..a31d82b --- /dev/null +++ b/common/V4L2Device.cpp @@ -0,0 +1,2010 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: 2f13d62f0c0d +// Note: Added some missing defines that are only defined in newer kernel +// versions (e.g. V4L2_PIX_FMT_VP8_FRAME) + +//#define LOG_NDEBUG 0 +#define LOG_TAG "V4L2Device" + +#include <v4l2_codec2/common/V4L2Device.h> + +#include <fcntl.h> +#include <inttypes.h> +#include <linux/media.h> +#include <linux/videodev2.h> +#include <poll.h> +#include <string.h> +#include <sys/eventfd.h> +#include <sys/ioctl.h> +#include <sys/mman.h> + +#include <algorithm> +#include <mutex> +#include <set> +#include <sstream> + +#include <base/bind.h> +#include <base/numerics/safe_conversions.h> +#include <base/posix/eintr_wrapper.h> +#include <base/strings/stringprintf.h> +#include <base/thread_annotations.h> +#include <utils/Log.h> + +#include <v4l2_codec2/common/Fourcc.h> +#include <v4l2_codec2/common/VideoPixelFormat.h> + +// VP8 parsed frames +#ifndef V4L2_PIX_FMT_VP8_FRAME +#define V4L2_PIX_FMT_VP8_FRAME v4l2_fourcc('V', 'P', '8', 'F') +#endif + +// VP9 parsed frames +#ifndef V4L2_PIX_FMT_VP9_FRAME +#define V4L2_PIX_FMT_VP9_FRAME v4l2_fourcc('V', 'P', '9', 'F') +#endif + +// H264 parsed slices +#ifndef V4L2_PIX_FMT_H264_SLICE +#define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') +#endif + +namespace android { + +struct v4l2_format buildV4L2Format(const enum v4l2_buf_type type, uint32_t fourcc, + const ui::Size& size, size_t buffer_size, uint32_t stride) { + struct v4l2_format format; + memset(&format, 0, sizeof(format)); + format.type = type; + format.fmt.pix_mp.pixelformat = fourcc; + format.fmt.pix_mp.width = size.width; + format.fmt.pix_mp.height = size.height; + format.fmt.pix_mp.num_planes = V4L2Device::getNumPlanesOfV4L2PixFmt(fourcc); + format.fmt.pix_mp.plane_fmt[0].sizeimage = buffer_size; + + // When the image format is planar the bytesperline value applies to the first plane and is + // divided by the same factor as the width field for the other planes. + format.fmt.pix_mp.plane_fmt[0].bytesperline = stride; + + return format; +} + +V4L2ExtCtrl::V4L2ExtCtrl(uint32_t id) { + memset(&ctrl, 0, sizeof(ctrl)); + ctrl.id = id; +} + +V4L2ExtCtrl::V4L2ExtCtrl(uint32_t id, int32_t val) : V4L2ExtCtrl(id) { + ctrl.value = val; +} + +// Class used to store the state of a buffer that should persist between reference creations. This +// includes: +// * Result of initial VIDIOC_QUERYBUF ioctl, +// * Plane mappings. +// +// Also provides helper functions. +class V4L2Buffer { +public: + static std::unique_ptr<V4L2Buffer> create(scoped_refptr<V4L2Device> device, + enum v4l2_buf_type type, enum v4l2_memory memory, + const struct v4l2_format& format, size_t bufferId); + ~V4L2Buffer(); + + V4L2Buffer(const V4L2Buffer&) = delete; + V4L2Buffer& operator=(const V4L2Buffer&) = delete; + + void* getPlaneMapping(const size_t plane); + size_t getMemoryUsage() const; + const struct v4l2_buffer& v4l2_buffer() const { return mV4l2Buffer; } + +private: + V4L2Buffer(scoped_refptr<V4L2Device> device, enum v4l2_buf_type type, enum v4l2_memory memory, + const struct v4l2_format& format, size_t bufferId); + bool query(); + + scoped_refptr<V4L2Device> mDevice; + std::vector<void*> mPlaneMappings; + + // V4L2 data as queried by QUERYBUF. + struct v4l2_buffer mV4l2Buffer; + // WARNING: do not change this to a vector or something smaller than VIDEO_MAX_PLANES, otherwise + // the Tegra libv4l2 will write data beyond the number of allocated planes, resulting in memory + // corruption. + struct v4l2_plane mV4l2Planes[VIDEO_MAX_PLANES]; + + struct v4l2_format mFormat __attribute__((unused)); +}; + +std::unique_ptr<V4L2Buffer> V4L2Buffer::create(scoped_refptr<V4L2Device> device, + enum v4l2_buf_type type, enum v4l2_memory memory, + const struct v4l2_format& format, size_t bufferId) { + // Not using std::make_unique because constructor is private. + std::unique_ptr<V4L2Buffer> buffer(new V4L2Buffer(device, type, memory, format, bufferId)); + + if (!buffer->query()) return nullptr; + + return buffer; +} + +V4L2Buffer::V4L2Buffer(scoped_refptr<V4L2Device> device, enum v4l2_buf_type type, + enum v4l2_memory memory, const struct v4l2_format& format, size_t bufferId) + : mDevice(device), mFormat(format) { + ALOG_ASSERT(V4L2_TYPE_IS_MULTIPLANAR(type)); + ALOG_ASSERT(format.fmt.pix_mp.num_planes <= base::size(mV4l2Planes)); + + memset(mV4l2Planes, 0, sizeof(mV4l2Planes)); + memset(&mV4l2Buffer, 0, sizeof(mV4l2Buffer)); + mV4l2Buffer.m.planes = mV4l2Planes; + // Just in case we got more planes than we want. + mV4l2Buffer.length = + std::min(static_cast<size_t>(format.fmt.pix_mp.num_planes), base::size(mV4l2Planes)); + mV4l2Buffer.index = bufferId; + mV4l2Buffer.type = type; + mV4l2Buffer.memory = memory; + mV4l2Buffer.memory = V4L2_MEMORY_DMABUF; + mPlaneMappings.resize(mV4l2Buffer.length); +} + +V4L2Buffer::~V4L2Buffer() { + if (mV4l2Buffer.memory == V4L2_MEMORY_MMAP) { + for (size_t i = 0; i < mPlaneMappings.size(); i++) { + if (mPlaneMappings[i] != nullptr) { + mDevice->munmap(mPlaneMappings[i], mV4l2Buffer.m.planes[i].length); + } + } + } +} + +bool V4L2Buffer::query() { + int ret = mDevice->ioctl(VIDIOC_QUERYBUF, &mV4l2Buffer); + if (ret) { + ALOGE("VIDIOC_QUERYBUF failed"); + return false; + } + + DCHECK(mPlaneMappings.size() == mV4l2Buffer.length); + + return true; +} + +void* V4L2Buffer::getPlaneMapping(const size_t plane) { + if (plane >= mPlaneMappings.size()) { + ALOGE("Invalid plane %zu requested.", plane); + return nullptr; + } + + void* p = mPlaneMappings[plane]; + if (p) { + return p; + } + + // Do this check here to avoid repeating it after a buffer has been successfully mapped (we know + // we are of MMAP type by then). + if (mV4l2Buffer.memory != V4L2_MEMORY_MMAP) { + ALOGE("Cannot create mapping on non-MMAP buffer"); + return nullptr; + } + + p = mDevice->mmap(NULL, mV4l2Buffer.m.planes[plane].length, PROT_READ | PROT_WRITE, MAP_SHARED, + mV4l2Buffer.m.planes[plane].m.mem_offset); + if (p == MAP_FAILED) { + ALOGE("mmap() failed: "); + return nullptr; + } + + mPlaneMappings[plane] = p; + return p; +} + +size_t V4L2Buffer::getMemoryUsage() const { + size_t usage = 0; + for (size_t i = 0; i < mV4l2Buffer.length; i++) { + usage += mV4l2Buffer.m.planes[i].length; + } + return usage; +} + +// A thread-safe pool of buffer indexes, allowing buffers to be obtained and returned from different +// threads. All the methods of this class are thread-safe. Users should keep a scoped_refptr to +// instances of this class in order to ensure the list remains alive as long as they need it. +class V4L2BuffersList : public base::RefCountedThreadSafe<V4L2BuffersList> { +public: + V4L2BuffersList() = default; + + V4L2BuffersList(const V4L2BuffersList&) = delete; + V4L2BuffersList& operator=(const V4L2BuffersList&) = delete; + + // Return a buffer to this list. Also can be called to set the initial pool of buffers. + // Note that it is illegal to return the same buffer twice. + void returnBuffer(size_t bufferId); + // Get any of the buffers in the list. There is no order guarantee whatsoever. + std::optional<size_t> getFreeBuffer(); + // Get the buffer with specified index. + std::optional<size_t> getFreeBuffer(size_t requestedBufferId); + // Number of buffers currently in this list. + size_t size() const; + +private: + friend class base::RefCountedThreadSafe<V4L2BuffersList>; + ~V4L2BuffersList() = default; + + mutable std::mutex mLock; + std::set<size_t> mFreeBuffers GUARDED_BY(mLock); +}; + +void V4L2BuffersList::returnBuffer(size_t bufferId) { + std::lock_guard<std::mutex> lock(mLock); + + auto inserted = mFreeBuffers.emplace(bufferId); + if (!inserted.second) { + ALOGE("Returning buffer failed"); + } +} + +std::optional<size_t> V4L2BuffersList::getFreeBuffer() { + std::lock_guard<std::mutex> lock(mLock); + + auto iter = mFreeBuffers.begin(); + if (iter == mFreeBuffers.end()) { + ALOGV("No free buffer available!"); + return std::nullopt; + } + + size_t bufferId = *iter; + mFreeBuffers.erase(iter); + + return bufferId; +} + +std::optional<size_t> V4L2BuffersList::getFreeBuffer(size_t requestedBufferId) { + std::lock_guard<std::mutex> lock(mLock); + + return (mFreeBuffers.erase(requestedBufferId) > 0) ? std::make_optional(requestedBufferId) + : std::nullopt; +} + +size_t V4L2BuffersList::size() const { + std::lock_guard<std::mutex> lock(mLock); + + return mFreeBuffers.size(); +} + +// Module-private class that let users query/write V4L2 buffer information. It also makes some +// private V4L2Queue methods available to this module only. +class V4L2BufferRefBase { +public: + V4L2BufferRefBase(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue); + ~V4L2BufferRefBase(); + + V4L2BufferRefBase(const V4L2BufferRefBase&) = delete; + V4L2BufferRefBase& operator=(const V4L2BufferRefBase&) = delete; + + bool queueBuffer(); + void* getPlaneMapping(const size_t plane); + + // Checks that the number of passed FDs is adequate for the current format and buffer + // configuration. Only useful for DMABUF buffers. + bool checkNumFDsForFormat(const size_t numFds) const; + + // Data from the buffer, that users can query and/or write. + struct v4l2_buffer mV4l2Buffer; + // WARNING: do not change this to a vector or something smaller than VIDEO_MAX_PLANES, otherwise + // the Tegra libv4l2 will write data beyond the number of allocated planes, resulting in memory + // corruption. + struct v4l2_plane mV4l2Planes[VIDEO_MAX_PLANES]; + +private: + size_t bufferId() const { return mV4l2Buffer.index; } + + friend class V4L2WritableBufferRef; + // A weak pointer to the queue this buffer belongs to. Will remain valid as long as the + // underlying V4L2 buffer is valid too. This can only be accessed from the sequence protected by + // sequence_checker_. Thread-safe methods (like ~V4L2BufferRefBase) must *never* access this. + base::WeakPtr<V4L2Queue> mQueue; + // Where to return this buffer if it goes out of scope without being queued. + scoped_refptr<V4L2BuffersList> mReturnTo; + bool queued = false; + + SEQUENCE_CHECKER(mSequenceChecker); +}; + +V4L2BufferRefBase::V4L2BufferRefBase(const struct v4l2_buffer& v4l2Buffer, + base::WeakPtr<V4L2Queue> queue) + : mQueue(std::move(queue)), mReturnTo(mQueue->mFreeBuffers) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(V4L2_TYPE_IS_MULTIPLANAR(v4l2Buffer.type)); + ALOG_ASSERT(v4l2Buffer.length <= base::size(mV4l2Planes)); + ALOG_ASSERT(mReturnTo); + + memcpy(&mV4l2Buffer, &v4l2Buffer, sizeof(mV4l2Buffer)); + memcpy(mV4l2Planes, v4l2Buffer.m.planes, sizeof(struct v4l2_plane) * v4l2Buffer.length); + mV4l2Buffer.m.planes = mV4l2Planes; +} + +V4L2BufferRefBase::~V4L2BufferRefBase() { + // We are the last reference and are only accessing the thread-safe mReturnTo, so we are safe + // to call from any sequence. If we have been queued, then the queue is our owner so we don't + // need to return to the free buffers list. + if (!queued) mReturnTo->returnBuffer(bufferId()); +} + +bool V4L2BufferRefBase::queueBuffer() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (!mQueue) return false; + + queued = mQueue->queueBuffer(&mV4l2Buffer); + + return queued; +} + +void* V4L2BufferRefBase::getPlaneMapping(const size_t plane) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (!mQueue) return nullptr; + + return mQueue->mBuffers[bufferId()]->getPlaneMapping(plane); +} + +bool V4L2BufferRefBase::checkNumFDsForFormat(const size_t numFds) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (!mQueue) return false; + + // We have not used SetFormat(), assume this is ok. + // Hopefully we standardize SetFormat() in the future. + if (!mQueue->mCurrentFormat) return true; + + const size_t requiredFds = mQueue->mCurrentFormat->fmt.pix_mp.num_planes; + // Sanity check. + ALOG_ASSERT(mV4l2Buffer.length == requiredFds); + if (numFds < requiredFds) { + ALOGE("Insufficient number of FDs given for the current format. " + "%zu provided, %zu required.", + numFds, requiredFds); + return false; + } + + const auto* planes = mV4l2Buffer.m.planes; + for (size_t i = mV4l2Buffer.length - 1; i >= numFds; --i) { + // Assume that an fd is a duplicate of a previous plane's fd if offset != 0. Otherwise, if + // offset == 0, return error as it is likely pointing to a new plane. + if (planes[i].data_offset == 0) { + ALOGE("Additional dmabuf fds point to a new buffer."); + return false; + } + } + + return true; +} + +V4L2WritableBufferRef::V4L2WritableBufferRef(const struct v4l2_buffer& v4l2Buffer, + base::WeakPtr<V4L2Queue> queue) + : mBufferData(std::make_unique<V4L2BufferRefBase>(v4l2Buffer, std::move(queue))) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); +} + +V4L2WritableBufferRef::V4L2WritableBufferRef(V4L2WritableBufferRef&& other) + : mBufferData(std::move(other.mBufferData)) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(other.mSequenceChecker); +} + +V4L2WritableBufferRef::~V4L2WritableBufferRef() { + // Only valid references should be sequence-checked + if (mBufferData) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + } +} + +V4L2WritableBufferRef& V4L2WritableBufferRef::operator=(V4L2WritableBufferRef&& other) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK_CALLED_ON_VALID_SEQUENCE(other.mSequenceChecker); + + if (this == &other) return *this; + + mBufferData = std::move(other.mBufferData); + + return *this; +} + +enum v4l2_memory V4L2WritableBufferRef::memory() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return static_cast<enum v4l2_memory>(mBufferData->mV4l2Buffer.memory); +} + +bool V4L2WritableBufferRef::doQueue() && { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + bool queued = mBufferData->queueBuffer(); + + // Clear our own reference. + mBufferData.reset(); + + return queued; +} + +bool V4L2WritableBufferRef::queueMMap() && { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + // Move ourselves so our data gets freed no matter when we return + V4L2WritableBufferRef self(std::move(*this)); + + if (self.memory() != V4L2_MEMORY_MMAP) { + ALOGE("Called on invalid buffer type!"); + return false; + } + + return std::move(self).doQueue(); +} + +bool V4L2WritableBufferRef::queueUserPtr(const std::vector<void*>& ptrs) && { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + // Move ourselves so our data gets freed no matter when we return + V4L2WritableBufferRef self(std::move(*this)); + + if (self.memory() != V4L2_MEMORY_USERPTR) { + ALOGE("Called on invalid buffer type!"); + return false; + } + + if (ptrs.size() != self.planesCount()) { + ALOGE("Provided %zu pointers while we require %u.", ptrs.size(), + self.mBufferData->mV4l2Buffer.length); + return false; + } + + for (size_t i = 0; i < ptrs.size(); i++) { + self.mBufferData->mV4l2Buffer.m.planes[i].m.userptr = + reinterpret_cast<unsigned long>(ptrs[i]); + } + + return std::move(self).doQueue(); +} + +bool V4L2WritableBufferRef::queueDMABuf(const std::vector<int>& fds) && { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + // Move ourselves so our data gets freed no matter when we return + V4L2WritableBufferRef self(std::move(*this)); + + if (self.memory() != V4L2_MEMORY_DMABUF) { + ALOGE("Called on invalid buffer type!"); + return false; + } + + if (!self.mBufferData->checkNumFDsForFormat(fds.size())) return false; + + size_t numPlanes = self.planesCount(); + for (size_t i = 0; i < numPlanes; i++) self.mBufferData->mV4l2Buffer.m.planes[i].m.fd = fds[i]; + + return std::move(self).doQueue(); +} + +size_t V4L2WritableBufferRef::planesCount() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.length; +} + +size_t V4L2WritableBufferRef::getPlaneSize(const size_t plane) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return 0; + } + + return mBufferData->mV4l2Buffer.m.planes[plane].length; +} + +void V4L2WritableBufferRef::setPlaneSize(const size_t plane, const size_t size) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + enum v4l2_memory mem = memory(); + if (mem == V4L2_MEMORY_MMAP) { + ALOG_ASSERT(mBufferData->mV4l2Buffer.m.planes[plane].length == size); + return; + } + ALOG_ASSERT(mem == V4L2_MEMORY_USERPTR || mem == V4L2_MEMORY_DMABUF); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return; + } + + mBufferData->mV4l2Buffer.m.planes[plane].length = size; +} + +void* V4L2WritableBufferRef::getPlaneMapping(const size_t plane) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->getPlaneMapping(plane); +} + +void V4L2WritableBufferRef::setTimeStamp(const struct timeval& timestamp) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + mBufferData->mV4l2Buffer.timestamp = timestamp; +} + +const struct timeval& V4L2WritableBufferRef::getTimeStamp() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.timestamp; +} + +void V4L2WritableBufferRef::setPlaneBytesUsed(const size_t plane, const size_t bytesUsed) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return; + } + + if (bytesUsed > getPlaneSize(plane)) { + ALOGE("Set bytes used %zu larger than plane size %zu.", bytesUsed, getPlaneSize(plane)); + return; + } + + mBufferData->mV4l2Buffer.m.planes[plane].bytesused = bytesUsed; +} + +size_t V4L2WritableBufferRef::getPlaneBytesUsed(const size_t plane) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return 0; + } + + return mBufferData->mV4l2Buffer.m.planes[plane].bytesused; +} + +void V4L2WritableBufferRef::setPlaneDataOffset(const size_t plane, const size_t dataOffset) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return; + } + + mBufferData->mV4l2Buffer.m.planes[plane].data_offset = dataOffset; +} + +size_t V4L2WritableBufferRef::bufferId() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.index; +} + +V4L2ReadableBuffer::V4L2ReadableBuffer(const struct v4l2_buffer& v4l2Buffer, + base::WeakPtr<V4L2Queue> queue) + : mBufferData(std::make_unique<V4L2BufferRefBase>(v4l2Buffer, std::move(queue))) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); +} + +V4L2ReadableBuffer::~V4L2ReadableBuffer() { + // This method is thread-safe. Since we are the destructor, we are guaranteed to be called from + // the only remaining reference to us. Also, we are just calling the destructor of buffer_data_, + // which is also thread-safe. + ALOG_ASSERT(mBufferData); +} + +bool V4L2ReadableBuffer::isLast() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.flags & V4L2_BUF_FLAG_LAST; +} + +bool V4L2ReadableBuffer::isKeyframe() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.flags & V4L2_BUF_FLAG_KEYFRAME; +} + +struct timeval V4L2ReadableBuffer::getTimeStamp() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.timestamp; +} + +size_t V4L2ReadableBuffer::planesCount() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.length; +} + +const void* V4L2ReadableBuffer::getPlaneMapping(const size_t plane) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + DCHECK(mBufferData); + + return mBufferData->getPlaneMapping(plane); +} + +size_t V4L2ReadableBuffer::getPlaneBytesUsed(const size_t plane) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return 0; + } + + return mBufferData->mV4l2Planes[plane].bytesused; +} + +size_t V4L2ReadableBuffer::getPlaneDataOffset(const size_t plane) const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + if (plane >= planesCount()) { + ALOGE("Invalid plane %zu requested.", plane); + return 0; + } + + return mBufferData->mV4l2Planes[plane].data_offset; +} + +size_t V4L2ReadableBuffer::bufferId() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(mBufferData); + + return mBufferData->mV4l2Buffer.index; +} + +// This class is used to expose buffer reference classes constructors to this module. This is to +// ensure that nobody else can create buffer references. +class V4L2BufferRefFactory { +public: + static V4L2WritableBufferRef CreateWritableRef(const struct v4l2_buffer& v4l2Buffer, + base::WeakPtr<V4L2Queue> queue) { + return V4L2WritableBufferRef(v4l2Buffer, std::move(queue)); + } + + static V4L2ReadableBufferRef CreateReadableRef(const struct v4l2_buffer& v4l2Buffer, + base::WeakPtr<V4L2Queue> queue) { + return new V4L2ReadableBuffer(v4l2Buffer, std::move(queue)); + } +}; + +//// Helper macros that print the queue type with logs. +#define ALOGEQ(fmt, ...) ALOGE("(%s)" fmt, V4L2Device::v4L2BufferTypeToString(mType), ##__VA_ARGS__) +#define ALOGVQ(fmt, ...) ALOGD("(%s)" fmt, V4L2Device::v4L2BufferTypeToString(mType), ##__VA_ARGS__) + +V4L2Queue::V4L2Queue(scoped_refptr<V4L2Device> dev, enum v4l2_buf_type type, + base::OnceClosure destroyCb) + : mType(type), mDevice(dev), mDestroyCb(std::move(destroyCb)) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); +} + +V4L2Queue::~V4L2Queue() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (mIsStreaming) { + ALOGEQ("Queue is still streaming, trying to stop it..."); + streamoff(); + } + + ALOG_ASSERT(mQueuedBuffers.empty()); + ALOG_ASSERT(!mFreeBuffers); + + if (!mBuffers.empty()) { + ALOGEQ("Buffers are still allocated, trying to deallocate them..."); + deallocateBuffers(); + } + + std::move(mDestroyCb).Run(); +} + +std::optional<struct v4l2_format> V4L2Queue::setFormat(uint32_t fourcc, const ui::Size& size, + size_t bufferSize, uint32_t stride) { + struct v4l2_format format = buildV4L2Format(mType, fourcc, size, bufferSize, stride); + if (mDevice->ioctl(VIDIOC_S_FMT, &format) != 0 || format.fmt.pix_mp.pixelformat != fourcc) { + ALOGEQ("Failed to set format (format_fourcc=0x%" PRIx32 ")", fourcc); + return std::nullopt; + } + + mCurrentFormat = format; + return mCurrentFormat; +} + +std::optional<struct v4l2_format> V4L2Queue::tryFormat(uint32_t fourcc, const ui::Size& size, + size_t bufferSize) { + struct v4l2_format format = buildV4L2Format(mType, fourcc, size, bufferSize, 0); + if (mDevice->ioctl(VIDIOC_TRY_FMT, &format) != 0 || format.fmt.pix_mp.pixelformat != fourcc) { + ALOGEQ("Tried format not supported (format_fourcc=0x%" PRIx32 ")", fourcc); + return std::nullopt; + } + + return format; +} + +std::pair<std::optional<struct v4l2_format>, int> V4L2Queue::getFormat() { + struct v4l2_format format; + memset(&format, 0, sizeof(format)); + format.type = mType; + if (mDevice->ioctl(VIDIOC_G_FMT, &format) != 0) { + ALOGEQ("Failed to get format"); + return std::make_pair(std::nullopt, errno); + } + + return std::make_pair(format, 0); +} + +size_t V4L2Queue::allocateBuffers(size_t count, enum v4l2_memory memory) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + ALOG_ASSERT(!mFreeBuffers); + ALOG_ASSERT(mQueuedBuffers.size() == 0u); + + if (isStreaming()) { + ALOGEQ("Cannot allocate buffers while streaming."); + return 0; + } + + if (mBuffers.size() != 0) { + ALOGEQ("Cannot allocate new buffers while others are still allocated."); + return 0; + } + + if (count == 0) { + ALOGEQ("Attempting to allocate 0 buffers."); + return 0; + } + + // First query the number of planes in the buffers we are about to request. This should not be + // required, but Tegra's VIDIOC_QUERYBUF will fail on output buffers if the number of specified + // planes does not exactly match the format. + struct v4l2_format format = {.type = mType}; + int ret = mDevice->ioctl(VIDIOC_G_FMT, &format); + if (ret) { + ALOGEQ("VIDIOC_G_FMT failed"); + return 0; + } + mPlanesCount = format.fmt.pix_mp.num_planes; + ALOG_ASSERT(mPlanesCount <= static_cast<size_t>(VIDEO_MAX_PLANES)); + + struct v4l2_requestbuffers reqbufs; + memset(&reqbufs, 0, sizeof(reqbufs)); + reqbufs.count = count; + reqbufs.type = mType; + reqbufs.memory = memory; + ALOGVQ("Requesting %zu buffers.", count); + + ret = mDevice->ioctl(VIDIOC_REQBUFS, &reqbufs); + if (ret) { + ALOGEQ("VIDIOC_REQBUFS failed"); + return 0; + } + ALOGVQ("Queue %u: got %u buffers.", mType, reqbufs.count); + + mMemory = memory; + + mFreeBuffers = new V4L2BuffersList(); + + // Now query all buffer information. + for (size_t i = 0; i < reqbufs.count; i++) { + auto buffer = V4L2Buffer::create(mDevice, mType, mMemory, format, i); + + if (!buffer) { + deallocateBuffers(); + + return 0; + } + + mBuffers.emplace_back(std::move(buffer)); + mFreeBuffers->returnBuffer(i); + } + + ALOG_ASSERT(mFreeBuffers); + ALOG_ASSERT(mFreeBuffers->size() == mBuffers.size()); + ALOG_ASSERT(mQueuedBuffers.size() == 0u); + + return mBuffers.size(); +} + +bool V4L2Queue::deallocateBuffers() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (isStreaming()) { + ALOGEQ("Cannot deallocate buffers while streaming."); + return false; + } + + if (mBuffers.size() == 0) return true; + + mWeakThisFactory.InvalidateWeakPtrs(); + mBuffers.clear(); + mFreeBuffers = nullptr; + + // Free all buffers. + struct v4l2_requestbuffers reqbufs; + memset(&reqbufs, 0, sizeof(reqbufs)); + reqbufs.count = 0; + reqbufs.type = mType; + reqbufs.memory = mMemory; + + int ret = mDevice->ioctl(VIDIOC_REQBUFS, &reqbufs); + if (ret) { + ALOGEQ("VIDIOC_REQBUFS failed"); + return false; + } + + ALOG_ASSERT(!mFreeBuffers); + ALOG_ASSERT(mQueuedBuffers.size() == 0u); + + return true; +} + +size_t V4L2Queue::getMemoryUsage() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + size_t usage = 0; + for (const auto& buf : mBuffers) { + usage += buf->getMemoryUsage(); + } + return usage; +} + +v4l2_memory V4L2Queue::getMemoryType() const { + return mMemory; +} + +std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + // No buffers allocated at the moment? + if (!mFreeBuffers) return std::nullopt; + + auto bufferId = mFreeBuffers->getFreeBuffer(); + if (!bufferId.has_value()) return std::nullopt; + + return V4L2BufferRefFactory::CreateWritableRef(mBuffers[bufferId.value()]->v4l2_buffer(), + mWeakThisFactory.GetWeakPtr()); +} + +std::optional<V4L2WritableBufferRef> V4L2Queue::getFreeBuffer(size_t requestedBufferIid) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + // No buffers allocated at the moment? + if (!mFreeBuffers) return std::nullopt; + + auto bufferId = mFreeBuffers->getFreeBuffer(requestedBufferIid); + if (!bufferId.has_value()) return std::nullopt; + + return V4L2BufferRefFactory::CreateWritableRef(mBuffers[bufferId.value()]->v4l2_buffer(), + mWeakThisFactory.GetWeakPtr()); +} + +bool V4L2Queue::queueBuffer(struct v4l2_buffer* v4l2Buffer) { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + int ret = mDevice->ioctl(VIDIOC_QBUF, v4l2Buffer); + if (ret) { + ALOGEQ("VIDIOC_QBUF failed"); + return false; + } + + auto inserted = mQueuedBuffers.emplace(v4l2Buffer->index); + if (!inserted.second) { + ALOGE("Queuing buffer failed"); + return false; + } + + mDevice->schedulePoll(); + + return true; +} + +std::pair<bool, V4L2ReadableBufferRef> V4L2Queue::dequeueBuffer() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + // No need to dequeue if no buffers queued. + if (queuedBuffersCount() == 0) return std::make_pair(true, nullptr); + + if (!isStreaming()) { + ALOGEQ("Attempting to dequeue a buffer while not streaming."); + return std::make_pair(true, nullptr); + } + + struct v4l2_buffer v4l2Buffer; + memset(&v4l2Buffer, 0, sizeof(v4l2Buffer)); + // WARNING: do not change this to a vector or something smaller than VIDEO_MAX_PLANES, otherwise + // the Tegra libv4l2 will write data beyond the number of allocated planes, resulting in memory + // corruption. + struct v4l2_plane planes[VIDEO_MAX_PLANES]; + memset(planes, 0, sizeof(planes)); + v4l2Buffer.type = mType; + v4l2Buffer.memory = mMemory; + v4l2Buffer.m.planes = planes; + v4l2Buffer.length = mPlanesCount; + int ret = mDevice->ioctl(VIDIOC_DQBUF, &v4l2Buffer); + if (ret) { + // TODO(acourbot): we should not have to check for EPIPE as codec clients should not call + // this method after the last buffer is dequeued. + switch (errno) { + case EAGAIN: + case EPIPE: + // This is not an error so we'll need to continue polling but won't provide a buffer. + mDevice->schedulePoll(); + return std::make_pair(true, nullptr); + default: + ALOGEQ("VIDIOC_DQBUF failed"); + return std::make_pair(false, nullptr); + } + } + + auto it = mQueuedBuffers.find(v4l2Buffer.index); + ALOG_ASSERT(it != mQueuedBuffers.end()); + mQueuedBuffers.erase(*it); + + if (queuedBuffersCount() > 0) mDevice->schedulePoll(); + + ALOG_ASSERT(mFreeBuffers); + return std::make_pair(true, V4L2BufferRefFactory::CreateReadableRef( + v4l2Buffer, mWeakThisFactory.GetWeakPtr())); +} + +bool V4L2Queue::isStreaming() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + return mIsStreaming; +} + +bool V4L2Queue::streamon() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + if (mIsStreaming) return true; + + int arg = static_cast<int>(mType); + int ret = mDevice->ioctl(VIDIOC_STREAMON, &arg); + if (ret) { + ALOGEQ("VIDIOC_STREAMON failed"); + return false; + } + + mIsStreaming = true; + + return true; +} + +bool V4L2Queue::streamoff() { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + // We do not check the value of IsStreaming(), because we may have queued buffers to the queue + // and wish to get them back - in such as case, we may need to do a VIDIOC_STREAMOFF on a + // stopped queue. + + int arg = static_cast<int>(mType); + int ret = mDevice->ioctl(VIDIOC_STREAMOFF, &arg); + if (ret) { + ALOGEQ("VIDIOC_STREAMOFF failed"); + return false; + } + + for (const auto& bufferId : mQueuedBuffers) { + ALOG_ASSERT(mFreeBuffers); + mFreeBuffers->returnBuffer(bufferId); + } + + mQueuedBuffers.clear(); + + mIsStreaming = false; + + return true; +} + +size_t V4L2Queue::allocatedBuffersCount() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + return mBuffers.size(); +} + +size_t V4L2Queue::freeBuffersCount() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + return mFreeBuffers ? mFreeBuffers->size() : 0; +} + +size_t V4L2Queue::queuedBuffersCount() const { + ALOG_ASSERT(mSequenceChecker.CalledOnValidSequence()); + + return mQueuedBuffers.size(); +} + +#undef ALOGEQ +#undef ALOGVQ + +// This class is used to expose V4L2Queue's constructor to this module. This is to ensure that +// nobody else can create instances of it. +class V4L2QueueFactory { +public: + static scoped_refptr<V4L2Queue> createQueue(scoped_refptr<V4L2Device> dev, + enum v4l2_buf_type type, + base::OnceClosure destroyCb) { + return new V4L2Queue(std::move(dev), type, std::move(destroyCb)); + } +}; + +V4L2Device::V4L2Device() { + DETACH_FROM_SEQUENCE(mClientSequenceChecker); +} + +V4L2Device::~V4L2Device() { + closeDevice(); +} + +scoped_refptr<V4L2Queue> V4L2Device::getQueue(enum v4l2_buf_type type) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + switch (type) { + // Supported queue types. + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + break; + default: + ALOGE("Unsupported V4L2 queue type: %u", type); + return nullptr; + } + + // TODO(acourbot): we should instead query the device for available queues, and allocate them + // accordingly. This will do for now though. + auto it = mQueues.find(type); + if (it != mQueues.end()) return scoped_refptr<V4L2Queue>(it->second); + + scoped_refptr<V4L2Queue> queue = V4L2QueueFactory::createQueue( + this, type, base::BindOnce(&V4L2Device::onQueueDestroyed, this, type)); + + mQueues[type] = queue.get(); + return queue; +} + +void V4L2Device::onQueueDestroyed(v4l2_buf_type bufType) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + auto it = mQueues.find(bufType); + ALOG_ASSERT(it != mQueues.end()); + mQueues.erase(it); +} + +// static +scoped_refptr<V4L2Device> V4L2Device::create() { + ALOGV("%s()", __func__); + return scoped_refptr<V4L2Device>(new V4L2Device()); +} + +bool V4L2Device::open(Type type, uint32_t v4l2PixFmt) { + ALOGV("%s()", __func__); + + std::string path = getDevicePathFor(type, v4l2PixFmt); + + if (path.empty()) { + ALOGE("No devices supporting %s for type: %u", fourccToString(v4l2PixFmt).c_str(), + static_cast<uint32_t>(type)); + return false; + } + + if (!openDevicePath(path, type)) { + ALOGE("Failed opening %s", path.c_str()); + return false; + } + + mDevicePollInterruptFd.reset(eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)); + if (!mDevicePollInterruptFd.is_valid()) { + ALOGE("Failed creating a poll interrupt fd"); + return false; + } + + return true; +} + +int V4L2Device::ioctl(int request, void* arg) { + ALOG_ASSERT(mDeviceFd.is_valid()); + return HANDLE_EINTR(::ioctl(mDeviceFd.get(), request, arg)); +} + +bool V4L2Device::poll(bool pollDevice, bool* eventPending) { + struct pollfd pollfds[2]; + nfds_t nfds; + int pollfd = -1; + + pollfds[0].fd = mDevicePollInterruptFd.get(); + pollfds[0].events = POLLIN | POLLERR; + nfds = 1; + + if (pollDevice) { + ALOGV("adding device fd to poll() set"); + pollfds[nfds].fd = mDeviceFd.get(); + pollfds[nfds].events = POLLIN | POLLOUT | POLLERR | POLLPRI; + pollfd = nfds; + nfds++; + } + + if (HANDLE_EINTR(::poll(pollfds, nfds, -1)) == -1) { + ALOGE("poll() failed"); + return false; + } + *eventPending = (pollfd != -1 && pollfds[pollfd].revents & POLLPRI); + return true; +} + +void* V4L2Device::mmap(void* addr, unsigned int len, int prot, int flags, unsigned int offset) { + DCHECK(mDeviceFd.is_valid()); + return ::mmap(addr, len, prot, flags, mDeviceFd.get(), offset); +} + +void V4L2Device::munmap(void* addr, unsigned int len) { + ::munmap(addr, len); +} + +bool V4L2Device::setDevicePollInterrupt() { + ALOGV("%s()", __func__); + + const uint64_t buf = 1; + if (HANDLE_EINTR(write(mDevicePollInterruptFd.get(), &buf, sizeof(buf))) == -1) { + ALOGE("write() failed"); + return false; + } + return true; +} + +bool V4L2Device::clearDevicePollInterrupt() { + ALOGV("%s()", __func__); + + uint64_t buf; + if (HANDLE_EINTR(read(mDevicePollInterruptFd.get(), &buf, sizeof(buf))) == -1) { + if (errno == EAGAIN) { + // No interrupt flag set, and we're reading nonblocking. Not an error. + return true; + } else { + ALOGE("read() failed"); + return false; + } + } + return true; +} + +std::vector<base::ScopedFD> V4L2Device::getDmabufsForV4L2Buffer(int index, size_t numPlanes, + enum v4l2_buf_type bufType) { + ALOGV("%s()", __func__); + ALOG_ASSERT(V4L2_TYPE_IS_MULTIPLANAR(bufType)); + + std::vector<base::ScopedFD> dmabufFds; + for (size_t i = 0; i < numPlanes; ++i) { + struct v4l2_exportbuffer expbuf; + memset(&expbuf, 0, sizeof(expbuf)); + expbuf.type = bufType; + expbuf.index = index; + expbuf.plane = i; + expbuf.flags = O_CLOEXEC; + if (ioctl(VIDIOC_EXPBUF, &expbuf) != 0) { + dmabufFds.clear(); + break; + } + + dmabufFds.push_back(base::ScopedFD(expbuf.fd)); + } + + return dmabufFds; +} + +std::vector<uint32_t> V4L2Device::preferredInputFormat(Type type) { + if (type == Type::kEncoder) return {V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_NV12}; + + return {}; +} + +// static +uint32_t V4L2Device::C2ProfileToV4L2PixFmt(C2Config::profile_t profile, bool sliceBased) { + if (profile >= C2Config::PROFILE_AVC_BASELINE && + profile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) { + if (sliceBased) { + return V4L2_PIX_FMT_H264_SLICE; + } else { + return V4L2_PIX_FMT_H264; + } + } else if (profile >= C2Config::PROFILE_VP8_0 && profile <= C2Config::PROFILE_VP8_3) { + if (sliceBased) { + return V4L2_PIX_FMT_VP8_FRAME; + } else { + return V4L2_PIX_FMT_VP8; + } + } else if (profile >= C2Config::PROFILE_VP9_0 && profile <= C2Config::PROFILE_VP9_3) { + if (sliceBased) { + return V4L2_PIX_FMT_VP9_FRAME; + } else { + return V4L2_PIX_FMT_VP9; + } + } else { + ALOGE("Unknown profile: %s", profileToString(profile)); + return 0; + } +} + +// static +C2Config::profile_t V4L2Device::v4L2ProfileToC2Profile(VideoCodec codec, uint32_t profile) { + switch (codec) { + case VideoCodec::H264: + switch (profile) { + case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE: + case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: + return C2Config::PROFILE_AVC_BASELINE; + case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN: + return C2Config::PROFILE_AVC_MAIN; + case V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED: + return C2Config::PROFILE_AVC_EXTENDED; + case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH: + return C2Config::PROFILE_AVC_HIGH; + } + break; + case VideoCodec::VP8: + switch (profile) { + case V4L2_MPEG_VIDEO_VP8_PROFILE_0: + return C2Config::PROFILE_VP8_0; + case V4L2_MPEG_VIDEO_VP8_PROFILE_1: + return C2Config::PROFILE_VP8_1; + case V4L2_MPEG_VIDEO_VP8_PROFILE_2: + return C2Config::PROFILE_VP8_2; + case V4L2_MPEG_VIDEO_VP8_PROFILE_3: + return C2Config::PROFILE_VP8_3; + } + break; + case VideoCodec::VP9: + switch (profile) { + case V4L2_MPEG_VIDEO_VP9_PROFILE_0: + return C2Config::PROFILE_VP9_0; + case V4L2_MPEG_VIDEO_VP9_PROFILE_1: + return C2Config::PROFILE_VP9_1; + case V4L2_MPEG_VIDEO_VP9_PROFILE_2: + return C2Config::PROFILE_VP9_2; + case V4L2_MPEG_VIDEO_VP9_PROFILE_3: + return C2Config::PROFILE_VP9_3; + } + break; + default: + ALOGE("Unknown codec: %u", codec); + } + ALOGE("Unknown profile: %u", profile); + return C2Config::PROFILE_UNUSED; +} + +std::vector<C2Config::profile_t> V4L2Device::v4L2PixFmtToC2Profiles(uint32_t pixFmt, + bool /*isEncoder*/) { + auto getSupportedProfiles = [this](VideoCodec codec, + std::vector<C2Config::profile_t>* profiles) { + uint32_t queryId = 0; + switch (codec) { + case VideoCodec::H264: + queryId = V4L2_CID_MPEG_VIDEO_H264_PROFILE; + break; + case VideoCodec::VP8: + queryId = V4L2_CID_MPEG_VIDEO_VP8_PROFILE; + break; + case VideoCodec::VP9: + queryId = V4L2_CID_MPEG_VIDEO_VP9_PROFILE; + break; + default: + return false; + } + + v4l2_queryctrl queryCtrl = {}; + queryCtrl.id = queryId; + if (ioctl(VIDIOC_QUERYCTRL, &queryCtrl) != 0) { + return false; + } + v4l2_querymenu queryMenu = {}; + queryMenu.id = queryCtrl.id; + for (queryMenu.index = queryCtrl.minimum; + static_cast<int>(queryMenu.index) <= queryCtrl.maximum; queryMenu.index++) { + if (ioctl(VIDIOC_QUERYMENU, &queryMenu) == 0) { + const C2Config::profile_t profile = + V4L2Device::v4L2ProfileToC2Profile(codec, queryMenu.index); + if (profile != C2Config::PROFILE_UNUSED) profiles->push_back(profile); + } + } + return true; + }; + + std::vector<C2Config::profile_t> profiles; + switch (pixFmt) { + case V4L2_PIX_FMT_H264: + case V4L2_PIX_FMT_H264_SLICE: + if (!getSupportedProfiles(VideoCodec::H264, &profiles)) { + ALOGW("Driver doesn't support QUERY H264 profiles, " + "use default values, Base, Main, High"); + profiles = { + C2Config::PROFILE_AVC_BASELINE, + C2Config::PROFILE_AVC_MAIN, + C2Config::PROFILE_AVC_HIGH, + }; + } + break; + case V4L2_PIX_FMT_VP8: + case V4L2_PIX_FMT_VP8_FRAME: + if (!getSupportedProfiles(VideoCodec::VP8, &profiles)) { + ALOGW("Driver doesn't support QUERY VP8 profiles, use default values, Profile0"); + profiles = {C2Config::PROFILE_VP8_0}; + } + break; + case V4L2_PIX_FMT_VP9: + case V4L2_PIX_FMT_VP9_FRAME: + if (!getSupportedProfiles(VideoCodec::VP9, &profiles)) { + ALOGW("Driver doesn't support QUERY VP9 profiles, use default values, Profile0"); + profiles = {C2Config::PROFILE_VP9_0}; + } + break; + default: + ALOGE("Unhandled pixelformat %s", fourccToString(pixFmt).c_str()); + return {}; + } + + // Erase duplicated profiles. + std::sort(profiles.begin(), profiles.end()); + profiles.erase(std::unique(profiles.begin(), profiles.end()), profiles.end()); + return profiles; +} + +// static +int32_t V4L2Device::c2ProfileToV4L2H264Profile(C2Config::profile_t profile) { + switch (profile) { + case C2Config::PROFILE_AVC_BASELINE: + return V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE; + case C2Config::PROFILE_AVC_MAIN: + return V4L2_MPEG_VIDEO_H264_PROFILE_MAIN; + case C2Config::PROFILE_AVC_EXTENDED: + return V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED; + case C2Config::PROFILE_AVC_HIGH: + return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH; + case C2Config::PROFILE_AVC_HIGH_10: + return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_10; + case C2Config::PROFILE_AVC_HIGH_422: + return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_422; + case C2Config::PROFILE_AVC_HIGH_444_PREDICTIVE: + return V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE; + case C2Config::PROFILE_AVC_SCALABLE_BASELINE: + return V4L2_MPEG_VIDEO_H264_PROFILE_SCALABLE_BASELINE; + case C2Config::PROFILE_AVC_SCALABLE_HIGH: + return V4L2_MPEG_VIDEO_H264_PROFILE_SCALABLE_HIGH; + case C2Config::PROFILE_AVC_STEREO_HIGH: + return V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH; + case C2Config::PROFILE_AVC_MULTIVIEW_HIGH: + return V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH; + default: + ALOGE("Add more cases as needed"); + return -1; + } +} + +// static +int32_t V4L2Device::h264LevelIdcToV4L2H264Level(uint8_t levelIdc) { + switch (levelIdc) { + case 10: + return V4L2_MPEG_VIDEO_H264_LEVEL_1_0; + case 9: + return V4L2_MPEG_VIDEO_H264_LEVEL_1B; + case 11: + return V4L2_MPEG_VIDEO_H264_LEVEL_1_1; + case 12: + return V4L2_MPEG_VIDEO_H264_LEVEL_1_2; + case 13: + return V4L2_MPEG_VIDEO_H264_LEVEL_1_3; + case 20: + return V4L2_MPEG_VIDEO_H264_LEVEL_2_0; + case 21: + return V4L2_MPEG_VIDEO_H264_LEVEL_2_1; + case 22: + return V4L2_MPEG_VIDEO_H264_LEVEL_2_2; + case 30: + return V4L2_MPEG_VIDEO_H264_LEVEL_3_0; + case 31: + return V4L2_MPEG_VIDEO_H264_LEVEL_3_1; + case 32: + return V4L2_MPEG_VIDEO_H264_LEVEL_3_2; + case 40: + return V4L2_MPEG_VIDEO_H264_LEVEL_4_0; + case 41: + return V4L2_MPEG_VIDEO_H264_LEVEL_4_1; + case 42: + return V4L2_MPEG_VIDEO_H264_LEVEL_4_2; + case 50: + return V4L2_MPEG_VIDEO_H264_LEVEL_5_0; + case 51: + return V4L2_MPEG_VIDEO_H264_LEVEL_5_1; + default: + ALOGE("Unrecognized levelIdc: %u", static_cast<uint32_t>(levelIdc)); + return -1; + } +} + +// static +ui::Size V4L2Device::allocatedSizeFromV4L2Format(const struct v4l2_format& format) { + ui::Size codedSize; + ui::Size visibleSize; + VideoPixelFormat frameFormat = VideoPixelFormat::UNKNOWN; + size_t bytesPerLine = 0; + // Total bytes in the frame. + size_t sizeimage = 0; + + if (V4L2_TYPE_IS_MULTIPLANAR(format.type)) { + ALOG_ASSERT(format.fmt.pix_mp.num_planes > 0); + bytesPerLine = base::checked_cast<int>(format.fmt.pix_mp.plane_fmt[0].bytesperline); + for (size_t i = 0; i < format.fmt.pix_mp.num_planes; ++i) { + sizeimage += base::checked_cast<int>(format.fmt.pix_mp.plane_fmt[i].sizeimage); + } + visibleSize.set(base::checked_cast<int>(format.fmt.pix_mp.width), + base::checked_cast<int>(format.fmt.pix_mp.height)); + const uint32_t pixFmt = format.fmt.pix_mp.pixelformat; + const auto frameFourcc = Fourcc::fromV4L2PixFmt(pixFmt); + if (!frameFourcc) { + ALOGE("Unsupported format %s", fourccToString(pixFmt).c_str()); + return codedSize; + } + frameFormat = frameFourcc->toVideoPixelFormat(); + } else { + bytesPerLine = base::checked_cast<int>(format.fmt.pix.bytesperline); + sizeimage = base::checked_cast<int>(format.fmt.pix.sizeimage); + visibleSize.set(base::checked_cast<int>(format.fmt.pix.width), + base::checked_cast<int>(format.fmt.pix.height)); + const uint32_t fourcc = format.fmt.pix.pixelformat; + const auto frameFourcc = Fourcc::fromV4L2PixFmt(fourcc); + if (!frameFourcc) { + ALOGE("Unsupported format %s", fourccToString(fourcc).c_str()); + return codedSize; + } + frameFormat = frameFourcc ? frameFourcc->toVideoPixelFormat() : VideoPixelFormat::UNKNOWN; + } + + // V4L2 does not provide per-plane bytesperline (bpl) when different components are sharing one + // physical plane buffer. In this case, it only provides bpl for the first component in the + // plane. So we can't depend on it for calculating height, because bpl may vary within one + // physical plane buffer. For example, YUV420 contains 3 components in one physical plane, with + // Y at 8 bits per pixel, and Cb/Cr at 4 bits per pixel per component, but we only get 8 pits + // per pixel from bytesperline in physical plane 0. So we need to get total frame bpp from + // elsewhere to calculate coded height. + + // We need bits per pixel for one component only to calculate the coded width from bytesperline. + int planeHorizBitsPerPixel = planeHorizontalBitsPerPixel(frameFormat, 0); + + // Adding up bpp for each component will give us total bpp for all components. + int totalBpp = 0; + for (size_t i = 0; i < numPlanes(frameFormat); ++i) + totalBpp += planeBitsPerPixel(frameFormat, i); + + if (sizeimage == 0 || bytesPerLine == 0 || planeHorizBitsPerPixel == 0 || totalBpp == 0 || + (bytesPerLine * 8) % planeHorizBitsPerPixel != 0) { + ALOGE("Invalid format provided"); + return codedSize; + } + + // Coded width can be calculated by taking the first component's bytesperline, which in V4L2 + // always applies to the first component in physical plane buffer. + int codedWidth = bytesPerLine * 8 / planeHorizBitsPerPixel; + // Sizeimage is codedWidth * codedHeight * totalBpp. + int codedHeight = sizeimage * 8 / codedWidth / totalBpp; + + codedSize.set(codedWidth, codedHeight); + ALOGV("codedSize=%s", toString(codedSize).c_str()); + + // Sanity checks. Calculated coded size has to contain given visible size and fulfill buffer + // byte size requirements. + ALOG_ASSERT(Rect(codedSize).Contains(Rect(visibleSize))); + ALOG_ASSERT(sizeimage <= allocationSize(frameFormat, codedSize)); + + return codedSize; +} + +// static +const char* V4L2Device::v4L2MemoryToString(const v4l2_memory memory) { + switch (memory) { + case V4L2_MEMORY_MMAP: + return "V4L2_MEMORY_MMAP"; + case V4L2_MEMORY_USERPTR: + return "V4L2_MEMORY_USERPTR"; + case V4L2_MEMORY_DMABUF: + return "V4L2_MEMORY_DMABUF"; + case V4L2_MEMORY_OVERLAY: + return "V4L2_MEMORY_OVERLAY"; + default: + return "UNKNOWN"; + } +} + +// static +const char* V4L2Device::v4L2BufferTypeToString(const enum v4l2_buf_type bufType) { + switch (bufType) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + return "OUTPUT"; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + return "CAPTURE"; + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + return "OUTPUT_MPLANE"; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + return "CAPTURE_MPLANE"; + default: + return "UNKNOWN"; + } +} + +// static +std::string V4L2Device::v4L2FormatToString(const struct v4l2_format& format) { + std::ostringstream s; + s << "v4l2_format type: " << format.type; + if (format.type == V4L2_BUF_TYPE_VIDEO_CAPTURE || format.type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { + // single-planar + const struct v4l2_pix_format& pix = format.fmt.pix; + s << ", width_height: " << toString(ui::Size(pix.width, pix.height)) + << ", pixelformat: " << fourccToString(pix.pixelformat) << ", field: " << pix.field + << ", bytesperline: " << pix.bytesperline << ", sizeimage: " << pix.sizeimage; + } else if (V4L2_TYPE_IS_MULTIPLANAR(format.type)) { + const struct v4l2_pix_format_mplane& pixMp = format.fmt.pix_mp; + // As long as num_planes's type is uint8_t, ostringstream treats it as a char instead of an + // integer, which is not what we want. Casting pix_mp.num_planes unsigned int solves the + // issue. + s << ", width_height: " << toString(ui::Size(pixMp.width, pixMp.height)) + << ", pixelformat: " << fourccToString(pixMp.pixelformat) << ", field: " << pixMp.field + << ", num_planes: " << static_cast<unsigned int>(pixMp.num_planes); + for (size_t i = 0; i < pixMp.num_planes; ++i) { + const struct v4l2_plane_pix_format& plane_fmt = pixMp.plane_fmt[i]; + s << ", plane_fmt[" << i << "].sizeimage: " << plane_fmt.sizeimage << ", plane_fmt[" + << i << "].bytesperline: " << plane_fmt.bytesperline; + } + } else { + s << " unsupported yet."; + } + return s.str(); +} + +// static +std::string V4L2Device::v4L2BufferToString(const struct v4l2_buffer& buffer) { + std::ostringstream s; + s << "v4l2_buffer type: " << buffer.type << ", memory: " << buffer.memory + << ", index: " << buffer.index << " bytesused: " << buffer.bytesused + << ", length: " << buffer.length; + if (buffer.type == V4L2_BUF_TYPE_VIDEO_CAPTURE || buffer.type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { + // single-planar + if (buffer.memory == V4L2_MEMORY_MMAP) { + s << ", m.offset: " << buffer.m.offset; + } else if (buffer.memory == V4L2_MEMORY_USERPTR) { + s << ", m.userptr: " << buffer.m.userptr; + } else if (buffer.memory == V4L2_MEMORY_DMABUF) { + s << ", m.fd: " << buffer.m.fd; + }; + } else if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) { + for (size_t i = 0; i < buffer.length; ++i) { + const struct v4l2_plane& plane = buffer.m.planes[i]; + s << ", m.planes[" << i << "](bytesused: " << plane.bytesused + << ", length: " << plane.length << ", data_offset: " << plane.data_offset; + if (buffer.memory == V4L2_MEMORY_MMAP) { + s << ", m.mem_offset: " << plane.m.mem_offset; + } else if (buffer.memory == V4L2_MEMORY_USERPTR) { + s << ", m.userptr: " << plane.m.userptr; + } else if (buffer.memory == V4L2_MEMORY_DMABUF) { + s << ", m.fd: " << plane.m.fd; + } + s << ")"; + } + } else { + s << " unsupported yet."; + } + return s.str(); +} + +// static +std::optional<VideoFrameLayout> V4L2Device::v4L2FormatToVideoFrameLayout( + const struct v4l2_format& format) { + if (!V4L2_TYPE_IS_MULTIPLANAR(format.type)) { + ALOGE("v4l2_buf_type is not multiplanar: 0x%" PRIx32, format.type); + return std::nullopt; + } + const v4l2_pix_format_mplane& pixMp = format.fmt.pix_mp; + const uint32_t& pixFmt = pixMp.pixelformat; + const auto videoFourcc = Fourcc::fromV4L2PixFmt(pixFmt); + if (!videoFourcc) { + ALOGE("Failed to convert pixel format to VideoPixelFormat: %s", + fourccToString(pixFmt).c_str()); + return std::nullopt; + } + const VideoPixelFormat videoFormat = videoFourcc->toVideoPixelFormat(); + const size_t numBuffers = pixMp.num_planes; + const size_t numColorPlanes = numPlanes(videoFormat); + if (numColorPlanes == 0) { + ALOGE("Unsupported video format for NumPlanes(): %s", + videoPixelFormatToString(videoFormat).c_str()); + return std::nullopt; + } + if (numBuffers > numColorPlanes) { + ALOGE("pix_mp.num_planes: %zu should not be larger than NumPlanes(%s): %zu", numBuffers, + videoPixelFormatToString(videoFormat).c_str(), numColorPlanes); + return std::nullopt; + } + // Reserve capacity in advance to prevent unnecessary vector reallocation. + std::vector<VideoFramePlane> planes; + planes.reserve(numColorPlanes); + for (size_t i = 0; i < numBuffers; ++i) { + const v4l2_plane_pix_format& planeFormat = pixMp.plane_fmt[i]; + planes.push_back(VideoFramePlane{planeFormat.bytesperline, 0u, planeFormat.sizeimage}); + } + // For the case that #color planes > #buffers, it fills stride of color plane which does not map + // to buffer. Right now only some pixel formats are supported: NV12, YUV420, YVU420. + if (numColorPlanes > numBuffers) { + const uint32_t yStride = planes[0].mStride; + // Note that y_stride is from v4l2 bytesperline and its type is uint32_t. It is safe to cast + // to size_t. + const size_t yStrideAbs = static_cast<size_t>(yStride); + switch (pixFmt) { + case V4L2_PIX_FMT_NV12: + // The stride of UV is the same as Y in NV12. The height is half of Y plane. + planes.push_back(VideoFramePlane{yStride, yStrideAbs * pixMp.height, + yStrideAbs * pixMp.height / 2}); + ALOG_ASSERT(2u == planes.size()); + break; + case V4L2_PIX_FMT_YUV420: + case V4L2_PIX_FMT_YVU420: { + // The spec claims that two Cx rows (including padding) is exactly as long as one Y row + // (including padding). So stride of Y must be even number. + if (yStride % 2 != 0 || pixMp.height % 2 != 0) { + ALOGE("Plane-Y stride and height should be even; stride: %u, height: %u", yStride, + pixMp.height); + return std::nullopt; + } + const uint32_t halfStride = yStride / 2; + const size_t plane0Area = yStrideAbs * pixMp.height; + const size_t plane1Area = plane0Area / 4; + planes.push_back(VideoFramePlane{halfStride, plane0Area, plane1Area}); + planes.push_back(VideoFramePlane{halfStride, plane0Area + plane1Area, plane1Area}); + ALOG_ASSERT(3u == planes.size()); + break; + } + default: + ALOGE("Cannot derive stride for each plane for pixel format %s", + fourccToString(pixFmt).c_str()); + return std::nullopt; + } + } + + return VideoFrameLayout{videoFormat, ui::Size(pixMp.width, pixMp.height), std::move(planes), + (numBuffers > 1)}; +} + +// static +size_t V4L2Device::getNumPlanesOfV4L2PixFmt(uint32_t pixFmt) { + std::optional<Fourcc> fourcc = Fourcc::fromV4L2PixFmt(pixFmt); + if (fourcc && fourcc->isMultiPlanar()) { + return numPlanes(fourcc->toVideoPixelFormat()); + } + return 1u; +} + +void V4L2Device::getSupportedResolution(uint32_t pixelFormat, ui::Size* minResolution, + ui::Size* maxResolution) { + maxResolution->set(0, 0); + minResolution->set(0, 0); + v4l2_frmsizeenum frameSize; + memset(&frameSize, 0, sizeof(frameSize)); + frameSize.pixel_format = pixelFormat; + for (; ioctl(VIDIOC_ENUM_FRAMESIZES, &frameSize) == 0; ++frameSize.index) { + if (frameSize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + if (frameSize.discrete.width >= base::checked_cast<uint32_t>(maxResolution->width) && + frameSize.discrete.height >= base::checked_cast<uint32_t>(maxResolution->height)) { + maxResolution->set(frameSize.discrete.width, frameSize.discrete.height); + } + if (isEmpty(*minResolution) || + (frameSize.discrete.width <= base::checked_cast<uint32_t>(minResolution->width) && + frameSize.discrete.height <= + base::checked_cast<uint32_t>(minResolution->height))) { + minResolution->set(frameSize.discrete.width, frameSize.discrete.height); + } + } else if (frameSize.type == V4L2_FRMSIZE_TYPE_STEPWISE || + frameSize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) { + maxResolution->set(frameSize.stepwise.max_width, frameSize.stepwise.max_height); + minResolution->set(frameSize.stepwise.min_width, frameSize.stepwise.min_height); + break; + } + } + if (isEmpty(*maxResolution)) { + maxResolution->set(1920, 1088); + ALOGE("GetSupportedResolution failed to get maximum resolution for fourcc %s, " + "fall back to %s", + fourccToString(pixelFormat).c_str(), toString(*maxResolution).c_str()); + } + if (isEmpty(*minResolution)) { + minResolution->set(16, 16); + ALOGE("GetSupportedResolution failed to get minimum resolution for fourcc %s, " + "fall back to %s", + fourccToString(pixelFormat).c_str(), toString(*minResolution).c_str()); + } +} + +std::vector<uint32_t> V4L2Device::enumerateSupportedPixelformats(v4l2_buf_type bufType) { + std::vector<uint32_t> pixelFormats; + + v4l2_fmtdesc fmtDesc; + memset(&fmtDesc, 0, sizeof(fmtDesc)); + fmtDesc.type = bufType; + + for (; ioctl(VIDIOC_ENUM_FMT, &fmtDesc) == 0; ++fmtDesc.index) { + ALOGV("Found %s (0x%" PRIx32 ")", fmtDesc.description, fmtDesc.pixelformat); + pixelFormats.push_back(fmtDesc.pixelformat); + } + + return pixelFormats; +} + +V4L2Device::SupportedDecodeProfiles V4L2Device::getSupportedDecodeProfiles( + const size_t numFormats, const uint32_t pixelFormats[]) { + SupportedDecodeProfiles supportedProfiles; + + Type type = Type::kDecoder; + const auto& devices = getDevicesForType(type); + for (const auto& device : devices) { + if (!openDevicePath(device.first, type)) { + ALOGV("Failed opening %s", device.first.c_str()); + continue; + } + + const auto& profiles = enumerateSupportedDecodeProfiles(numFormats, pixelFormats); + supportedProfiles.insert(supportedProfiles.end(), profiles.begin(), profiles.end()); + closeDevice(); + } + + return supportedProfiles; +} + +V4L2Device::SupportedEncodeProfiles V4L2Device::getSupportedEncodeProfiles() { + SupportedEncodeProfiles supportedProfiles; + + Type type = Type::kEncoder; + const auto& devices = getDevicesForType(type); + for (const auto& device : devices) { + if (!openDevicePath(device.first, type)) { + ALOGV("Failed opening %s", device.first.c_str()); + continue; + } + + const auto& profiles = enumerateSupportedEncodeProfiles(); + supportedProfiles.insert(supportedProfiles.end(), profiles.begin(), profiles.end()); + closeDevice(); + } + + return supportedProfiles; +} + +V4L2Device::SupportedDecodeProfiles V4L2Device::enumerateSupportedDecodeProfiles( + const size_t numFormats, const uint32_t pixelFormats[]) { + SupportedDecodeProfiles profiles; + + const auto& supportedPixelformats = + enumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + + for (uint32_t pixelFormat : supportedPixelformats) { + if (std::find(pixelFormats, pixelFormats + numFormats, pixelFormat) == + pixelFormats + numFormats) + continue; + + SupportedDecodeProfile profile; + getSupportedResolution(pixelFormat, &profile.min_resolution, &profile.max_resolution); + + const auto videoCodecProfiles = v4L2PixFmtToC2Profiles(pixelFormat, false); + + for (const auto& videoCodecProfile : videoCodecProfiles) { + profile.profile = videoCodecProfile; + profiles.push_back(profile); + + ALOGV("Found decoder profile %s, resolutions: %s %s", profileToString(profile.profile), + toString(profile.min_resolution).c_str(), + toString(profile.max_resolution).c_str()); + } + } + + return profiles; +} + +V4L2Device::SupportedEncodeProfiles V4L2Device::enumerateSupportedEncodeProfiles() { + SupportedEncodeProfiles profiles; + + const auto& supportedPixelformats = + enumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + + for (const auto& pixelformat : supportedPixelformats) { + SupportedEncodeProfile profile; + profile.max_framerate_numerator = 30; + profile.max_framerate_denominator = 1; + ui::Size minResolution; + getSupportedResolution(pixelformat, &minResolution, &profile.max_resolution); + + const auto videoCodecProfiles = v4L2PixFmtToC2Profiles(pixelformat, true); + + for (const auto& videoCodecProfile : videoCodecProfiles) { + profile.profile = videoCodecProfile; + profiles.push_back(profile); + + ALOGV("Found encoder profile %s, max resolution: %s", profileToString(profile.profile), + toString(profile.max_resolution).c_str()); + } + } + + return profiles; +} + +bool V4L2Device::startPolling(android::V4L2DevicePoller::EventCallback eventCallback, + base::RepeatingClosure errorCallback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + if (!mDevicePoller) { + mDevicePoller = std::make_unique<android::V4L2DevicePoller>(this, "V4L2DeviceThreadPoller"); + } + + bool ret = mDevicePoller->startPolling(std::move(eventCallback), std::move(errorCallback)); + + if (!ret) mDevicePoller = nullptr; + + return ret; +} + +bool V4L2Device::stopPolling() { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + return !mDevicePoller || mDevicePoller->stopPolling(); +} + +void V4L2Device::schedulePoll() { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + if (!mDevicePoller || !mDevicePoller->isPolling()) return; + + mDevicePoller->schedulePoll(); +} + +bool V4L2Device::isCtrlExposed(uint32_t ctrlId) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + struct v4l2_queryctrl queryCtrl; + memset(&queryCtrl, 0, sizeof(queryCtrl)); + queryCtrl.id = ctrlId; + + return ioctl(VIDIOC_QUERYCTRL, &queryCtrl) == 0; +} + +bool V4L2Device::setExtCtrls(uint32_t ctrlClass, std::vector<V4L2ExtCtrl> ctrls) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + if (ctrls.empty()) return true; + + struct v4l2_ext_controls extCtrls; + memset(&extCtrls, 0, sizeof(extCtrls)); + extCtrls.ctrl_class = ctrlClass; + extCtrls.count = ctrls.size(); + extCtrls.controls = &ctrls[0].ctrl; + return ioctl(VIDIOC_S_EXT_CTRLS, &extCtrls) == 0; +} + +bool V4L2Device::isCommandSupported(uint32_t commandId) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + struct v4l2_encoder_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = commandId; + + return ioctl(VIDIOC_TRY_ENCODER_CMD, &cmd) == 0; +} + +bool V4L2Device::hasCapabilities(uint32_t capabilities) { + DCHECK_CALLED_ON_VALID_SEQUENCE(mClientSequenceChecker); + + struct v4l2_capability caps; + memset(&caps, 0, sizeof(caps)); + if (ioctl(VIDIOC_QUERYCAP, &caps) != 0) { + ALOGE("Failed to query capabilities"); + return false; + } + + return (caps.capabilities & capabilities) == capabilities; +} + +bool V4L2Device::openDevicePath(const std::string& path, Type /*type*/) { + ALOG_ASSERT(!mDeviceFd.is_valid()); + + mDeviceFd.reset(HANDLE_EINTR(::open(path.c_str(), O_RDWR | O_NONBLOCK | O_CLOEXEC))); + if (!mDeviceFd.is_valid()) return false; + + return true; +} + +void V4L2Device::closeDevice() { + ALOGV("%s()", __func__); + + mDeviceFd.reset(); +} + +void V4L2Device::enumerateDevicesForType(Type type) { + // video input/output devices are registered as /dev/videoX in V4L2. + static const std::string kVideoDevicePattern = "/dev/video"; + + std::string devicePattern; + v4l2_buf_type bufType; + switch (type) { + case Type::kDecoder: + devicePattern = kVideoDevicePattern; + bufType = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + break; + case Type::kEncoder: + devicePattern = kVideoDevicePattern; + bufType = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + break; + default: + ALOGE("Only decoder and encoder types are supported!!"); + return; + } + + std::vector<std::string> candidatePaths; + + // TODO(posciak): Remove this legacy unnumbered device once all platforms are updated to use + // numbered devices. + candidatePaths.push_back(devicePattern); + + // We are sandboxed, so we can't query directory contents to check which devices are actually + // available. Try to open the first 10; if not present, we will just fail to open immediately. + for (int i = 0; i < 10; ++i) { + candidatePaths.push_back(base::StringPrintf("%s%d", devicePattern.c_str(), i)); + } + + Devices devices; + for (const auto& path : candidatePaths) { + if (!openDevicePath(path, type)) { + continue; + } + + const auto& supportedPixelformats = enumerateSupportedPixelformats(bufType); + if (!supportedPixelformats.empty()) { + ALOGV("Found device: %s", path.c_str()); + devices.push_back(std::make_pair(path, supportedPixelformats)); + } + + closeDevice(); + } + + ALOG_ASSERT(mDevicesByType.count(type) == 0u); + mDevicesByType[type] = devices; +} + +const V4L2Device::Devices& V4L2Device::getDevicesForType(Type type) { + if (mDevicesByType.count(type) == 0) enumerateDevicesForType(type); + + ALOG_ASSERT(mDevicesByType.count(type) != 0u); + return mDevicesByType[type]; +} + +std::string V4L2Device::getDevicePathFor(Type type, uint32_t pixFmt) { + const Devices& devices = getDevicesForType(type); + + for (const auto& device : devices) { + if (std::find(device.second.begin(), device.second.end(), pixFmt) != device.second.end()) + return device.first; + } + + return std::string(); +} + +} // namespace android diff --git a/common/V4L2DevicePoller.cpp b/common/V4L2DevicePoller.cpp new file mode 100644 index 0000000..5f2d0a5 --- /dev/null +++ b/common/V4L2DevicePoller.cpp @@ -0,0 +1,130 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: 22d34680c8ac + +#include <v4l2_codec2/common/V4L2DevicePoller.h> + +#include <string> + +#include <base/bind.h> +#include <base/threading/sequenced_task_runner_handle.h> +#include <base/threading/thread_checker.h> +#include <log/log.h> + +#include <v4l2_codec2/common/V4L2Device.h> + +namespace android { + +V4L2DevicePoller::V4L2DevicePoller(V4L2Device* const device, const std::string& threadName) + : mDevice(device), + mPollThread(std::move(threadName)), + mTriggerPoll(base::WaitableEvent::ResetPolicy::AUTOMATIC, + base::WaitableEvent::InitialState::NOT_SIGNALED), + mStopPolling(false) {} + +V4L2DevicePoller::~V4L2DevicePoller() { + ALOG_ASSERT(mClientTaskTunner->RunsTasksInCurrentSequence()); + + stopPolling(); +} + +bool V4L2DevicePoller::startPolling(EventCallback eventCallback, + base::RepeatingClosure errorCallback) { + if (isPolling()) return true; + + ALOGV("Starting polling"); + + mClientTaskTunner = base::SequencedTaskRunnerHandle::Get(); + mErrorCallback = errorCallback; + + if (!mPollThread.Start()) { + ALOGE("Failed to start device poll thread"); + return false; + } + + mEventCallback = std::move(eventCallback); + + mStopPolling.store(false); + mPollThread.task_runner()->PostTask( + FROM_HERE, base::BindOnce(&V4L2DevicePoller::devicePollTask, base::Unretained(this))); + + ALOGV("Polling thread started"); + + schedulePoll(); + + return true; +} + +bool V4L2DevicePoller::stopPolling() { + ALOG_ASSERT(mClientTaskTunner->RunsTasksInCurrentSequence()); + + if (!isPolling()) return true; + + ALOGV("Stopping polling"); + + mStopPolling.store(true); + + mTriggerPoll.Signal(); + + if (!mDevice->setDevicePollInterrupt()) { + ALOGE("Failed to interrupt device poll."); + return false; + } + + ALOGV("Stop device poll thread"); + mPollThread.Stop(); + + if (!mDevice->clearDevicePollInterrupt()) { + ALOGE("Failed to clear interrupting device poll."); + return false; + } + + ALOGV("Polling thread stopped"); + + return true; +} + +bool V4L2DevicePoller::isPolling() const { + ALOG_ASSERT(mClientTaskTunner->RunsTasksInCurrentSequence()); + + return mPollThread.IsRunning(); +} + +void V4L2DevicePoller::schedulePoll() { + ALOG_ASSERT(mClientTaskTunner->RunsTasksInCurrentSequence()); + + // A call to DevicePollTask() will be posted when we actually start polling. + if (!isPolling()) return; + + ALOGV("Scheduling poll"); + + mTriggerPoll.Signal(); +} + +void V4L2DevicePoller::devicePollTask() { + ALOG_ASSERT(mClientTaskTunner->RunsTasksInCurrentSequence()); + + while (true) { + ALOGV("Waiting for poll to be scheduled."); + mTriggerPoll.Wait(); + + if (mStopPolling) { + ALOGV("Poll stopped, exiting."); + break; + } + + bool event_pending = false; + ALOGV("Polling device."); + if (!mDevice->poll(true, &event_pending)) { + ALOGE("An error occurred while polling, calling error callback"); + mClientTaskTunner->PostTask(FROM_HERE, mErrorCallback); + return; + } + + ALOGV("Poll returned, calling event callback."); + mClientTaskTunner->PostTask(FROM_HERE, base::Bind(mEventCallback, event_pending)); + } +} + +} // namespace android diff --git a/common/VideoPixelFormat.cpp b/common/VideoPixelFormat.cpp new file mode 100644 index 0000000..f175c26 --- /dev/null +++ b/common/VideoPixelFormat.cpp @@ -0,0 +1,371 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: 3b7ce92816e2 +// Note: only necessary functions are ported from video_types.cc + +#include <v4l2_codec2/common/VideoPixelFormat.h> + +#include <base/bits.h> +#include <utils/Log.h> + +namespace android { + +namespace { + +enum { + kMaxPlanes = 4, + kYPlane = 0, + kARGBPlane = kYPlane, + kUPlane = 1, + kUVPlane = kUPlane, + kVPlane = 2, + kAPlane = 3, +}; +} + +std::string videoPixelFormatToString(VideoPixelFormat format) { + switch (format) { + case VideoPixelFormat::I420: + return "I420"; + case VideoPixelFormat::YV12: + return "YV12"; + case VideoPixelFormat::I422: + return "I422"; + case VideoPixelFormat::I420A: + return "I420A"; + case VideoPixelFormat::I444: + return "I444"; + case VideoPixelFormat::NV12: + return "NV12"; + case VideoPixelFormat::NV21: + return "NV21"; + case VideoPixelFormat::YUY2: + return "YUY2"; + case VideoPixelFormat::ARGB: + return "ARGB"; + case VideoPixelFormat::XRGB: + return "XRGB"; + case VideoPixelFormat::RGB24: + return "RGB24"; + case VideoPixelFormat::MJPEG: + return "MJPEG"; + case VideoPixelFormat::YUV420P9: + return "YUV420P9"; + case VideoPixelFormat::YUV420P10: + return "YUV420P10"; + case VideoPixelFormat::YUV422P9: + return "YUV422P9"; + case VideoPixelFormat::YUV422P10: + return "YUV422P10"; + case VideoPixelFormat::YUV444P9: + return "YUV444P9"; + case VideoPixelFormat::YUV444P10: + return "YUV444P10"; + case VideoPixelFormat::YUV420P12: + return "YUV420P12"; + case VideoPixelFormat::YUV422P12: + return "YUV422P12"; + case VideoPixelFormat::YUV444P12: + return "YUV444P12"; + case VideoPixelFormat::Y16: + return "Y16"; + case VideoPixelFormat::ABGR: + return "ABGR"; + case VideoPixelFormat::XBGR: + return "XBGR"; + case VideoPixelFormat::P016LE: + return "P016LE"; + case VideoPixelFormat::XR30: + return "XR30"; + case VideoPixelFormat::XB30: + return "XB30"; + case VideoPixelFormat::BGRA: + return "BGRA"; + case VideoPixelFormat::UNKNOWN: + return "UNKNOWN"; + } +} + +std::string fourccToString(uint32_t fourcc) { + std::string result = "0000"; + for (size_t i = 0; i < 4; ++i, fourcc >>= 8) { + const char c = static_cast<char>(fourcc & 0xFF); + if (c <= 0x1f || c >= 0x7f) { + return (std::stringstream("0x") << std::hex << fourcc).str(); + } + result[i] = c; + } + return result; +} + +size_t bitDepth(VideoPixelFormat format) { + switch (format) { + case VideoPixelFormat::I420: + case VideoPixelFormat::YV12: + case VideoPixelFormat::I422: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + case VideoPixelFormat::NV21: + case VideoPixelFormat::YUY2: + case VideoPixelFormat::ARGB: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::BGRA: + return 8; + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV444P9: + return 9; + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + return 10; + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + return 12; + case VideoPixelFormat::Y16: + case VideoPixelFormat::P016LE: + return 16; + case VideoPixelFormat::UNKNOWN: + ALOGE("Invalid pixel format"); + return 0; + } +} + +// If it is required to allocate aligned to multiple-of-two size overall for the +// frame of pixel |format|. +static bool RequiresEvenSizeAllocation(VideoPixelFormat format) { + switch (format) { + case VideoPixelFormat::ARGB: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::Y16: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + case VideoPixelFormat::BGRA: + return false; + case VideoPixelFormat::NV12: + case VideoPixelFormat::NV21: + case VideoPixelFormat::I420: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::YUY2: + case VideoPixelFormat::YV12: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + case VideoPixelFormat::I420A: + case VideoPixelFormat::P016LE: + return true; + case VideoPixelFormat::UNKNOWN: + ALOGE("Invalid pixel format"); + return false; + } +} + +size_t numPlanes(VideoPixelFormat format) { + switch (format) { + case VideoPixelFormat::YUY2: + case VideoPixelFormat::ARGB: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::Y16: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + return 1; + case VideoPixelFormat::NV12: + case VideoPixelFormat::NV21: + case VideoPixelFormat::P016LE: + return 2; + case VideoPixelFormat::I420: + case VideoPixelFormat::YV12: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + return 3; + case VideoPixelFormat::I420A: + return 4; + case VideoPixelFormat::UNKNOWN: + // Note: VideoPixelFormat::UNKNOWN is used for end-of-stream frame. + return 0; + } +} + +size_t allocationSize(VideoPixelFormat format, const android::ui::Size& coded_size) { + size_t total = 0; + for (size_t i = 0; i < numPlanes(format); ++i) { + android::ui::Size plane_size = planeSize(format, i, coded_size); + total += (plane_size.width * plane_size.height); + } + + return total; +} + +android::ui::Size planeSize(VideoPixelFormat format, size_t plane, + const android::ui::Size& coded_size) { + ALOG_ASSERT(isValidPlane(plane, format)); + + int width = coded_size.width; + int height = coded_size.height; + if (RequiresEvenSizeAllocation(format)) { + // Align to multiple-of-two size overall. This ensures that non-subsampled + // planes can be addressed by pixel with the same scaling as the subsampled + // planes. + width = base::bits::Align(width, 2); + height = base::bits::Align(height, 2); + } + + const android::ui::Size subsample = SampleSize(format, plane); + ALOG_ASSERT(width % subsample.width == 0); + ALOG_ASSERT(height % subsample.height == 0); + return android::ui::Size(bytesPerElement(format, plane) * width / subsample.width, + height / subsample.height); +} + +int planeHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane) { + ALOG_ASSERT(isValidPlane(plane, format)); + const int bitsPerElement = 8 * bytesPerElement(format, plane); + const int horizPixelsPerElement = SampleSize(format, plane).width; + ALOG_ASSERT(bitsPerElement % horizPixelsPerElement == 0); + return bitsPerElement / horizPixelsPerElement; +} + +int planeBitsPerPixel(VideoPixelFormat format, size_t plane) { + ALOG_ASSERT(isValidPlane(plane, format)); + return planeHorizontalBitsPerPixel(format, plane) / SampleSize(format, plane).height; +} + +int bytesPerElement(VideoPixelFormat format, size_t plane) { + ALOG_ASSERT(isValidPlane(format, plane)); + switch (format) { + case VideoPixelFormat::ARGB: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + return 4; + case VideoPixelFormat::RGB24: + return 3; + case VideoPixelFormat::Y16: + case VideoPixelFormat::YUY2: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::YUV422P12: + case VideoPixelFormat::YUV444P12: + case VideoPixelFormat::P016LE: + return 2; + case VideoPixelFormat::NV12: + case VideoPixelFormat::NV21: { + static const int bytes_per_element[] = {1, 2}; + ALOG_ASSERT(plane < base::size(bytes_per_element)); + return bytes_per_element[plane]; + } + case VideoPixelFormat::YV12: + case VideoPixelFormat::I420: + case VideoPixelFormat::I422: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I444: + return 1; + case VideoPixelFormat::MJPEG: + return 0; + case VideoPixelFormat::UNKNOWN: + ALOGE("Invalid pixel format"); + return 0; + } +} + +bool isValidPlane(VideoPixelFormat format, size_t plane) { + ALOG_ASSERT(numPlanes(format) <= static_cast<size_t>(kMaxPlanes)); + return plane < numPlanes(format); +} + +android::ui::Size SampleSize(VideoPixelFormat format, size_t plane) { + ALOG_ASSERT(isValidPlane(format, plane)); + + switch (plane) { + case kYPlane: // and kARGBPlane: + case kAPlane: + return android::ui::Size(1, 1); + + case kUPlane: // and kUVPlane: + case kVPlane: + switch (format) { + case VideoPixelFormat::I444: + case VideoPixelFormat::YUV444P9: + case VideoPixelFormat::YUV444P10: + case VideoPixelFormat::YUV444P12: + case VideoPixelFormat::Y16: + return android::ui::Size(1, 1); + + case VideoPixelFormat::I422: + case VideoPixelFormat::YUV422P9: + case VideoPixelFormat::YUV422P10: + case VideoPixelFormat::YUV422P12: + return android::ui::Size(2, 1); + + case VideoPixelFormat::YV12: + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::NV12: + case VideoPixelFormat::NV21: + case VideoPixelFormat::YUV420P9: + case VideoPixelFormat::YUV420P10: + case VideoPixelFormat::YUV420P12: + case VideoPixelFormat::P016LE: + return android::ui::Size(2, 2); + + case VideoPixelFormat::UNKNOWN: + case VideoPixelFormat::YUY2: + case VideoPixelFormat::ARGB: + case VideoPixelFormat::XRGB: + case VideoPixelFormat::RGB24: + case VideoPixelFormat::MJPEG: + case VideoPixelFormat::ABGR: + case VideoPixelFormat::XBGR: + case VideoPixelFormat::XR30: + case VideoPixelFormat::XB30: + case VideoPixelFormat::BGRA: + ALOGE("Invalid pixel format"); + } + } + + return android::ui::Size(); +} + +} // namespace android diff --git a/common/VideoTypes.cpp b/common/VideoTypes.cpp index 1ecceca..c123ad1 100644 --- a/common/VideoTypes.cpp +++ b/common/VideoTypes.cpp @@ -22,6 +22,65 @@ const char* VideoCodecToString(VideoCodec codec) { } } +const char* profileToString(C2Config::profile_t profile) { + switch (profile) { + case C2Config::PROFILE_UNUSED: + return "unused"; + case C2Config::PROFILE_AVC_BASELINE: + return "h264 baseline"; + case C2Config::PROFILE_AVC_MAIN: + return "h264 main"; + case C2Config::PROFILE_AVC_EXTENDED: + return "h264 extended"; + case C2Config::PROFILE_AVC_HIGH: + return "h264 high"; + case C2Config::PROFILE_AVC_HIGH_10: + return "h264 high 10"; + case C2Config::PROFILE_AVC_HIGH_422: + return "h264 high 4:2:2"; + case C2Config::PROFILE_AVC_HIGH_444_PREDICTIVE: + return "h264 high 4:4:4 predictive"; + case C2Config::PROFILE_AVC_SCALABLE_BASELINE: + return "h264 scalable baseline"; + case C2Config::PROFILE_AVC_SCALABLE_HIGH: + return "h264 scalable high"; + case C2Config::PROFILE_AVC_STEREO_HIGH: + return "h264 stereo high"; + case C2Config::PROFILE_AVC_MULTIVIEW_HIGH: + return "h264 multiview high"; + case C2Config::PROFILE_HEVC_MAIN: + return "hevc main"; + case C2Config::PROFILE_HEVC_MAIN_10: + return "hevc main 10"; + case C2Config::PROFILE_HEVC_MAIN_STILL: + return "hevc main still-picture"; + case C2Config::PROFILE_VP8_0: + return "vp8 profile0"; + case C2Config::PROFILE_VP8_1: + return "vp8 profile1"; + case C2Config::PROFILE_VP8_2: + return "vp8 profile2"; + case C2Config::PROFILE_VP8_3: + return "vp8 profile3"; + case C2Config::PROFILE_VP9_0: + return "vp9 profile0"; + case C2Config::PROFILE_VP9_1: + return "vp9 profile1"; + case C2Config::PROFILE_VP9_2: + return "vp9 profile2"; + case C2Config::PROFILE_VP9_3: + return "vp9 profile3"; + case C2Config::PROFILE_AV1_0: + return "av1 profile 0"; + case C2Config::PROFILE_AV1_1: + return "av1 profile 1"; + case C2Config::PROFILE_AV1_2: + return "av1 profile 2"; + default: + return "unknown"; + } +} + const char* HalPixelFormatToString(HalPixelFormat format) { switch (format) { case HalPixelFormat::UNKNOWN: diff --git a/common/include/v4l2_codec2/common/Common.h b/common/include/v4l2_codec2/common/Common.h index 650b7a7..0775af1 100644 --- a/common/include/v4l2_codec2/common/Common.h +++ b/common/include/v4l2_codec2/common/Common.h @@ -7,14 +7,47 @@ #include <inttypes.h> +#include <optional> +#include <string> +#include <vector> + +#include <ui/Rect.h> +#include <ui/Size.h> + +#include <v4l2_codec2/common/VideoPixelFormat.h> + namespace android { -// The offset and stride of a video frame plane. +// The stride, offset and size of a video frame plane. struct VideoFramePlane { - uint32_t mOffset; - uint32_t mStride; + uint32_t mStride = 0; + size_t mOffset = 0; + size_t mSize = 0; +}; + +// A video frame's layout, containing pixel format, size and layout of individual planes. +struct VideoFrameLayout { + VideoPixelFormat mFormat = VideoPixelFormat::UNKNOWN; + android::ui::Size mCodedSize; + std::vector<VideoFramePlane> mPlanes; + bool mMultiPlanar = false; }; +// Check whether |rect1| completely contains |rect2|. +bool contains(const Rect& rect1, const Rect& rect2); + +// Convert the specified |rect| to a string. +std::string toString(const Rect& rect); + +// Get the area encapsulated by the |size|. Returns nullopt if multiplication causes overflow. +std::optional<int> getArea(const ui::Size& size); + +// Check whether the specified |size| is empty +bool isEmpty(const ui::Size& size); + +// Convert the specified |size| to a string. +std::string toString(const ui::Size& size); + } // namespace android #endif // ANDROID_V4L2_CODEC2_COMMON_COMMON_H diff --git a/common/include/v4l2_codec2/common/EncodeHelpers.h b/common/include/v4l2_codec2/common/EncodeHelpers.h index d152ba8..bfbdd05 100644 --- a/common/include/v4l2_codec2/common/EncodeHelpers.h +++ b/common/include/v4l2_codec2/common/EncodeHelpers.h @@ -6,12 +6,10 @@ #define ANDROID_V4L2_CODEC2_COMMON_HELPERS_H #include <C2Config.h> -#include <C2ParamDef.h> #include <system/graphics.h> +#include <ui/Size.h> -#include <size.h> -#include <video_codecs.h> -#include <video_pixel_format.h> +#include <v4l2_codec2/common/VideoPixelFormat.h> namespace android { @@ -29,20 +27,17 @@ struct VideoEncoderAcceleratorConfig { DMABUF = 1, }; - media::VideoPixelFormat mInputFormat; - media::Size mInputVisibleSize; - media::VideoCodecProfile mOutputProfile; + VideoPixelFormat mInputFormat; + ui::Size mInputVisibleSize; + C2Config::profile_t mOutputProfile; uint32_t mInitialBitrate; uint32_t mInitialFramerate; uint8_t mH264OutputLevel; VideoFrameStorageType mStorageType; }; -// Convert the specified C2Config profile to a media::VideoCodecProfile. -media::VideoCodecProfile c2ProfileToVideoCodecProfile(C2Config::profile_t profile); - -// Convert the specified C2Config level to an integer value. -uint8_t c2LevelToLevelIDC(C2Config::level_t level); +// Convert the specified C2Config level to a V4L2 level. +uint8_t c2LevelToV4L2Level(C2Config::level_t level); // Get the specified graphics block in YCbCr format. android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block); @@ -53,38 +48,6 @@ android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block); void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data, size_t length); -// Helper class to parse H264 NAL units from data. -class NalParser { -public: - NalParser(const uint8_t* data, size_t length); - - // Locates the next NAL after |mNextNalStartCodePos|. If there is one, updates |mCurrNalDataPos| - // to the first byte of the NAL data (start code is not included), and |mNextNalStartCodePos| to - // the position of the next start code, and returns true. - // If there is no more NAL, returns false. - // - // Note: This method must be called prior to data() and length(). - bool locateNextNal(); - - // Gets current NAL data (start code is not included). - const uint8_t* data() const; - - // Gets the byte length of current NAL data (start code is not included). - size_t length() const; - -private: - const uint8_t* findNextStartCodePos() const; - - // The byte pattern for the start of a H264 NAL unit. - const uint8_t kNalStartCode[3] = {0x00, 0x00, 0x01}; - // The length in bytes of the NAL-unit start pattern. - const size_t kNalStartCodeLength = 3; - - const uint8_t* mCurrNalDataPos; - const uint8_t* mDataEnd; - const uint8_t* mNextNalStartCodePos; -}; - } // namespace android #endif // ANDROID_V4L2_CODEC2_COMMON_HELPERS_H diff --git a/common/include/v4l2_codec2/common/FormatConverter.h b/common/include/v4l2_codec2/common/FormatConverter.h index f00f115..bc3f85a 100644 --- a/common/include/v4l2_codec2/common/FormatConverter.h +++ b/common/include/v4l2_codec2/common/FormatConverter.h @@ -10,9 +10,10 @@ #include <vector> #include <C2Buffer.h> -#include <size.h> +#include <ui/Size.h> #include <utils/StrongPointer.h> -#include <video_pixel_format.h> + +#include <v4l2_codec2/common/VideoPixelFormat.h> namespace android { @@ -50,10 +51,9 @@ public: // Create FormatConverter instance and initialize it, nullptr will be returned on // initialization error. - static std::unique_ptr<FormatConverter> Create(media::VideoPixelFormat outFormat, - const media::Size& visibleSize, - uint32_t inputCount, - const media::Size& codedSize); + static std::unique_ptr<FormatConverter> Create(VideoPixelFormat outFormat, + const ui::Size& visibleSize, uint32_t inputCount, + const ui::Size& codedSize); // Convert the input block into the alternative block with required pixel format and return it, // or return the original block if zero-copy is applied. @@ -93,8 +93,8 @@ private: // Initialize foramt converter. It will pre-allocate a set of graphic blocks as |codedSize| and // |outFormat|. This function should be called prior to other functions. - c2_status_t initialize(media::VideoPixelFormat outFormat, const media::Size& visibleSize, - uint32_t inputCount, const media::Size& codedSize); + c2_status_t initialize(VideoPixelFormat outFormat, const ui::Size& visibleSize, + uint32_t inputCount, const ui::Size& codedSize); // The array of block entries. std::vector<std::unique_ptr<BlockEntry>> mGraphicBlocks; @@ -106,8 +106,8 @@ private: std::unique_ptr<uint8_t[]> mTempPlaneU; std::unique_ptr<uint8_t[]> mTempPlaneV; - media::VideoPixelFormat mOutFormat = media::VideoPixelFormat::PIXEL_FORMAT_UNKNOWN; - media::Size mVisibleSize; + VideoPixelFormat mOutFormat = VideoPixelFormat::UNKNOWN; + ui::Size mVisibleSize; }; } // namespace android diff --git a/common/include/v4l2_codec2/common/Fourcc.h b/common/include/v4l2_codec2/common/Fourcc.h new file mode 100644 index 0000000..a0f5fc4 --- /dev/null +++ b/common/include/v4l2_codec2/common/Fourcc.h @@ -0,0 +1,157 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: 27c98933749f + +#ifndef ANDROID_V4L2_CODEC2_COMMON_FOURCC_H +#define ANDROID_V4L2_CODEC2_COMMON_FOURCC_H + +#include <stdint.h> +#include <optional> +#include <string> + +#include <v4l2_codec2/common/VideoPixelFormat.h> + +namespace android { + +// Composes a Fourcc value. +constexpr uint32_t composeFourcc(char a, char b, char c, char d) { + return static_cast<uint32_t>(a) | (static_cast<uint32_t>(b) << 8) | + (static_cast<uint32_t>(c) << 16) | (static_cast<uint32_t>(d) << 24); +} + +// Fourcc enum holder and converters. +// Usage: +// Fourcc f1(Fourcc::AR24); +// EXPECT_EQ("AR24", f1.ToString()); +// Fourcc f2 = Fourcc::FromVideoPixelFormat(PIXEL_FORMAT_ARGB); +// EXPECT_EQ(f2, f1); +class Fourcc { +public: + enum Value : uint32_t { + // RGB formats. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-rgb.html + // Maps to PIXEL_FORMAT_ARGB, V4L2_PIX_FMT_ABGR32, VA_FOURCC_BGRA. + // 32bpp BGRA (byte-order), 1 plane. + AR24 = composeFourcc('A', 'R', '2', '4'), + + // Maps to PIXEL_FORMAT_ABGR, V4L2_PIX_FMT_RGBA32, VA_FOURCC_RGBA. + // 32bpp RGBA (byte-order), 1 plane + AB24 = composeFourcc('A', 'B', '2', '4'), + + // Maps to PIXEL_FORMAT_XRGB, V4L2_PIX_FMT_XBGR32, VA_FOURCC_BGRX. + // 32bpp BGRX (byte-order), 1 plane. + XR24 = composeFourcc('X', 'R', '2', '4'), + + // Maps to PIXEL_FORMAT_XBGR, V4L2_PIX_FMT_RGBX32, VA_FOURCC_RGBX. + // 32bpp RGBX (byte-order), 1 plane. + XB24 = composeFourcc('X', 'B', '2', '4'), + + // Maps to PIXEL_FORMAT_BGRA, V4L2_PIX_FMT_RGB32, VA_FOURCC_ARGB. + // 32bpp ARGB (byte-order), 1 plane. + // Note that V4L2_PIX_FMT_RGB32("RGB4") is deprecated and replaced by + // V4L2_PIX_FMT_ARGB32("BA24"), however, some board relies on the fourcc mapping so we keep + // it as-is. + RGB4 = composeFourcc('R', 'G', 'B', '4'), + + // YUV420 single-planar formats. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv420.html + // Maps to PIXEL_FORMAT_I420, V4L2_PIX_FMT_YUV420, VA_FOURCC_I420. + // 12bpp YUV planar 1x1 Y, 2x2 UV samples. + YU12 = composeFourcc('Y', 'U', '1', '2'), + // Maps to PIXEL_FORMAT_YV12, V4L2_PIX_FMT_YVU420, VA_FOURCC_YV12. + // 12bpp YVU planar 1x1 Y, 2x2 VU samples. + YV12 = composeFourcc('Y', 'V', '1', '2'), + + // YUV420 multi-planar format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv420m.htm + // Maps to PIXEL_FORMAT_I420, V4L2_PIX_FMT_YUV420M. + YM12 = composeFourcc('Y', 'M', '1', '2'), + // Maps to PIXEL_FORMAT_YV12, V4L2_PIX_FMT_YVU420M. + YM21 = composeFourcc('Y', 'M', '2', '1'), + + // YUYV format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuyv.html + // Maps to PIXEL_FORMAT_YUY2, V4L2_PIX_FMT_YUYV, VA_FOURCC_YUY2. + // 16bpp YUV planar (YUV 4:2:2), YUYV (byte-order), 1 plane. + YUYV = composeFourcc('Y', 'U', 'Y', 'V'), + + // NV12 single-planar format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-nv12.html + // Maps to PIXEL_FORMAT_NV12, V4L2_PIX_FMT_NV12, VA_FOURCC_NV12. + // 12bpp with Y plane followed by a 2x2 interleaved UV plane. + NV12 = composeFourcc('N', 'V', '1', '2'), + // Maps to PIXEL_FORMAT_NV21, V4L2_PIX_FMT_NV21, VA_FOURCC_NV21. + // 12bpp with Y plane followed by a 2x2 interleaved VU plane. + NV21 = composeFourcc('N', 'V', '2', '1'), + + // NV12 multi-planar format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-nv12m.html + // Maps to PIXEL_FORMAT_NV12, V4L2_PIX_FMT_NV12M, + NM12 = composeFourcc('N', 'M', '1', '2'), + // Maps to PIXEL_FORMAT_NV21, V4L2_PIX_FMT_NV21M. + NM21 = composeFourcc('N', 'M', '2', '1'), + + // YUV422 multi-planar format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-yuv422m.html + // Maps to PIXEL_FORMAT_I422, V4L2_PIX_FMT_YUV422M + // 16bpp YUV planar 1x1 Y, 2x1 UV samples. + YM16 = composeFourcc('Y', 'M', '1', '6'), + + // V4L2 proprietary format. + // https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/pixfmt-reserved.html + // Maps to V4L2_PIX_FMT_MT21C. + // It is used for MT8173 hardware video decoder output and should be converted by MT8173 image + // processor for compositor to render. + MT21 = composeFourcc('M', 'T', '2', '1'), + // Maps to V4L2_PIX_FMT_MM21. + // It is used for MT8183 hardware video decoder. + MM21 = composeFourcc('M', 'M', '2', '1'), + }; + + explicit Fourcc(Fourcc::Value fourcc); + Fourcc& operator=(const Fourcc& fourcc); + ~Fourcc(); + + bool operator==(const Fourcc& rhs) const { return mValue == rhs.mValue; } + + // Factory methods: + + // Builds a Fourcc from a given fourcc code. This will return a valid Fourcc if the argument is + // part of the |Value| enum, or nullopt otherwise. + static std::optional<Fourcc> fromUint32(uint32_t fourcc); + + // Converts a VideoPixelFormat to Fourcc. Returns nullopt for invalid input. Note that a + // VideoPixelFormat may have two Fourcc counterparts. Caller has to specify if it is for + // single-planar or multi-planar format. + static std::optional<Fourcc> fromVideoPixelFormat(VideoPixelFormat pixelFormat, + bool singlePlanar = true); + // Converts a V4L2PixFmt to Fourcc. Returns nullopt for invalid input. + static std::optional<Fourcc> fromV4L2PixFmt(uint32_t v4l2PixFmt); + + // Value getters: + // Returns the VideoPixelFormat counterpart of the value. Returns PIXEL_FORMAT_UNKNOWN if no + // mapping is found. + VideoPixelFormat toVideoPixelFormat() const; + // Returns the V4L2PixFmt counterpart of the value. Returns 0 if no mapping is found. + uint32_t toV4L2PixFmt() const; + + // Returns the single-planar Fourcc of the value. If value is a single-planar, returns the same + // Fourcc. Returns nullopt if no mapping is found. + std::optional<Fourcc> toSinglePlanar() const; + + // Returns whether |value_| is multi planar format. + bool isMultiPlanar() const; + + // Outputs human readable fourcc string, e.g. "NV12". + std::string toString() const; + +private: + Value mValue; +}; + +bool operator!=(const Fourcc& lhs, const Fourcc& rhs); + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMMON_FOURCC_H diff --git a/common/include/v4l2_codec2/common/NalParser.h b/common/include/v4l2_codec2/common/NalParser.h new file mode 100644 index 0000000..69f56c3 --- /dev/null +++ b/common/include/v4l2_codec2/common/NalParser.h @@ -0,0 +1,68 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_COMMON_NALPARSER_H +#define ANDROID_V4L2_CODEC2_COMMON_NALPARSER_H + +#include <stdint.h> + +namespace android { + +// Helper class to parse H264 NAL units from data. +class NalParser { +public: + // Type of a SPS NAL unit. + static constexpr uint8_t kSPSType = 7; + // Type of a PPS NAL unit. + static constexpr uint8_t kPPSType = 8; + + // Parameters related to a video's color aspects. + struct ColorAspects { + int32_t primaries; + int32_t transfer; + int32_t coeffs; + bool fullRange; + }; + + NalParser(const uint8_t* data, size_t length); + + // Locates the next NAL after |mNextNalStartCodePos|. If there is one, updates |mCurrNalDataPos| + // to the first byte of the NAL data (start code is not included), and |mNextNalStartCodePos| to + // the position of the next start code, and returns true. + // If there is no more NAL, returns false. + // + // Note: This method must be called prior to data() and length(). + bool locateNextNal(); + + // Locate the sequence parameter set (SPS). + bool locateSPS(); + + // Gets current NAL data (start code is not included). + const uint8_t* data() const; + + // Gets the byte length of current NAL data (start code is not included). + size_t length() const; + + // Get the type of the current NAL unit. + uint8_t type() const; + + // Find the H.264 video's color aspects in the current SPS NAL. + bool findCodedColorAspects(ColorAspects* colorAspects); + +private: + const uint8_t* findNextStartCodePos() const; + + // The byte pattern for the start of a H264 NAL unit. + const uint8_t kNalStartCode[3] = {0x00, 0x00, 0x01}; + // The length in bytes of the NAL-unit start pattern. + const size_t kNalStartCodeLength = 3; + + const uint8_t* mCurrNalDataPos; + const uint8_t* mDataEnd; + const uint8_t* mNextNalStartCodePos; +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMMON_NALPARSER_H diff --git a/common/include/v4l2_codec2/common/V4L2ComponentCommon.h b/common/include/v4l2_codec2/common/V4L2ComponentCommon.h index 6e3efa8..b8cf031 100644 --- a/common/include/v4l2_codec2/common/V4L2ComponentCommon.h +++ b/common/include/v4l2_codec2/common/V4L2ComponentCommon.h @@ -12,6 +12,8 @@ namespace android { // Defines the names of all supported components. struct V4L2ComponentName { static const std::string kH264Encoder; + static const std::string kVP8Encoder; + static const std::string kVP9Encoder; static const std::string kH264Decoder; static const std::string kVP8Decoder; diff --git a/common/include/v4l2_codec2/common/V4L2Device.h b/common/include/v4l2_codec2/common/V4L2Device.h new file mode 100644 index 0000000..b4c909c --- /dev/null +++ b/common/include/v4l2_codec2/common/V4L2Device.h @@ -0,0 +1,518 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// This file defines the V4L2Device which is used by the V4L2Decoder and V4L2Encoder classes to +// delegate/pass the device specific handling of any of the functionalities. +// Note: ported from Chromium commit head: 2f13d62f0c0d, but some parts have been removed. + +#ifndef ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_H +#define ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_H + +#include <linux/videodev2.h> +#include <stddef.h> +#include <stdint.h> + +#include <optional> +#include <vector> + +#include <C2Config.h> +#include <base/containers/flat_map.h> +#include <base/files/scoped_file.h> +#include <base/memory/ref_counted.h> + +#include <ui/Size.h> +#include <v4l2_codec2/common/Common.h> +#include <v4l2_codec2/common/V4L2DevicePoller.h> +#include <v4l2_codec2/common/VideoTypes.h> + +namespace android { + +class V4L2Queue; +class V4L2BufferRefBase; +class V4L2BuffersList; +class V4L2DecodeSurface; + +// Wrapper for the 'v4l2_ext_control' structure. +struct V4L2ExtCtrl { + V4L2ExtCtrl(uint32_t id); + V4L2ExtCtrl(uint32_t id, int32_t val); + struct v4l2_ext_control ctrl; +}; + +// A unique reference to a buffer for clients to prepare and submit. +// +// Clients can prepare a buffer for queuing using the methods of this class, and then either queue +// it using the Queue() method corresponding to the memory type of the buffer, or drop the reference +// to make the buffer available again. +class V4L2WritableBufferRef { +public: + V4L2WritableBufferRef(V4L2WritableBufferRef&& other); + V4L2WritableBufferRef() = delete; + V4L2WritableBufferRef& operator=(V4L2WritableBufferRef&& other); + + // Return the memory type of the buffer. Useful to e.g. decide which Queue() method to use. + enum v4l2_memory memory() const; + + // Queue a MMAP buffer. If successful, true is returned and the reference to the buffer is + // dropped so this reference becomes invalid. In case of error, false is returned and the buffer + // is returned to the free list. + bool queueMMap() &&; + // Queue a USERPTR buffer, assigning |ptrs| as pointer for each plane. The size of |ptrs| must + // be equal to the number of planes of this buffer. If successful, true is returned and the + // reference to the buffer is dropped so this reference becomes invalid. In case of error, false + // is returned and the buffer is returned to the free list. + bool queueUserPtr(const std::vector<void*>& ptrs) &&; + // Queue a DMABUF buffer, assigning |fds| as file descriptors for each plane. It is allowed the + // number of |fds| might be greater than the number of planes of this buffer. It happens when + // the v4l2 pixel format is single planar. The fd of the first plane is only used in that case. + // If successful, true is returned and the reference to the buffer is dropped so this reference + // becomes invalid. In case of error, false is returned and the buffer is returned to the free + // list. + bool queueDMABuf(const std::vector<int>& fds) &&; + + // Returns the number of planes in this buffer. + size_t planesCount() const; + // Returns the size of the requested |plane|, in bytes. + size_t getPlaneSize(const size_t plane) const; + // Set the size of the requested |plane|, in bytes. It is only valid for USERPTR and DMABUF + // buffers. When using an MMAP buffer, this method triggers an assert and is a no-op for release + // builds. + void setPlaneSize(const size_t plane, const size_t size); + // This method can only be used with MMAP buffers. It will return a pointer to the data of the + // |plane|th plane. In case of error (invalid plane index or mapping failed), a nullptr is + // returned. + void* getPlaneMapping(const size_t plane); + // Set the timestamp field for this buffer. + void setTimeStamp(const struct timeval& timestamp); + // Return the previously-set timestamp field for this buffer. + const struct timeval& getTimeStamp() const; + // Set the number of bytes used for |plane|. + void setPlaneBytesUsed(const size_t plane, const size_t bytesUsed); + // Returns the previously-set number of bytes used for |plane|. + size_t getPlaneBytesUsed(const size_t plane) const; + // Set the data offset for |plane|, in bytes. + void setPlaneDataOffset(const size_t plane, const size_t dataOffset); + + // Return the V4L2 buffer ID of the underlying buffer. + size_t bufferId() const; + + ~V4L2WritableBufferRef(); + +private: + friend class V4L2BufferRefFactory; + + // Do the actual queue operation once the v4l2_buffer structure is properly filled. + bool doQueue() &&; + + V4L2WritableBufferRef(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue); + + V4L2WritableBufferRef(const V4L2WritableBufferRef&) = delete; + V4L2WritableBufferRef& operator=(const V4L2WritableBufferRef&) = delete; + + std::unique_ptr<V4L2BufferRefBase> mBufferData; + + SEQUENCE_CHECKER(mSequenceChecker); +}; + +// A reference to a read-only, dequeued buffer. +// +// Clients use this class to query the buffer state and content, and are guaranteed that the buffer +// will not be reused until all references are destroyed. +// All methods of this class must be called from the same sequence, but instances of +// V4L2ReadableBuffer objects can be destroyed from any sequence. They can even outlive the V4L2 +// buffers they originate from. This flexibility is required because V4L2ReadableBufferRefs can be +// embedded into VideoFrames, which are then passed to other threads and not necessarily destroyed +// before the V4L2Queue buffers are freed. +class V4L2ReadableBuffer : public base::RefCountedThreadSafe<V4L2ReadableBuffer> { +public: + // Returns whether the V4L2_BUF_FLAG_LAST flag is set for this buffer. + bool isLast() const; + // Returns whether the V4L2_BUF_FLAG_KEYFRAME flag is set for this buffer. + bool isKeyframe() const; + // Return the timestamp set by the driver on this buffer. + struct timeval getTimeStamp() const; + // Returns the number of planes in this buffer. + size_t planesCount() const; + // Returns the number of bytes used for |plane|. + size_t getPlaneBytesUsed(size_t plane) const; + // Returns the data offset for |plane|. + size_t getPlaneDataOffset(size_t plane) const; + // This method can only be used with MMAP buffers. It will return a pointer to the data of the + // |plane|th plane. In case of error (invalid plane index or mapping failed), a nullptr is + // returned. + const void* getPlaneMapping(const size_t plane) const; + + // Return the V4L2 buffer ID of the underlying buffer. + size_t bufferId() const; + +private: + friend class V4L2BufferRefFactory; + friend class base::RefCountedThreadSafe<V4L2ReadableBuffer>; + + ~V4L2ReadableBuffer(); + + V4L2ReadableBuffer(const struct v4l2_buffer& v4l2Buffer, base::WeakPtr<V4L2Queue> queue); + + V4L2ReadableBuffer(const V4L2ReadableBuffer&) = delete; + V4L2ReadableBuffer& operator=(const V4L2ReadableBuffer&) = delete; + + std::unique_ptr<V4L2BufferRefBase> mBufferData; + + SEQUENCE_CHECKER(mSequenceChecker); +}; + +// Shortcut for naming consistency. +using V4L2ReadableBufferRef = scoped_refptr<V4L2ReadableBuffer>; + +class V4L2Device; +class V4L2Buffer; + +// Interface representing a specific queue of a |V4L2Device|. It provides free and queued buffer +// management that is commonly required by clients. +// +// Buffers managed by this class undergo the following cycle: +// 1) Allocated buffers are put into a free buffers pool, indicating that they are used neither by +// the client nor the hardware. +// 2) The client obtains a unique, writable reference to one of the free buffers in order to set +// its content and other parameters. +// 3) The client then queues the buffer obtained in 2), which invalidates its reference. The buffer +// is now prepared to be processed by the hardware. +// 4) Once the hardware is done with the buffer, it is ready to be dequeued by the client. The +// client obtains a read-only, counted reference to the buffer and can read its content and +// metadata, as well as making other references to it. The buffer will not be reused until all +// the references are dropped. Once this happens, the buffer goes back to the free list described +// in 1). +class V4L2Queue : public base::RefCountedThreadSafe<V4L2Queue> { +public: + // Set |fourcc| as the current format on this queue. |size| corresponds to the desired buffer's + // dimensions (i.e. width and height members of v4l2_pix_format_mplane (if not applicable, pass + // Size()). + // |bufferSize| is the desired size in bytes of the buffer for single-planar formats (i.e. + // sizeimage of the first plane). It can be set to 0 if not relevant for the desired format. + // |stride| is the desired stride in bytes of the buffer (i.e. bytesperline). It can be set to 0 + // if not relevant or to let the driver decide. If the format could be set, then the + // |v4l2_format| reflecting the actual format is returned. It is guaranteed to feature the + // specified |fourcc|, but any other parameter (including |size| and |bufferSize| may have been + // adjusted by the driver, so the caller must check their values. + std::optional<struct v4l2_format> setFormat(uint32_t fourcc, const ui::Size& size, + size_t bufferSize, + uint32_t stride = 0) WARN_UNUSED_RESULT; + + // Identical to |setFormat|, but does not actually apply the format, and can be called anytime. + // Returns an adjusted V4L2 format if |fourcc| is supported by the queue, or |nullopt| if + // |fourcc| is not supported or an ioctl error happened. + std::optional<struct v4l2_format> tryFormat(uint32_t fourcc, const ui::Size& size, + size_t bufferSize) WARN_UNUSED_RESULT; + + // Returns the currently set format on the queue. The result is returned as a std::pair where + // the first member is the format, or base::nullopt if the format could not be obtained due to + // an ioctl error. The second member is only used in case of an error and contains the |errno| + // set by the failing ioctl. If the first member is not base::nullopt, the second member will + // always be zero. + // + // If the second member is 0, then the first member is guaranteed to have a valid value. So + // clients that are not interested in the precise error message can just check that the first + // member is valid and go on. + // + // This pair is used because not all failures to get the format are necessarily errors, so we + // need to way to let the use decide whether it is one or not. + std::pair<std::optional<struct v4l2_format>, int> getFormat(); + + // Allocate |count| buffers for the current format of this queue, with a specific |memory| + // allocation, and returns the number of buffers allocated or zero if an error occurred, or if + // references to any previously allocated buffers are still held by any clients. + // + // The number of allocated buffers may be larger than the number requested, so callers must + // always check the return value. + // + // Calling this method while buffers are still allocated results in an error. + size_t allocateBuffers(size_t count, enum v4l2_memory memory) WARN_UNUSED_RESULT; + + // Deallocate all buffers previously allocated by |allocateBuffers|. Any references to buffers + // previously allocated held by the client must be released, or this call will fail. + bool deallocateBuffers(); + + // Returns the memory usage of v4l2 buffers owned by this V4L2Queue which are mapped in user + // space memory. + size_t getMemoryUsage() const; + + // Returns |mMemory|, memory type of last buffers allocated by this V4L2Queue. + v4l2_memory getMemoryType() const; + + // Return a reference to a free buffer for the caller to prepare and submit, or nullopt if no + // buffer is currently free. + // + // If the caller discards the returned reference, the underlying buffer is made available to + // clients again. + std::optional<V4L2WritableBufferRef> getFreeBuffer(); + std::optional<V4L2WritableBufferRef> getFreeBuffer(size_t requestedBufferId); + + // Attempt to dequeue a buffer, and return a reference to it if one was available. + // + // The first element of the returned pair will be false if an error occurred, in which case the + // second element will be nullptr. If no error occurred, then the first element will be true and + // the second element will contain a reference to the dequeued buffer if one was available, or + // nullptr otherwise. Dequeued buffers will not be reused by the driver until all references to + // them are dropped. + std::pair<bool, V4L2ReadableBufferRef> dequeueBuffer(); + + // Returns true if this queue is currently streaming. + bool isStreaming() const; + // If not currently streaming, starts streaming. Returns true if we started streaming, or were + // already streaming, or false if we were not streaming and an error occurred when attempting to + // start the stream. On failure, any previously-queued buffers will be dequeued without + // processing and made available to the client, while any buffers held by the client will remain + // unchanged and their ownership will remain with the client. + bool streamon(); + // If currently streaming, stops streaming. Also make all queued buffers available to the client + // again regardless of the streaming state. If an error occurred while attempting to stop + // streaming, then false is returned and queued buffers are left untouched since the V4L2 queue + // may still be using them. + bool streamoff(); + + // Returns the number of buffers currently allocated for this queue. + size_t allocatedBuffersCount() const; + // Returns the number of currently free buffers on this queue. + size_t freeBuffersCount() const; + // Returns the number of buffers currently queued on this queue. + size_t queuedBuffersCount() const; + +private: + ~V4L2Queue(); + + V4L2Queue(const V4L2Queue&) = delete; + V4L2Queue& operator=(const V4L2Queue&) = delete; + + // Called when clients request a buffer to be queued. + bool queueBuffer(struct v4l2_buffer* v4l2Buffer); + + const enum v4l2_buf_type mType; + enum v4l2_memory mMemory = V4L2_MEMORY_MMAP; + bool mIsStreaming = false; + size_t mPlanesCount = 0; + // Current format as set by SetFormat. + std::optional<struct v4l2_format> mCurrentFormat; + + std::vector<std::unique_ptr<V4L2Buffer>> mBuffers; + + // Buffers that are available for client to get and submit. Buffers in this list are not + // referenced by anyone else than ourselves. + scoped_refptr<V4L2BuffersList> mFreeBuffers; + // Buffers that have been queued by the client, and not dequeued yet. + std::set<size_t> mQueuedBuffers; + + scoped_refptr<V4L2Device> mDevice; + // Callback to call in this queue's destructor. + base::OnceClosure mDestroyCb; + + V4L2Queue(scoped_refptr<V4L2Device> dev, enum v4l2_buf_type type, base::OnceClosure destroyCb); + friend class V4L2QueueFactory; + friend class V4L2BufferRefBase; + friend class base::RefCountedThreadSafe<V4L2Queue>; + + SEQUENCE_CHECKER(mSequenceChecker); + + base::WeakPtrFactory<V4L2Queue> mWeakThisFactory{this}; +}; + +class V4L2Device : public base::RefCountedThreadSafe<V4L2Device> { +public: + // Specification of an encoding profile supported by an encoder. + struct SupportedEncodeProfile { + C2Config::profile_t profile = C2Config::PROFILE_UNUSED; + ui::Size min_resolution; + ui::Size max_resolution; + uint32_t max_framerate_numerator = 0; + uint32_t max_framerate_denominator = 0; + }; + using SupportedEncodeProfiles = std::vector<SupportedEncodeProfile>; + + // Specification of a decoding profile supported by an decoder. + // |max_resolution| and |min_resolution| are inclusive. + struct SupportedDecodeProfile { + C2Config::profile_t profile = C2Config::PROFILE_UNUSED; + ui::Size max_resolution; + ui::Size min_resolution; + bool encrypted_only = false; + }; + using SupportedDecodeProfiles = std::vector<SupportedDecodeProfile>; + + // Utility format conversion functions + // If there is no corresponding single- or multi-planar format, returns 0. + static uint32_t C2ProfileToV4L2PixFmt(C2Config::profile_t profile, bool sliceBased); + static C2Config::profile_t v4L2ProfileToC2Profile(VideoCodec codec, uint32_t profile); + std::vector<C2Config::profile_t> v4L2PixFmtToC2Profiles(uint32_t pixFmt, bool isEncoder); + // Calculates the largest plane's allocation size requested by a V4L2 device. + static ui::Size allocatedSizeFromV4L2Format(const struct v4l2_format& format); + + // Convert required H264 profile and level to V4L2 enums. + static int32_t c2ProfileToV4L2H264Profile(C2Config::profile_t profile); + static int32_t h264LevelIdcToV4L2H264Level(uint8_t levelIdc); + + // Converts v4l2_memory to a string. + static const char* v4L2MemoryToString(const v4l2_memory memory); + + // Returns the printable name of a v4l2_buf_type. + static const char* v4L2BufferTypeToString(const enum v4l2_buf_type bufType); + + // Composes human readable string of v4l2_format. + static std::string v4L2FormatToString(const struct v4l2_format& format); + + // Composes human readable string of v4l2_buffer. + static std::string v4L2BufferToString(const struct v4l2_buffer& buffer); + + // Composes VideoFrameLayout based on v4l2_format. If error occurs, it returns base::nullopt. + static std::optional<VideoFrameLayout> v4L2FormatToVideoFrameLayout( + const struct v4l2_format& format); + + // Returns number of planes of |pixFmt|. + static size_t getNumPlanesOfV4L2PixFmt(uint32_t pixFmt); + + enum class Type { kDecoder, kEncoder }; + + // Create and initialize an appropriate V4L2Device instance for the current platform, or return + // nullptr if not available. + static scoped_refptr<V4L2Device> create(); + + // Open a V4L2 device of |type| for use with |v4l2PixFmt|. Return true on success. The device + // will be closed in the destructor. + bool open(Type type, uint32_t v4l2PixFmt); + + // Returns the V4L2Queue corresponding to the requested |type|, or nullptr if the requested + // queue type is not supported. + scoped_refptr<V4L2Queue> getQueue(enum v4l2_buf_type type); + + // Parameters and return value are the same as for the standard ioctl() system call. + int ioctl(int request, void* arg); + + // This method sleeps until either: + // - SetDevicePollInterrupt() is called (on another thread), + // - |pollDevice| is true, and there is new data to be read from the device, + // or an event from the device has arrived; in the latter case + // |*eventPending| will be set to true. + // Returns false on error, true otherwise. This method should be called from a separate thread. + bool poll(bool pollDevice, bool* eventPending); + + // These methods are used to interrupt the thread sleeping on poll() and force it to return + // regardless of device state, which is usually when the client is no longer interested in what + // happens with the device (on cleanup, client state change, etc.). When + // setDevicePollInterrupt() is called, poll() will return immediately, and any subsequent calls + // to it will also do so until clearDevicePollInterrupt() is called. + bool setDevicePollInterrupt(); + bool clearDevicePollInterrupt(); + + // Wrappers for standard mmap/munmap system calls. + void* mmap(void* addr, unsigned int len, int prot, int flags, unsigned int offset); + void munmap(void* addr, unsigned int len); + + // Return a vector of dmabuf file descriptors, exported for V4L2 buffer with |index|, assuming + // the buffer contains |numPlanes| V4L2 planes and is of |bufType|. Return an empty vector on + // failure. The caller is responsible for closing the file descriptors after use. + std::vector<base::ScopedFD> getDmabufsForV4L2Buffer(int index, size_t numPlanes, + enum v4l2_buf_type bufType); + + // Returns the preferred V4L2 input formats for |type| or empty if none. + std::vector<uint32_t> preferredInputFormat(Type type); + + // NOTE: The below methods to query capabilities have a side effect of closing the + // previously-open device, if any, and should not be called after Open(). + + // Get minimum and maximum resolution for fourcc |pixelFormat| and store to |minResolution| and + // |maxResolution|. + void getSupportedResolution(uint32_t pixelFormat, ui::Size* minResolution, + ui::Size* maxResolution); + + std::vector<uint32_t> enumerateSupportedPixelformats(v4l2_buf_type bufType); + + // Return supported profiles for decoder, including only profiles for given fourcc + // |pixelFormats|. + SupportedDecodeProfiles getSupportedDecodeProfiles(const size_t numFormats, + const uint32_t pixelFormats[]); + + // Return supported profiles for encoder. + SupportedEncodeProfiles getSupportedEncodeProfiles(); + + // Start polling on this V4L2Device. |eventCallback| will be posted to the caller's sequence if + // a buffer is ready to be dequeued and/or a V4L2 event has been posted. |errorCallback| will + // be posted to the client's + // sequence if a polling error has occurred. + bool startPolling(android::V4L2DevicePoller::EventCallback eventCallback, + base::RepeatingClosure errorCallback); + // Stop polling this V4L2Device if polling was active. No new events will be posted after this + // method has returned. + bool stopPolling(); + // Schedule a polling event if polling is enabled. This method is intended to be called from + // V4L2Queue, clients should not need to call it directly. + void schedulePoll(); + + // Check whether the V4L2 control with specified |ctrlId| is supported. + bool isCtrlExposed(uint32_t ctrlId); + // Set the specified list of |ctrls| for the specified |ctrlClass|, returns whether the + // operation succeeded. + bool setExtCtrls(uint32_t ctrlClass, std::vector<V4L2ExtCtrl> ctrls); + + // Check whether the V4L2 command with specified |commandId| is supported. + bool isCommandSupported(uint32_t commandId); + // Check whether the V4L2 device has the specified |capabilities|. + bool hasCapabilities(uint32_t capabilities); + +private: + // Vector of video device node paths and corresponding pixelformats supported by each device node. + using Devices = std::vector<std::pair<std::string, std::vector<uint32_t>>>; + + friend class base::RefCountedThreadSafe<V4L2Device>; + V4L2Device(); + ~V4L2Device(); + + V4L2Device(const V4L2Device&) = delete; + V4L2Device& operator=(const V4L2Device&) = delete; + + SupportedDecodeProfiles enumerateSupportedDecodeProfiles(const size_t numFormats, + const uint32_t pixelFormats[]); + + SupportedEncodeProfiles enumerateSupportedEncodeProfiles(); + + // Open device node for |path| as a device of |type|. + bool openDevicePath(const std::string& path, Type type); + + // Close the currently open device. + void closeDevice(); + + // Enumerate all V4L2 devices on the system for |type| and store the results under + // mDevicesByType[type]. + void enumerateDevicesForType(V4L2Device::Type type); + + // Return device information for all devices of |type| available in the system. Enumerates and + // queries devices on first run and caches the results for subsequent calls. + const Devices& getDevicesForType(V4L2Device::Type type); + + // Return device node path for device of |type| supporting |pixFmt|, or an empty string if the + // given combination is not supported by the system. + std::string getDevicePathFor(V4L2Device::Type type, uint32_t pixFmt); + + // Callback that is called upon a queue's destruction, to cleanup its pointer in mQueues. + void onQueueDestroyed(v4l2_buf_type buf_type); + + // Stores information for all devices available on the system for each device Type. + std::map<V4L2Device::Type, Devices> mDevicesByType; + + // The actual device fd. + base::ScopedFD mDeviceFd; + + // eventfd fd to signal device poll thread when its poll() should be interrupted. + base::ScopedFD mDevicePollInterruptFd; + + // Associates a v4l2_buf_type to its queue. + base::flat_map<enum v4l2_buf_type, V4L2Queue*> mQueues; + + // Used if EnablePolling() is called to signal the user that an event happened or a buffer is + // ready to be dequeued. + std::unique_ptr<android::V4L2DevicePoller> mDevicePoller; + + SEQUENCE_CHECKER(mClientSequenceChecker); +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_H diff --git a/common/include/v4l2_codec2/common/V4L2DevicePoller.h b/common/include/v4l2_codec2/common/V4L2DevicePoller.h new file mode 100644 index 0000000..ad256be --- /dev/null +++ b/common/include/v4l2_codec2/common/V4L2DevicePoller.h @@ -0,0 +1,88 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: f65c38dcdac2 + +#ifndef ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_POLLER_H +#define ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_POLLER_H + +#include <atomic> + +#include <base/callback_forward.h> +#include <base/sequence_checker.h> +#include <base/sequenced_task_runner.h> +#include <base/synchronization/waitable_event.h> +#include <base/threading/thread.h> + +namespace android { + +class V4L2Device; + +// Allows a client to poll() on a given V4L2Device and be signaled when a buffer is ready to be +// dequeued or a V4L2 event has been received. Polling is done on a dedicated thread, and +// notifications are delivered in the form of a callback to the listener's sequence. +// +// All the methods of this class (with the exception of the constructor) must be called from the +// same sequence. +// +// Note that the service callback may also be called when no particular event occurred due to the +// way poll() works. It is the responsibility of the caller to call SchedulePoll() again if there +// may still be pending events. +class V4L2DevicePoller { +public: + // Callback to be called when buffer ready/V4L2 event has potentially been polled. |event| is + // set if a V4L2 event has been detected. + using EventCallback = base::RepeatingCallback<void(bool event)>; + + // Create a poller for |device|, using a thread named |threadName|. Notification won't start + // until |startPolling()| is called. + V4L2DevicePoller(V4L2Device* const device, const std::string& threadName); + ~V4L2DevicePoller(); + + // Starts polling. |mEventCallback| will be posted on the caller's sequence every time an event + // occurs. The client is then responsible for consuming all pending events in that callback. If + // new events may still happen after the callback has run, the client must call |schedulePoll()| + // again in order to be notified for them. + // + // If an error occurs during polling, |mErrorCallback| will be posted on the caller's sequence. + bool startPolling(EventCallback eventCallback, base::RepeatingClosure errorCallback); + // Stop polling and stop the thread. The poller won't post any new event to the caller's + // sequence after this method has returned. + bool stopPolling(); + // Returns true if currently polling, false otherwise. + bool isPolling() const; + // Attempts polling the V4L2 device. This method should be called whenever doing something that + // may trigger an event of interest (buffer dequeue or V4L2 event), for instance queueing a + // buffer. In the absence of a pending event, poll() will return immediately and the service + // callback will be posted to the caller's sequence. The client is then responsible for calling + // this method again when it is interested in receiving events. + void schedulePoll(); + +private: + // Perform a poll() on |mDevice| and post either |mEventCallback| or |mErrorCallback| on the + // client's sequence when poll() returns. + void devicePollTask(); + + // V4L2 device we are polling. + V4L2Device* const mDevice; + // Thread on which polling is done. + base::Thread mPollThread; + // Callback to post to the client's sequence when an event occurs. + EventCallback mEventCallback; + // Closure to post to the client's sequence when an error occurs. + base::RepeatingClosure mErrorCallback; + // Client sequence's task runner, where closures are posted. + scoped_refptr<base::SequencedTaskRunner> mClientTaskTunner; + + // Since poll() returns immediately if no buffers have been queued, we cannot rely on it to + // pause the polling thread until an event occurs. Instead, + // the polling thread will wait on this WaitableEvent (signaled by |schedulePoll| before calling + // poll(), so we only call it when we are actually waiting for an event. + base::WaitableEvent mTriggerPoll; + // Set to true when we wish to stop polling, instructing the poller thread to break its loop. + std::atomic_bool mStopPolling; +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMMON_V4L2_DEVICE_POLLER_H diff --git a/common/include/v4l2_codec2/common/VideoPixelFormat.h b/common/include/v4l2_codec2/common/VideoPixelFormat.h new file mode 100644 index 0000000..2cfe910 --- /dev/null +++ b/common/include/v4l2_codec2/common/VideoPixelFormat.h @@ -0,0 +1,90 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// Note: ported from Chromium commit head: 3b7ce92816e2 +// Note: only necessary functions are ported from video_types.h + +#ifndef ANDROID_V4L2_CODEC2_COMMON_VIDEO_PIXEL_FORMAT_H +#define ANDROID_V4L2_CODEC2_COMMON_VIDEO_PIXEL_FORMAT_H + +#include <string> + +#include "ui/Size.h" + +namespace android { + +// Pixel formats roughly based on FOURCC labels, see: +// http://www.fourcc.org/rgb.php and http://www.fourcc.org/yuv.php +enum class VideoPixelFormat { + I420, // 12bpp YUV planar 1x1 Y, 2x2 UV samples, a.k.a. YU12. + YV12, // 12bpp YVU planar 1x1 Y, 2x2 VU samples. + I422, // 16bpp YUV planar 1x1 Y, 2x1 UV samples. + I420A, // 20bpp YUVA planar 1x1 Y, 2x2 UV, 1x1 A samples. + I444, // 24bpp YUV planar, no subsampling. + NV12, // 12bpp with Y plane followed by a 2x2 interleaved UV plane. + NV21, // 12bpp with Y plane followed by a 2x2 interleaved VU plane. + YUY2, // 16bpp interleaved 1x1 Y, 2x1 U, 1x1 Y, 2x1 V samples. + ARGB, // 32bpp BGRA (byte-order), 1 plane. + XRGB, // 24bpp BGRX (byte-order), 1 plane. + RGB24, // 24bpp BGR (byte-order), 1 plane. + MJPEG, // MJPEG compressed. + Y16, // single 16bpp plane. + ABGR, // 32bpp RGBA (byte-order), 1 plane. + XBGR, // 24bpp RGBX (byte-order), 1 plane. + P016LE, // 24bpp NV12, 16 bits per channel + XR30, // 32bpp BGRX, 10 bits per channel, 2 bits ignored, 1 plane + XB30, // 32bpp RGBX, 10 bits per channel, 2 bits ignored, 1 plane + BGRA, // 32bpp ARGB (byte-order), 1 plane. + // The P* in the formats below designates the number of bits per pixel component. I.e. P9 is + // 9-bits per pixel component, P10 is 10-bits per pixel component, etc. + YUV420P9, + YUV420P10, + YUV422P9, + YUV422P10, + YUV444P9, + YUV444P10, + YUV420P12, + YUV422P12, + YUV444P12, + UNKNOWN, // Unknown or unspecified format value. +}; + +// Returns the name of a Format as a string. +std::string videoPixelFormatToString(VideoPixelFormat format); + +// Returns human readable fourcc string. If any of the four characters is non-printable, it outputs +// "0x<32-bit integer in hex>", e.g. FourccToString(0x66616b00) returns "0x66616b00". +std::string fourccToString(uint32_t fourcc); + +// Returns the number of significant bits per channel. +size_t bitDepth(VideoPixelFormat format); + +// Returns the number of planes for the |format|. +size_t numPlanes(VideoPixelFormat format); + +// Returns required allocation size for a (tightly packed) frame of the given coded size and format. +size_t allocationSize(VideoPixelFormat format, const android::ui::Size& coded_size); + +// Returns the plane Size (in bytes) for a plane of the given coded size and format. +android::ui::Size planeSize(VideoPixelFormat format, size_t plane, + const android::ui::Size& coded_size); + +// Returns horizontal bits per pixel for given |plane| and |format|. +int planeHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane); + +// Returns bits per pixel for given |plane| and |format|. +int planeBitsPerPixel(VideoPixelFormat format, size_t plane); + +// Returns the number of bytes per element for given |plane| and |format|. +int bytesPerElement(VideoPixelFormat format, size_t plane); + +// Returns true if |plane| is a valid plane index for the given |format|. +bool isValidPlane(VideoPixelFormat format, size_t plane); + +// Returns the pixel size of each subsample for a given |plane| and |format|. +// E.g. 2x2 for the U-plane in I420. +android::ui::Size SampleSize(VideoPixelFormat format, size_t plane); + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMMON_VIDEO_PIXEL_FORMAT_H diff --git a/common/include/v4l2_codec2/common/VideoTypes.h b/common/include/v4l2_codec2/common/VideoTypes.h index a5130d2..076f096 100644 --- a/common/include/v4l2_codec2/common/VideoTypes.h +++ b/common/include/v4l2_codec2/common/VideoTypes.h @@ -8,6 +8,7 @@ #include <optional> #include <string> +#include <C2Config.h> #include <android/hardware/graphics/common/1.0/types.h> namespace android { @@ -18,7 +19,12 @@ enum class VideoCodec { VP8, VP9, }; + +constexpr std::initializer_list<VideoCodec> kAllCodecs = {VideoCodec::H264, VideoCodec::VP8, + VideoCodec::VP9}; + const char* VideoCodecToString(VideoCodec codec); +const char* profileToString(C2Config::profile_t profile); // Enumeration of supported pixel format. The value should be the same as // ::android::hardware::graphics::common::V1_0::PixelFormat. diff --git a/components/Android.bp b/components/Android.bp index 8273412..16c7d20 100644 --- a/components/Android.bp +++ b/components/Android.bp @@ -1,26 +1,41 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_v4l2_codec2_license" + // to get the below license kinds: + // SPDX-license-identifier-BSD + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + cc_library { name: "libv4l2_codec2_components", vendor: true, defaults: [ - "libcodec2-impl-defaults", + "libcodec2-hidl-defaults", ], srcs: [ "VideoFrame.cpp", "VideoFramePool.cpp", - "V4L2Decoder.cpp", "V4L2ComponentFactory.cpp", + "V4L2ComponentStore.cpp", + "V4L2Decoder.cpp", "V4L2DecodeComponent.cpp", "V4L2DecodeInterface.cpp", + "V4L2Encoder.cpp", "V4L2EncodeComponent.cpp", "V4L2EncodeInterface.cpp", "VideoDecoder.cpp", + "VideoEncoder.cpp", ], export_include_dirs: [ "include", ], + header_libs: [ + "libcodec2_internal", + ], shared_libs: [ "android.hardware.graphics.common@1.0", "libc2plugin_store", @@ -31,11 +46,9 @@ cc_library { "libsfplugin_ccodec_utils", "libstagefright_bufferqueue_helper", "libstagefright_foundation", - "libv4l2_codec2_store", "libui", ], static_libs: [ - "libv4l2_codec2_accel", "libv4l2_codec2_common", ], diff --git a/components/V4L2ComponentFactory.cpp b/components/V4L2ComponentFactory.cpp index a535c34..a3f8837 100644 --- a/components/V4L2ComponentFactory.cpp +++ b/components/V4L2ComponentFactory.cpp @@ -5,51 +5,57 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "V4L2ComponentFactory" -#include <string> +#include <v4l2_codec2/components/V4L2ComponentFactory.h> -#include <C2ComponentFactory.h> -#include <SimpleC2Interface.h> +#include <codec2/hidl/1.0/InputBufferManager.h> #include <log/log.h> -#include <util/C2InterfaceHelper.h> #include <v4l2_codec2/common/V4L2ComponentCommon.h> #include <v4l2_codec2/components/V4L2DecodeComponent.h> #include <v4l2_codec2/components/V4L2DecodeInterface.h> #include <v4l2_codec2/components/V4L2EncodeComponent.h> #include <v4l2_codec2/components/V4L2EncodeInterface.h> -#include <v4l2_codec2/store/V4L2ComponentStore.h> namespace android { -class V4L2ComponentFactory : public C2ComponentFactory { -public: - V4L2ComponentFactory(const char* componentName, bool isEncoder); - ~V4L2ComponentFactory() override; - - // Implementation of C2ComponentFactory. - c2_status_t createComponent(c2_node_id_t id, std::shared_ptr<C2Component>* const component, - ComponentDeleter deleter) override; - c2_status_t createInterface(c2_node_id_t id, - std::shared_ptr<C2ComponentInterface>* const interface, - InterfaceDeleter deleter) override; - -private: - const std::string mComponentName; - const bool mIsEncoder; - std::shared_ptr<C2ReflectorHelper> mReflector; -}; - -V4L2ComponentFactory::V4L2ComponentFactory(const char* componentName, bool isEncoder) - : mComponentName(componentName), mIsEncoder(isEncoder) { - auto componentStore = V4L2ComponentStore::Create(); - if (componentStore == nullptr) { - ALOGE("Could not create V4L2ComponentStore."); - return; +// static +std::unique_ptr<V4L2ComponentFactory> V4L2ComponentFactory::create( + const std::string& componentName, std::shared_ptr<C2ReflectorHelper> reflector) { + ALOGV("%s(%s)", __func__, componentName.c_str()); + + if (!android::V4L2ComponentName::isValid(componentName.c_str())) { + ALOGE("Invalid component name: %s", componentName.c_str()); + return nullptr; + } + if (reflector == nullptr) { + ALOGE("reflector is null"); + return nullptr; } - mReflector = std::static_pointer_cast<C2ReflectorHelper>(componentStore->getParamReflector()); + + bool isEncoder = android::V4L2ComponentName::isEncoder(componentName.c_str()); + return std::make_unique<V4L2ComponentFactory>(componentName, isEncoder, std::move(reflector)); } -V4L2ComponentFactory::~V4L2ComponentFactory() = default; +V4L2ComponentFactory::V4L2ComponentFactory(const std::string& componentName, bool isEncoder, + std::shared_ptr<C2ReflectorHelper> reflector) + : mComponentName(componentName), mIsEncoder(isEncoder), mReflector(std::move(reflector)) { + using namespace ::android::hardware::media::c2::V1_0; + // To minimize IPC, we generally want the codec2 framework to release and + // recycle input buffers when the corresponding work item is done. However, + // sometimes it is necessary to provide more input to unblock a decoder. + // + // Optimally we would configure this on a per-context basis. However, the + // InputBufferManager is a process-wide singleton, so we need to configure it + // pessimistically. Basing the interval on frame timing can be suboptimal if + // the decoded output isn't being displayed, but that's not a primary use case + // and few videos will actually rely on this behavior. + constexpr nsecs_t kMinFrameIntervalNs = 1000000000ull / 60; + uint32_t delayCount = 0; + for (auto c : kAllCodecs) { + delayCount = std::max(delayCount, V4L2DecodeInterface::getOutputDelay(c)); + } + utils::InputBufferManager::setNotificationInterval(delayCount * kMinFrameIntervalNs / 2); +} c2_status_t V4L2ComponentFactory::createComponent(c2_node_id_t id, std::shared_ptr<C2Component>* const component, @@ -67,7 +73,7 @@ c2_status_t V4L2ComponentFactory::createComponent(c2_node_id_t id, } else { *component = V4L2DecodeComponent::create(mComponentName, id, mReflector, deleter); } - return *component ? C2_OK : C2_BAD_VALUE; + return *component ? C2_OK : C2_NO_MEMORY; } c2_status_t V4L2ComponentFactory::createInterface( @@ -98,20 +104,3 @@ c2_status_t V4L2ComponentFactory::createInterface( } } // namespace android - -extern "C" ::C2ComponentFactory* CreateCodec2Factory(const char* componentName) { - ALOGV("%s(%s)", __func__, componentName); - - if (!android::V4L2ComponentName::isValid(componentName)) { - ALOGE("Invalid component name: %s", componentName); - return nullptr; - } - - bool isEncoder = android::V4L2ComponentName::isEncoder(componentName); - return new android::V4L2ComponentFactory(componentName, isEncoder); -} - -extern "C" void DestroyCodec2Factory(::C2ComponentFactory* factory) { - ALOGV("%s()", __func__); - delete factory; -} diff --git a/store/V4L2ComponentStore.cpp b/components/V4L2ComponentStore.cpp index 8942866..4004ce5 100644 --- a/store/V4L2ComponentStore.cpp +++ b/components/V4L2ComponentStore.cpp @@ -5,9 +5,8 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "V4L2ComponentStore" -#include <v4l2_codec2/store/V4L2ComponentStore.h> +#include <v4l2_codec2/components/V4L2ComponentStore.h> -#include <dlfcn.h> #include <stdint.h> #include <memory> @@ -16,16 +15,31 @@ #include <C2.h> #include <C2Config.h> #include <log/log.h> +#include <media/stagefright/foundation/MediaDefs.h> #include <v4l2_codec2/common/V4L2ComponentCommon.h> +#include <v4l2_codec2/components/V4L2ComponentFactory.h> namespace android { namespace { -const char* kLibPath = "libv4l2_codec2_components.so"; -const char* kCreateFactoryFuncName = "CreateCodec2Factory"; -const char* kDestroyFactoryFuncName = "DestroyCodec2Factory"; - const uint32_t kComponentRank = 0x80; + +std::string getMediaTypeFromComponentName(const std::string& name) { + if (name == V4L2ComponentName::kH264Decoder || name == V4L2ComponentName::kH264SecureDecoder || + name == V4L2ComponentName::kH264Encoder) { + return MEDIA_MIMETYPE_VIDEO_AVC; + } + if (name == V4L2ComponentName::kVP8Decoder || name == V4L2ComponentName::kVP8SecureDecoder || + name == V4L2ComponentName::kVP8Encoder) { + return MEDIA_MIMETYPE_VIDEO_VP8; + } + if (name == V4L2ComponentName::kVP9Decoder || name == V4L2ComponentName::kVP9SecureDecoder || + name == V4L2ComponentName::kVP9Encoder) { + return MEDIA_MIMETYPE_VIDEO_VP9; + } + return ""; +} + } // namespace // static @@ -39,32 +53,12 @@ std::shared_ptr<C2ComponentStore> V4L2ComponentStore::Create() { std::shared_ptr<C2ComponentStore> store = platformStore.lock(); if (store != nullptr) return store; - void* libHandle = dlopen(kLibPath, RTLD_NOW | RTLD_NODELETE); - if (!libHandle) { - ALOGE("Failed to load library: %s", kLibPath); - return nullptr; - } - - auto createFactoryFunc = (CreateV4L2FactoryFunc)dlsym(libHandle, kCreateFactoryFuncName); - auto destroyFactoryFunc = (DestroyV4L2FactoryFunc)dlsym(libHandle, kDestroyFactoryFuncName); - if (!createFactoryFunc || !destroyFactoryFunc) { - ALOGE("Failed to load functions: %s, %s", kCreateFactoryFuncName, kDestroyFactoryFuncName); - dlclose(libHandle); - return nullptr; - } - - store = std::shared_ptr<C2ComponentStore>( - new V4L2ComponentStore(libHandle, createFactoryFunc, destroyFactoryFunc)); + store = std::shared_ptr<C2ComponentStore>(new V4L2ComponentStore()); platformStore = store; return store; } -V4L2ComponentStore::V4L2ComponentStore(void* libHandle, CreateV4L2FactoryFunc createFactoryFunc, - DestroyV4L2FactoryFunc destroyFactoryFunc) - : mLibHandle(libHandle), - mCreateFactoryFunc(createFactoryFunc), - mDestroyFactoryFunc(destroyFactoryFunc), - mReflector(std::make_shared<C2ReflectorHelper>()) { +V4L2ComponentStore::V4L2ComponentStore() : mReflector(std::make_shared<C2ReflectorHelper>()) { ALOGV("%s()", __func__); } @@ -72,10 +66,7 @@ V4L2ComponentStore::~V4L2ComponentStore() { ALOGV("%s()", __func__); std::lock_guard<std::mutex> lock(mCachedFactoriesLock); - for (const auto& kv : mCachedFactories) mDestroyFactoryFunc(kv.second); mCachedFactories.clear(); - - dlclose(mLibHandle); } C2String V4L2ComponentStore::getName() const { @@ -86,6 +77,11 @@ c2_status_t V4L2ComponentStore::createComponent(C2String name, std::shared_ptr<C2Component>* const component) { ALOGV("%s(%s)", __func__, name.c_str()); + if (!V4L2ComponentName::isValid(name.c_str())) { + ALOGI("%s(): Invalid component name: %s", __func__, name.c_str()); + return C2_NOT_FOUND; + } + auto factory = GetFactory(name); if (factory == nullptr) return C2_CORRUPTED; @@ -97,6 +93,11 @@ c2_status_t V4L2ComponentStore::createInterface( C2String name, std::shared_ptr<C2ComponentInterface>* const interface) { ALOGV("%s(%s)", __func__, name.c_str()); + if (!V4L2ComponentName::isValid(name.c_str())) { + ALOGI("%s(): Invalid component name: %s", __func__, name.c_str()); + return C2_NOT_FOUND; + } + auto factory = GetFactory(name); if (factory == nullptr) return C2_CORRUPTED; @@ -111,8 +112,10 @@ std::vector<std::shared_ptr<const C2Component::Traits>> V4L2ComponentStore::list ret.push_back(GetTraits(V4L2ComponentName::kH264Encoder)); ret.push_back(GetTraits(V4L2ComponentName::kH264Decoder)); ret.push_back(GetTraits(V4L2ComponentName::kH264SecureDecoder)); + ret.push_back(GetTraits(V4L2ComponentName::kVP8Encoder)); ret.push_back(GetTraits(V4L2ComponentName::kVP8Decoder)); ret.push_back(GetTraits(V4L2ComponentName::kVP8SecureDecoder)); + ret.push_back(GetTraits(V4L2ComponentName::kVP9Encoder)); ret.push_back(GetTraits(V4L2ComponentName::kVP9Decoder)); ret.push_back(GetTraits(V4L2ComponentName::kVP9SecureDecoder)); return ret; @@ -155,24 +158,22 @@ c2_status_t V4L2ComponentStore::querySupportedValues_sm( ::C2ComponentFactory* V4L2ComponentStore::GetFactory(const C2String& name) { ALOGV("%s(%s)", __func__, name.c_str()); - - if (!V4L2ComponentName::isValid(name.c_str())) { - ALOGE("Invalid component name: %s", name.c_str()); - return nullptr; - } + ALOG_ASSERT(V4L2ComponentName::isValid(name.c_str())); std::lock_guard<std::mutex> lock(mCachedFactoriesLock); const auto it = mCachedFactories.find(name); - if (it != mCachedFactories.end()) return it->second; + if (it != mCachedFactories.end()) return it->second.get(); - ::C2ComponentFactory* factory = mCreateFactoryFunc(name.c_str()); + std::unique_ptr<::C2ComponentFactory> factory = V4L2ComponentFactory::create( + name, std::static_pointer_cast<C2ReflectorHelper>(getParamReflector())); if (factory == nullptr) { ALOGE("Failed to create factory for %s", name.c_str()); return nullptr; } - mCachedFactories.emplace(name, factory); - return factory; + auto ret = factory.get(); + mCachedFactories.emplace(name, std::move(factory)); + return ret; } std::shared_ptr<const C2Component::Traits> V4L2ComponentStore::GetTraits(const C2String& name) { @@ -187,39 +188,13 @@ std::shared_ptr<const C2Component::Traits> V4L2ComponentStore::GetTraits(const C auto it = mCachedTraits.find(name); if (it != mCachedTraits.end()) return it->second; - std::shared_ptr<C2ComponentInterface> intf; - auto res = createInterface(name, &intf); - if (res != C2_OK) { - ALOGE("failed to create interface for %s: %d", name.c_str(), res); - return nullptr; - } - - bool isEncoder = V4L2ComponentName::isEncoder(name.c_str()); - uint32_t mediaTypeIndex = isEncoder ? C2PortMediaTypeSetting::output::PARAM_TYPE - : C2PortMediaTypeSetting::input::PARAM_TYPE; - std::vector<std::unique_ptr<C2Param>> params; - res = intf->query_vb({}, {mediaTypeIndex}, C2_MAY_BLOCK, ¶ms); - if (res != C2_OK) { - ALOGE("failed to query interface: %d", res); - return nullptr; - } - if (params.size() != 1u) { - ALOGE("failed to query interface: unexpected vector size: %zu", params.size()); - return nullptr; - } - - C2PortMediaTypeSetting* mediaTypeConfig = (C2PortMediaTypeSetting*)(params[0].get()); - if (mediaTypeConfig == nullptr) { - ALOGE("failed to query media type"); - return nullptr; - } - auto traits = std::make_shared<C2Component::Traits>(); - traits->name = intf->getName(); + traits->name = name; traits->domain = C2Component::DOMAIN_VIDEO; - traits->kind = isEncoder ? C2Component::KIND_ENCODER : C2Component::KIND_DECODER; - traits->mediaType = mediaTypeConfig->m.value; traits->rank = kComponentRank; + traits->mediaType = getMediaTypeFromComponentName(name); + traits->kind = V4L2ComponentName::isEncoder(name.c_str()) ? C2Component::KIND_ENCODER + : C2Component::KIND_DECODER; mCachedTraits.emplace(name, traits); return traits; diff --git a/components/V4L2DecodeComponent.cpp b/components/V4L2DecodeComponent.cpp index 1ea9a7b..400c765 100644 --- a/components/V4L2DecodeComponent.cpp +++ b/components/V4L2DecodeComponent.cpp @@ -20,10 +20,12 @@ #include <base/bind.h> #include <base/callback_helpers.h> #include <base/time/time.h> +#include <cutils/properties.h> #include <log/log.h> #include <media/stagefright/foundation/ColorUtils.h> -#include <h264_parser.h> +#include <v4l2_codec2/common/Common.h> +#include <v4l2_codec2/common/NalParser.h> #include <v4l2_codec2/common/VideoTypes.h> #include <v4l2_codec2/components/BitstreamBuffer.h> #include <v4l2_codec2/components/V4L2Decoder.h> @@ -32,8 +34,6 @@ namespace android { namespace { -// TODO(b/151128291): figure out why we cannot open V4L2Device in 0.5 second? -const ::base::TimeDelta kBlockingMethodTimeout = ::base::TimeDelta::FromMilliseconds(5000); // Mask against 30 bits to avoid (undefined) wraparound on signed integer. int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) { @@ -43,44 +43,23 @@ int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) { bool parseCodedColorAspects(const C2ConstLinearBlock& input, C2StreamColorAspectsInfo::input* codedAspects) { C2ReadView view = input.map().get(); - const uint8_t* data = view.data(); - const uint32_t size = view.capacity(); - - std::unique_ptr<media::H264Parser> h264Parser = std::make_unique<media::H264Parser>(); - h264Parser->SetStream(data, static_cast<off_t>(size)); - media::H264NALU nalu; - media::H264Parser::Result parRes = h264Parser->AdvanceToNextNALU(&nalu); - if (parRes != media::H264Parser::kEOStream && parRes != media::H264Parser::kOk) { - ALOGE("H264 AdvanceToNextNALU error: %d", static_cast<int>(parRes)); - return false; - } - if (nalu.nal_unit_type != media::H264NALU::kSPS) { - ALOGV("NALU is not SPS"); - return false; - } + NalParser parser(view.data(), view.capacity()); - int spsId; - parRes = h264Parser->ParseSPS(&spsId); - if (parRes != media::H264Parser::kEOStream && parRes != media::H264Parser::kOk) { - ALOGE("H264 ParseSPS error: %d", static_cast<int>(parRes)); + if (!parser.locateSPS()) { + ALOGV("Couldn't find SPS"); return false; } - // Parse ISO color aspects from H264 SPS bitstream. - const media::H264SPS* sps = h264Parser->GetSPS(spsId); - if (!sps->colour_description_present_flag) { - ALOGV("No Color Description in SPS"); + NalParser::ColorAspects aspects; + if (!parser.findCodedColorAspects(&aspects)) { + ALOGV("Couldn't find color description in SPS"); return false; } - int32_t primaries = sps->colour_primaries; - int32_t transfer = sps->transfer_characteristics; - int32_t coeffs = sps->matrix_coefficients; - bool fullRange = sps->video_full_range_flag; // Convert ISO color aspects to ColorUtils::ColorAspects. ColorAspects colorAspects; - ColorUtils::convertIsoColorAspectsToCodecAspects(primaries, transfer, coeffs, fullRange, - colorAspects); + ColorUtils::convertIsoColorAspectsToCodecAspects( + aspects.primaries, aspects.transfer, aspects.coeffs, aspects.fullRange, colorAspects); ALOGV("Parsed ColorAspects from bitstream: (R:%d, P:%d, M:%d, T:%d)", colorAspects.mRange, colorAspects.mPrimaries, colorAspects.mMatrixCoeffs, colorAspects.mTransfer); @@ -138,9 +117,23 @@ bool isNoShowFrameWork(const C2Work& work, const C2WorkOrdinalStruct& currOrdina } // namespace // static +std::atomic<int32_t> V4L2DecodeComponent::sConcurrentInstances = 0; + +// static std::shared_ptr<C2Component> V4L2DecodeComponent::create( const std::string& name, c2_node_id_t id, const std::shared_ptr<C2ReflectorHelper>& helper, C2ComponentFactory::ComponentDeleter deleter) { + static const int32_t kMaxConcurrentInstances = + property_get_int32("debug.v4l2_codec2.decode.concurrent-instances", -1); + static std::mutex mutex; + + std::lock_guard<std::mutex> lock(mutex); + + if (kMaxConcurrentInstances >= 0 && sConcurrentInstances.load() >= kMaxConcurrentInstances) { + ALOGW("Reject to Initialize() due to too many instances: %d", sConcurrentInstances.load()); + return nullptr; + } + auto intfImpl = std::make_shared<V4L2DecodeInterface>(name, helper); if (intfImpl->status() != C2_OK) { ALOGE("Failed to initialize V4L2DecodeInterface."); @@ -158,26 +151,17 @@ V4L2DecodeComponent::V4L2DecodeComponent(const std::string& name, c2_node_id_t i mIntf(std::make_shared<SimpleInterface<V4L2DecodeInterface>>(name.c_str(), id, mIntfImpl)) { ALOGV("%s(%s)", __func__, name.c_str()); + sConcurrentInstances.fetch_add(1, std::memory_order_relaxed); mIsSecure = name.find(".secure") != std::string::npos; } V4L2DecodeComponent::~V4L2DecodeComponent() { ALOGV("%s()", __func__); - if (mDecoderThread.IsRunning()) { - mDecoderTaskRunner->PostTask( - FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::destroyTask, mWeakThis)); - mDecoderThread.Stop(); - } - ALOGV("%s() done", __func__); -} - -void V4L2DecodeComponent::destroyTask() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + release(); - mWeakThisFactory.InvalidateWeakPtrs(); - mDecoder = nullptr; + sConcurrentInstances.fetch_sub(1, std::memory_order_relaxed); + ALOGV("%s() done", __func__); } c2_status_t V4L2DecodeComponent::start() { @@ -196,27 +180,25 @@ c2_status_t V4L2DecodeComponent::start() { } mDecoderTaskRunner = mDecoderThread.task_runner(); mWeakThis = mWeakThisFactory.GetWeakPtr(); + mStdWeakThis = weak_from_this(); c2_status_t status = C2_CORRUPTED; - mStartStopDone.Reset(); - mDecoderTaskRunner->PostTask(FROM_HERE, - ::base::BindOnce(&V4L2DecodeComponent::startTask, mWeakThis, - ::base::Unretained(&status))); - if (!mStartStopDone.TimedWait(kBlockingMethodTimeout)) { - ALOGE("startTask() timeout..."); - return C2_TIMED_OUT; - } + ::base::WaitableEvent done; + mDecoderTaskRunner->PostTask( + FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::startTask, mWeakThis, + ::base::Unretained(&status), ::base::Unretained(&done))); + done.Wait(); if (status == C2_OK) mComponentState.store(ComponentState::RUNNING); return status; } -void V4L2DecodeComponent::startTask(c2_status_t* status) { +void V4L2DecodeComponent::startTask(c2_status_t* status, ::base::WaitableEvent* done) { ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); ::base::ScopedClosureRunner done_caller( - ::base::BindOnce(&::base::WaitableEvent::Signal, ::base::Unretained(&mStartStopDone))); + ::base::BindOnce(&::base::WaitableEvent::Signal, ::base::Unretained(done))); *status = C2_CORRUPTED; const auto codec = mIntfImpl->getVideoCodec(); @@ -225,12 +207,16 @@ void V4L2DecodeComponent::startTask(c2_status_t* status) { return; } const size_t inputBufferSize = mIntfImpl->getInputBufferSize(); - mDecoder = V4L2Decoder::Create( - *codec, inputBufferSize, - ::base::BindRepeating(&V4L2DecodeComponent::getVideoFramePool, mWeakThis), - ::base::BindRepeating(&V4L2DecodeComponent::onOutputFrameReady, mWeakThis), - ::base::BindRepeating(&V4L2DecodeComponent::reportError, mWeakThis, C2_CORRUPTED), - mDecoderTaskRunner); + // ::base::Unretained(this) is safe here because |mDecoder| is always destroyed before + // |mDecoderThread| is stopped, so |*this| is always valid during |mDecoder|'s lifetime. + mDecoder = V4L2Decoder::Create(*codec, inputBufferSize, + ::base::BindRepeating(&V4L2DecodeComponent::getVideoFramePool, + ::base::Unretained(this)), + ::base::BindRepeating(&V4L2DecodeComponent::onOutputFrameReady, + ::base::Unretained(this)), + ::base::BindRepeating(&V4L2DecodeComponent::reportError, + ::base::Unretained(this), C2_CORRUPTED), + mDecoderTaskRunner); if (!mDecoder) { ALOGE("Failed to create V4L2Decoder for %s", VideoCodecToString(*codec)); return; @@ -245,36 +231,40 @@ void V4L2DecodeComponent::startTask(c2_status_t* status) { *status = C2_OK; } -void V4L2DecodeComponent::getVideoFramePool(std::unique_ptr<VideoFramePool>* pool, - const media::Size& size, HalPixelFormat pixelFormat, - size_t numBuffers) { +std::unique_ptr<VideoFramePool> V4L2DecodeComponent::getVideoFramePool(const ui::Size& size, + HalPixelFormat pixelFormat, + size_t numBuffers) { ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + auto sharedThis = mStdWeakThis.lock(); + if (sharedThis == nullptr) { + ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); + return nullptr; + } + // (b/157113946): Prevent malicious dynamic resolution change exhausts system memory. constexpr int kMaximumSupportedArea = 4096 * 4096; - if (size.width() * size.height() > kMaximumSupportedArea) { - ALOGE("The output size (%dx%d) is larger than supported size (4096x4096)", size.width(), - size.height()); + if (getArea(size).value_or(INT_MAX) > kMaximumSupportedArea) { + ALOGE("The output size (%dx%d) is larger than supported size (4096x4096)", size.width, + size.height); reportError(C2_BAD_VALUE); - *pool = nullptr; - return; + return nullptr; } // Get block pool ID configured from the client. auto poolId = mIntfImpl->getBlockPoolId(); ALOGI("Using C2BlockPool ID = %" PRIu64 " for allocating output buffers", poolId); std::shared_ptr<C2BlockPool> blockPool; - auto status = GetCodec2BlockPool(poolId, shared_from_this(), &blockPool); + auto status = GetCodec2BlockPool(poolId, std::move(sharedThis), &blockPool); if (status != C2_OK) { ALOGE("Graphic block allocator is invalid: %d", status); reportError(status); - *pool = nullptr; - return; + return nullptr; } - *pool = VideoFramePool::Create(std::move(blockPool), numBuffers, size, pixelFormat, mIsSecure, - mDecoderTaskRunner); + return VideoFramePool::Create(std::move(blockPool), numBuffers, size, pixelFormat, mIsSecure, + mDecoderTaskRunner); } c2_status_t V4L2DecodeComponent::stop() { @@ -287,19 +277,13 @@ c2_status_t V4L2DecodeComponent::stop() { return C2_BAD_STATE; } - // Return immediately if the component is already stopped. - if (!mDecoderThread.IsRunning()) return C2_OK; - - mStartStopDone.Reset(); - mDecoderTaskRunner->PostTask(FROM_HERE, - ::base::BindOnce(&V4L2DecodeComponent::stopTask, mWeakThis)); - if (!mStartStopDone.TimedWait(kBlockingMethodTimeout)) { - ALOGE("stopTask() timeout..."); - return C2_TIMED_OUT; + if (mDecoderThread.IsRunning()) { + mDecoderTaskRunner->PostTask(FROM_HERE, + ::base::BindOnce(&V4L2DecodeComponent::stopTask, mWeakThis)); + mDecoderThread.Stop(); + mDecoderTaskRunner = nullptr; } - mDecoderThread.Stop(); - mDecoderTaskRunner = nullptr; mComponentState.store(ComponentState::STOPPED); return C2_OK; } @@ -310,10 +294,38 @@ void V4L2DecodeComponent::stopTask() { reportAbandonedWorks(); mIsDraining = false; - mDecoder = nullptr; - mWeakThisFactory.InvalidateWeakPtrs(); - mStartStopDone.Signal(); + releaseTask(); +} + +c2_status_t V4L2DecodeComponent::reset() { + ALOGV("%s()", __func__); + + return stop(); +} + +c2_status_t V4L2DecodeComponent::release() { + ALOGV("%s()", __func__); + std::lock_guard<std::mutex> lock(mStartStopLock); + + if (mDecoderThread.IsRunning()) { + mDecoderTaskRunner->PostTask( + FROM_HERE, ::base::BindOnce(&V4L2DecodeComponent::releaseTask, mWeakThis)); + mDecoderThread.Stop(); + mDecoderTaskRunner = nullptr; + } + + mComponentState.store(ComponentState::RELEASED); + return C2_OK; +} + +void V4L2DecodeComponent::releaseTask() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + + mWeakThisFactory.InvalidateWeakPtrs(); + mStdWeakThis.reset(); + mDecoder = nullptr; } c2_status_t V4L2DecodeComponent::setListener_vb( @@ -418,16 +430,21 @@ void V4L2DecodeComponent::pumpPendingWorks() { } while (!mPendingWorks.empty() && !mIsDraining) { - std::unique_ptr<C2Work> work(std::move(mPendingWorks.front())); + std::unique_ptr<C2Work> pendingWork(std::move(mPendingWorks.front())); mPendingWorks.pop(); - const int32_t bitstreamId = frameIndexToBitstreamId(work->input.ordinal.frameIndex); - const bool isCSDWork = work->input.flags & C2FrameData::FLAG_CODEC_CONFIG; - const bool isEmptyWork = work->input.buffers.front() == nullptr; + const int32_t bitstreamId = frameIndexToBitstreamId(pendingWork->input.ordinal.frameIndex); + const bool isCSDWork = pendingWork->input.flags & C2FrameData::FLAG_CODEC_CONFIG; + const bool isEmptyWork = pendingWork->input.buffers.front() == nullptr; + const bool isEOSWork = pendingWork->input.flags & C2FrameData::FLAG_END_OF_STREAM; + const C2Work* work = pendingWork.get(); ALOGV("Process C2Work bitstreamId=%d isCSDWork=%d, isEmptyWork=%d", bitstreamId, isCSDWork, isEmptyWork); - if (work->input.buffers.front() != nullptr) { + auto res = mWorksAtDecoder.insert(std::make_pair(bitstreamId, std::move(pendingWork))); + ALOGW_IF(!res.second, "We already inserted bitstreamId %d to decoder?", bitstreamId); + + if (!isEmptyWork) { // If input.buffers is not empty, the buffer should have meaningful content inside. C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front(); @@ -464,14 +481,11 @@ void V4L2DecodeComponent::pumpPendingWorks() { mWeakThis, bitstreamId)); } - if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) { + if (isEOSWork) { mDecoder->drain(::base::BindOnce(&V4L2DecodeComponent::onDrainDone, mWeakThis)); mIsDraining = true; } - auto res = mWorksAtDecoder.insert(std::make_pair(bitstreamId, std::move(work))); - ALOGW_IF(!res.second, "We already inserted bitstreamId %d to decoder?", bitstreamId); - // Directly report the empty CSD work as finished. if (isCSDWork && isEmptyWork) reportWorkIfFinished(bitstreamId); } @@ -482,8 +496,18 @@ void V4L2DecodeComponent::onDecodeDone(int32_t bitstreamId, VideoDecoder::Decode VideoDecoder::DecodeStatusToString(status)); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + auto it = mWorksAtDecoder.find(bitstreamId); + ALOG_ASSERT(it != mWorksAtDecoder.end()); + C2Work* work = it->second.get(); + switch (status) { case VideoDecoder::DecodeStatus::kAborted: + work->input.buffers.front().reset(); + work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>( + work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME); + mOutputBitstreamIds.push(bitstreamId); + + pumpReportWork(); return; case VideoDecoder::DecodeStatus::kError: @@ -491,10 +515,6 @@ void V4L2DecodeComponent::onDecodeDone(int32_t bitstreamId, VideoDecoder::Decode return; case VideoDecoder::DecodeStatus::kOk: - auto it = mWorksAtDecoder.find(bitstreamId); - ALOG_ASSERT(it != mWorksAtDecoder.end()); - C2Work* work = it->second.get(); - // Release the input buffer. work->input.buffers.front().reset(); @@ -522,9 +542,6 @@ void V4L2DecodeComponent::onOutputFrameReady(std::unique_ptr<VideoFrame> frame) C2Work* work = it->second.get(); C2ConstGraphicBlock constBlock = std::move(frame)->getGraphicBlock(); - // TODO(b/160307705): Consider to remove the dependency of C2VdaBqBlockPool. - MarkBlockPoolDataAsShared(constBlock); - std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(std::move(constBlock)); if (mPendingColorAspectsChange && work->input.ordinal.frameIndex.peeku() >= mPendingColorAspectsChangeFrameIndex) { @@ -597,7 +614,10 @@ bool V4L2DecodeComponent::reportWorkIfFinished(int32_t bitstreamId) { } auto it = mWorksAtDecoder.find(bitstreamId); - ALOG_ASSERT(it != mWorksAtDecoder.end()); + if (it == mWorksAtDecoder.end()) { + ALOGI("work(bitstreamId = %d) is dropped, skip.", bitstreamId); + return true; + } if (!isWorkDone(*(it->second))) { ALOGV("work(bitstreamId = %d) is not done yet.", bitstreamId); @@ -621,25 +641,33 @@ bool V4L2DecodeComponent::reportEOSWork() { ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); - // In this moment all works prior to EOS work should be done and returned to listener. - if (mWorksAtDecoder.size() != 1u) { - ALOGE("It shouldn't have remaining works in mWorksAtDecoder except EOS work."); - for (const auto& kv : mWorksAtDecoder) { - ALOGE("bitstreamId(%d) => Work index=%llu, timestamp=%llu", kv.first, - kv.second->input.ordinal.frameIndex.peekull(), - kv.second->input.ordinal.timestamp.peekull()); - } + const auto it = + std::find_if(mWorksAtDecoder.begin(), mWorksAtDecoder.end(), [](const auto& kv) { + return kv.second->input.flags & C2FrameData::FLAG_END_OF_STREAM; + }); + if (it == mWorksAtDecoder.end()) { + ALOGE("Failed to find EOS work."); return false; } - std::unique_ptr<C2Work> eosWork(std::move(mWorksAtDecoder.begin()->second)); - mWorksAtDecoder.clear(); + std::unique_ptr<C2Work> eosWork(std::move(it->second)); + mWorksAtDecoder.erase(it); eosWork->result = C2_OK; eosWork->workletsProcessed = static_cast<uint32_t>(eosWork->worklets.size()); eosWork->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM; if (!eosWork->input.buffers.empty()) eosWork->input.buffers.front().reset(); + if (!mWorksAtDecoder.empty()) { + ALOGW("There are remaining works except EOS work. abandon them."); + for (const auto& kv : mWorksAtDecoder) { + ALOGW("bitstreamId(%d) => Work index=%llu, timestamp=%llu", kv.first, + kv.second->input.ordinal.frameIndex.peekull(), + kv.second->input.ordinal.timestamp.peekull()); + } + reportAbandonedWorks(); + } + return reportWork(std::move(eosWork)); } @@ -647,6 +675,12 @@ bool V4L2DecodeComponent::reportWork(std::unique_ptr<C2Work> work) { ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull()); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + auto sharedThis = mStdWeakThis.lock(); + if (sharedThis == nullptr) { + ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); + return false; + } + if (!mListener) { ALOGE("mListener is nullptr, setListener_vb() not called?"); return false; @@ -654,7 +688,7 @@ bool V4L2DecodeComponent::reportWork(std::unique_ptr<C2Work> work) { std::list<std::unique_ptr<C2Work>> finishedWorks; finishedWorks.emplace_back(std::move(work)); - mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorks)); + mListener->onWorkDone_nb(std::move(sharedThis), std::move(finishedWorks)); return true; } @@ -691,6 +725,12 @@ void V4L2DecodeComponent::reportAbandonedWorks() { ALOGV("%s()", __func__); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + auto sharedThis = mStdWeakThis.lock(); + if (sharedThis == nullptr) { + ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); + return; + } + std::list<std::unique_ptr<C2Work>> abandonedWorks; while (!mPendingWorks.empty()) { abandonedWorks.emplace_back(std::move(mPendingWorks.front())); @@ -714,7 +754,7 @@ void V4L2DecodeComponent::reportAbandonedWorks() { ALOGE("mListener is nullptr, setListener_vb() not called?"); return; } - mListener->onWorkDone_nb(shared_from_this(), std::move(abandonedWorks)); + mListener->onWorkDone_nb(std::move(sharedThis), std::move(abandonedWorks)); } } @@ -788,6 +828,12 @@ void V4L2DecodeComponent::reportError(c2_status_t error) { ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error)); ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence()); + auto sharedThis = mStdWeakThis.lock(); + if (sharedThis == nullptr) { + ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__); + return; + } + if (mComponentState.load() == ComponentState::ERROR) return; mComponentState.store(ComponentState::ERROR); @@ -795,21 +841,7 @@ void V4L2DecodeComponent::reportError(c2_status_t error) { ALOGE("mListener is nullptr, setListener_vb() not called?"); return; } - mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error)); -} - -c2_status_t V4L2DecodeComponent::reset() { - ALOGV("%s()", __func__); - - return stop(); -} - -c2_status_t V4L2DecodeComponent::release() { - ALOGV("%s()", __func__); - - c2_status_t ret = reset(); - mComponentState.store(ComponentState::RELEASED); - return ret; + mListener->onError_nb(std::move(sharedThis), static_cast<uint32_t>(error)); } c2_status_t V4L2DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) { diff --git a/components/V4L2DecodeInterface.cpp b/components/V4L2DecodeInterface.cpp index a09fcc4..4bc4121 100644 --- a/components/V4L2DecodeInterface.cpp +++ b/components/V4L2DecodeInterface.cpp @@ -14,8 +14,8 @@ #include <media/stagefright/foundation/MediaDefs.h> #include <v4l2_codec2/common/V4L2ComponentCommon.h> +#include <v4l2_codec2/common/V4L2Device.h> #include <v4l2_codec2/plugin_store/V4L2AllocatorId.h> -#include <v4l2_device.h> namespace android { namespace { @@ -49,22 +49,6 @@ size_t calculateInputBufferSize(size_t area) { if (area > k1080pArea) return kInputBufferSizeFor4K; return kInputBufferSizeFor1080p; } - -uint32_t getOutputDelay(VideoCodec codec) { - switch (codec) { - case VideoCodec::H264: - // Due to frame reordering an H264 decoder might need multiple additional input frames to be - // queued before being able to output the associated decoded buffers. We need to tell the - // codec2 framework that it should not stop queuing new work items until the maximum number - // of frame reordering is reached, to avoid stalling the decoder. - return 16; - case VideoCodec::VP8: - return 0; - case VideoCodec::VP9: - return 0; - } -} - } // namespace // static @@ -138,6 +122,10 @@ V4L2DecodeInterface::V4L2DecodeInterface(const std::string& name, return; } + addParameter(DefineParam(mKind, C2_PARAMKEY_COMPONENT_KIND) + .withConstValue(new C2ComponentKindSetting(C2Component::KIND_DECODER)) + .build()); + std::string inputMime; switch (*mVideoCodec) { case VideoCodec::H264: @@ -246,7 +234,7 @@ V4L2DecodeInterface::V4L2DecodeInterface(const std::string& name, bool secureMode = name.find(".secure") != std::string::npos; const C2Allocator::id_t inputAllocators[] = {secureMode ? V4L2AllocatorId::SECURE_LINEAR - : C2PlatformAllocatorStore::BLOB}; + : C2AllocatorStore::DEFAULT_LINEAR}; const C2Allocator::id_t outputAllocators[] = {V4L2AllocatorId::V4L2_BUFFERPOOL}; const C2Allocator::id_t surfaceAllocator = @@ -334,7 +322,7 @@ V4L2DecodeInterface::V4L2DecodeInterface(const std::string& name, } size_t V4L2DecodeInterface::getInputBufferSize() const { - return calculateInputBufferSize(getMaxSize().GetArea()); + return calculateInputBufferSize(mSize->width * mSize->height); } c2_status_t V4L2DecodeInterface::queryColorAspects( @@ -350,4 +338,19 @@ c2_status_t V4L2DecodeInterface::queryColorAspects( return status; } +uint32_t V4L2DecodeInterface::getOutputDelay(VideoCodec codec) { + switch (codec) { + case VideoCodec::H264: + // Due to frame reordering an H264 decoder might need multiple additional input frames to be + // queued before being able to output the associated decoded buffers. We need to tell the + // codec2 framework that it should not stop queuing new work items until the maximum number + // of frame reordering is reached, to avoid stalling the decoder. + return 16; + case VideoCodec::VP8: + return 0; + case VideoCodec::VP9: + return 0; + } +} + } // namespace android diff --git a/components/V4L2Decoder.cpp b/components/V4L2Decoder.cpp index d52bd6c..d694837 100644 --- a/components/V4L2Decoder.cpp +++ b/components/V4L2Decoder.cpp @@ -16,6 +16,9 @@ #include <base/memory/ptr_util.h> #include <log/log.h> +#include <v4l2_codec2/common/Common.h> +#include <v4l2_codec2/common/Fourcc.h> + namespace android { namespace { @@ -23,6 +26,13 @@ constexpr size_t kNumInputBuffers = 16; // Extra buffers for transmitting in the whole video pipeline. constexpr size_t kNumExtraOutputBuffers = 4; +// Currently we only support flexible pixel 420 format YCBCR_420_888 in Android. +// Here is the list of flexible 420 format. +constexpr std::initializer_list<uint32_t> kSupportedOutputFourccs = { + Fourcc::YU12, Fourcc::YV12, Fourcc::YM12, Fourcc::YM21, + Fourcc::NV12, Fourcc::NV21, Fourcc::NM12, Fourcc::NM21, +}; + uint32_t VideoCodecToV4L2PixFmt(VideoCodec codec) { switch (codec) { case VideoCodec::H264: @@ -64,17 +74,17 @@ V4L2Decoder::~V4L2Decoder() { // Streamoff input and output queue. if (mOutputQueue) { - mOutputQueue->Streamoff(); - mOutputQueue->DeallocateBuffers(); + mOutputQueue->streamoff(); + mOutputQueue->deallocateBuffers(); mOutputQueue = nullptr; } if (mInputQueue) { - mInputQueue->Streamoff(); - mInputQueue->DeallocateBuffers(); + mInputQueue->streamoff(); + mInputQueue->deallocateBuffers(); mInputQueue = nullptr; } if (mDevice) { - mDevice->StopPolling(); + mDevice->stopPolling(); mDevice = nullptr; } } @@ -94,15 +104,15 @@ bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, G return false; } - mDevice = media::V4L2Device::Create(); + mDevice = V4L2Device::create(); const uint32_t inputPixelFormat = VideoCodecToV4L2PixFmt(codec); - if (!mDevice->Open(media::V4L2Device::Type::kDecoder, inputPixelFormat)) { + if (!mDevice->open(V4L2Device::Type::kDecoder, inputPixelFormat)) { ALOGE("Failed to open device for %s", VideoCodecToString(codec)); return false; } - if (!mDevice->HasCapabilities(V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING)) { + if (!mDevice->hasCapabilities(V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING)) { ALOGE("Device does not have VIDEO_M2M_MPLANE and STREAMING capabilities."); return false; } @@ -110,7 +120,7 @@ bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, G struct v4l2_decoder_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.cmd = V4L2_DEC_CMD_STOP; - if (mDevice->Ioctl(VIDIOC_TRY_DECODER_CMD, &cmd) != 0) { + if (mDevice->ioctl(VIDIOC_TRY_DECODER_CMD, &cmd) != 0) { ALOGE("Device does not support flushing (V4L2_DEC_CMD_STOP)"); return false; } @@ -119,14 +129,14 @@ bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, G struct v4l2_event_subscription sub; memset(&sub, 0, sizeof(sub)); sub.type = V4L2_EVENT_SOURCE_CHANGE; - if (mDevice->Ioctl(VIDIOC_SUBSCRIBE_EVENT, &sub) != 0) { + if (mDevice->ioctl(VIDIOC_SUBSCRIBE_EVENT, &sub) != 0) { ALOGE("ioctl() failed: VIDIOC_SUBSCRIBE_EVENT: V4L2_EVENT_SOURCE_CHANGE"); return false; } // Create Input/Output V4L2Queue, and setup input queue. - mInputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); - mOutputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + mInputQueue = mDevice->getQueue(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + mOutputQueue = mDevice->getQueue(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); if (!mInputQueue || !mOutputQueue) { ALOGE("Failed to create V4L2 queue."); return false; @@ -136,7 +146,7 @@ bool V4L2Decoder::start(const VideoCodec& codec, const size_t inputBufferSize, G return false; } - if (!mDevice->StartPolling(::base::BindRepeating(&V4L2Decoder::serviceDeviceTask, mWeakThis), + if (!mDevice->startPolling(::base::BindRepeating(&V4L2Decoder::serviceDeviceTask, mWeakThis), ::base::BindRepeating(&V4L2Decoder::onError, mWeakThis))) { ALOGE("Failed to start polling V4L2 device."); return false; @@ -153,25 +163,25 @@ bool V4L2Decoder::setupInputFormat(const uint32_t inputPixelFormat, const size_t // Check if the format is supported. std::vector<uint32_t> formats = - mDevice->EnumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + mDevice->enumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); if (std::find(formats.begin(), formats.end(), inputPixelFormat) == formats.end()) { ALOGE("Input codec s not supported by device."); return false; } // Setup the input format. - auto format = mInputQueue->SetFormat(inputPixelFormat, media::Size(), inputBufferSize); + auto format = mInputQueue->setFormat(inputPixelFormat, ui::Size(), inputBufferSize, 0); if (!format) { ALOGE("Failed to call IOCTL to set input format."); return false; } ALOG_ASSERT(format->fmt.pix_mp.pixelformat == inputPixelFormat); - if (mInputQueue->AllocateBuffers(kNumInputBuffers, V4L2_MEMORY_DMABUF) == 0) { + if (mInputQueue->allocateBuffers(kNumInputBuffers, V4L2_MEMORY_DMABUF) == 0) { ALOGE("Failed to allocate input buffer."); return false; } - if (!mInputQueue->Streamon()) { + if (!mInputQueue->streamon()) { ALOGE("Failed to streamon input queue."); return false; } @@ -203,7 +213,7 @@ void V4L2Decoder::drain(DecodeCB drainCb) { switch (mState) { case State::Idle: - ALOGD("Nothing need to drain, ignore."); + ALOGV("Nothing need to drain, ignore."); mTaskRunner->PostTask( FROM_HERE, ::base::BindOnce(std::move(drainCb), VideoDecoder::DecodeStatus::kOk)); return; @@ -242,8 +252,8 @@ void V4L2Decoder::pumpDecodeRequest() { // yet. Also, V4L2VDA calls STREAMOFF and STREAMON after resolution // change. They implicitly send a V4L2_DEC_CMD_STOP and V4L2_DEC_CMD_START // to the decoder. - if (mInputQueue->QueuedBuffersCount() > 0) { - ALOGD("Wait for all input buffers dequeued."); + if (mInputQueue->queuedBuffersCount() > 0) { + ALOGV("Wait for all input buffers dequeued."); return; } @@ -261,7 +271,7 @@ void V4L2Decoder::pumpDecodeRequest() { } // Pause if no free input buffer. We resume decoding after dequeueing input buffers. - auto inputBuffer = mInputQueue->GetFreeBuffer(); + auto inputBuffer = mInputQueue->getFreeBuffer(); if (!inputBuffer) { ALOGV("There is no free input buffer."); return; @@ -272,8 +282,8 @@ void V4L2Decoder::pumpDecodeRequest() { const int32_t bitstreamId = request.buffer->id; ALOGV("QBUF to input queue, bitstreadId=%d", bitstreamId); - inputBuffer->SetTimeStamp({.tv_sec = bitstreamId}); - size_t planeSize = inputBuffer->GetPlaneSize(0); + inputBuffer->setTimeStamp({.tv_sec = bitstreamId}); + size_t planeSize = inputBuffer->getPlaneSize(0); if (request.buffer->size > planeSize) { ALOGE("The input size (%zu) is not enough, we need %zu", planeSize, request.buffer->size); @@ -283,11 +293,11 @@ void V4L2Decoder::pumpDecodeRequest() { ALOGV("Set bytes_used=%zu, offset=%zu", request.buffer->offset + request.buffer->size, request.buffer->offset); - inputBuffer->SetPlaneDataOffset(0, request.buffer->offset); - inputBuffer->SetPlaneBytesUsed(0, request.buffer->offset + request.buffer->size); + inputBuffer->setPlaneDataOffset(0, request.buffer->offset); + inputBuffer->setPlaneBytesUsed(0, request.buffer->offset + request.buffer->size); std::vector<int> fds; fds.push_back(std::move(request.buffer->dmabuf_fd)); - if (!std::move(*inputBuffer).QueueDMABuf(fds)) { + if (!std::move(*inputBuffer).queueDMABuf(fds)) { ALOGE("%s(): Failed to QBUF to input queue, bitstreamId=%d", __func__, bitstreamId); onError(); return; @@ -302,7 +312,7 @@ void V4L2Decoder::flush() { ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); if (mState == State::Idle) { - ALOGD("Nothing need to flush, ignore."); + ALOGV("Nothing need to flush, ignore."); return; } if (mState == State::Error) { @@ -320,16 +330,23 @@ void V4L2Decoder::flush() { } // Streamoff both V4L2 queues to drop input and output buffers. - mDevice->StopPolling(); - mOutputQueue->Streamoff(); + mDevice->stopPolling(); + mOutputQueue->streamoff(); mFrameAtDevice.clear(); - mInputQueue->Streamoff(); + mInputQueue->streamoff(); // Streamon both V4L2 queues. - mInputQueue->Streamon(); - mOutputQueue->Streamon(); + mInputQueue->streamon(); + mOutputQueue->streamon(); + + // If there is no free buffer at mOutputQueue, tryFetchVideoFrame() should be triggerred after + // a buffer is DQBUF from output queue. Now all the buffers are dropped at mOutputQueue, we + // have to trigger tryFetchVideoFrame() here. + if (mVideoFramePool) { + tryFetchVideoFrame(); + } - if (!mDevice->StartPolling(::base::BindRepeating(&V4L2Decoder::serviceDeviceTask, mWeakThis), + if (!mDevice->startPolling(::base::BindRepeating(&V4L2Decoder::serviceDeviceTask, mWeakThis), ::base::BindRepeating(&V4L2Decoder::onError, mWeakThis))) { ALOGE("Failed to start polling V4L2 device."); onError(); @@ -341,22 +358,22 @@ void V4L2Decoder::flush() { void V4L2Decoder::serviceDeviceTask(bool event) { ALOGV("%s(event=%d) state=%s InputQueue(%s):%zu+%zu/%zu, OutputQueue(%s):%zu+%zu/%zu", __func__, - event, StateToString(mState), (mInputQueue->IsStreaming() ? "streamon" : "streamoff"), - mInputQueue->FreeBuffersCount(), mInputQueue->QueuedBuffersCount(), - mInputQueue->AllocatedBuffersCount(), - (mOutputQueue->IsStreaming() ? "streamon" : "streamoff"), - mOutputQueue->FreeBuffersCount(), mOutputQueue->QueuedBuffersCount(), - mOutputQueue->AllocatedBuffersCount()); + event, StateToString(mState), (mInputQueue->isStreaming() ? "streamon" : "streamoff"), + mInputQueue->freeBuffersCount(), mInputQueue->queuedBuffersCount(), + mInputQueue->allocatedBuffersCount(), + (mOutputQueue->isStreaming() ? "streamon" : "streamoff"), + mOutputQueue->freeBuffersCount(), mOutputQueue->queuedBuffersCount(), + mOutputQueue->allocatedBuffersCount()); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); if (mState == State::Error) return; // Dequeue output and input queue. bool inputDequeued = false; - while (mInputQueue->QueuedBuffersCount() > 0) { + while (mInputQueue->queuedBuffersCount() > 0) { bool success; - media::V4L2ReadableBufferRef dequeuedBuffer; - std::tie(success, dequeuedBuffer) = mInputQueue->DequeueBuffer(); + V4L2ReadableBufferRef dequeuedBuffer; + std::tie(success, dequeuedBuffer) = mInputQueue->dequeueBuffer(); if (!success) { ALOGE("Failed to dequeue buffer from input queue."); onError(); @@ -367,7 +384,7 @@ void V4L2Decoder::serviceDeviceTask(bool event) { inputDequeued = true; // Run the corresponding decode callback. - int32_t id = dequeuedBuffer->GetTimeStamp().tv_sec; + int32_t id = dequeuedBuffer->getTimeStamp().tv_sec; ALOGV("DQBUF from input queue, bitstreamId=%d", id); auto it = mPendingDecodeCbs.find(id); if (it == mPendingDecodeCbs.end()) { @@ -379,10 +396,10 @@ void V4L2Decoder::serviceDeviceTask(bool event) { } bool outputDequeued = false; - while (mOutputQueue->QueuedBuffersCount() > 0) { + while (mOutputQueue->queuedBuffersCount() > 0) { bool success; - media::V4L2ReadableBufferRef dequeuedBuffer; - std::tie(success, dequeuedBuffer) = mOutputQueue->DequeueBuffer(); + V4L2ReadableBufferRef dequeuedBuffer; + std::tie(success, dequeuedBuffer) = mOutputQueue->dequeueBuffer(); if (!success) { ALOGE("Failed to dequeue buffer from output queue."); onError(); @@ -392,12 +409,12 @@ void V4L2Decoder::serviceDeviceTask(bool event) { outputDequeued = true; - const size_t bufferId = dequeuedBuffer->BufferId(); - const int32_t bitstreamId = static_cast<int32_t>(dequeuedBuffer->GetTimeStamp().tv_sec); - const size_t bytesUsed = dequeuedBuffer->GetPlaneBytesUsed(0); - const bool isLast = dequeuedBuffer->IsLast(); - ALOGV("DQBUF from output queue, bufferId=%zu, corresponding bitstreamId=%d, bytesused=%zu", - bufferId, bitstreamId, bytesUsed); + const size_t bufferId = dequeuedBuffer->bufferId(); + const int32_t bitstreamId = static_cast<int32_t>(dequeuedBuffer->getTimeStamp().tv_sec); + const size_t bytesUsed = dequeuedBuffer->getPlaneBytesUsed(0); + const bool isLast = dequeuedBuffer->isLast(); + ALOGV("DQBUF from output queue, bufferId=%zu, bitstreamId=%d, bytesused=%zu, isLast=%d", + bufferId, bitstreamId, bytesUsed, isLast); // Get the corresponding VideoFrame of the dequeued buffer. auto it = mFrameAtDevice.find(bufferId); @@ -416,10 +433,10 @@ void V4L2Decoder::serviceDeviceTask(bool event) { // then the driver will fail to notify EOS. So we recycle the buffer immediately. ALOGV("Recycle empty buffer %zu back to V4L2 output queue.", bufferId); dequeuedBuffer.reset(); - auto outputBuffer = mOutputQueue->GetFreeBuffer(bufferId); + auto outputBuffer = mOutputQueue->getFreeBuffer(bufferId); ALOG_ASSERT(outputBuffer, "V4L2 output queue slot %zu is not freed.", bufferId); - if (!std::move(*outputBuffer).QueueDMABuf(frame->getFDs())) { + if (!std::move(*outputBuffer).queueDMABuf(frame->getFDs())) { ALOGE("%s(): Failed to recycle empty buffer to output queue.", __func__); onError(); return; @@ -461,7 +478,7 @@ bool V4L2Decoder::dequeueResolutionChangeEvent() { struct v4l2_event ev; memset(&ev, 0, sizeof(ev)); - while (mDevice->Ioctl(VIDIOC_DQEVENT, &ev) == 0) { + while (mDevice->ioctl(VIDIOC_DQEVENT, &ev) == 0) { if (ev.type == V4L2_EVENT_SOURCE_CHANGE && ev.u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { return true; @@ -474,40 +491,52 @@ bool V4L2Decoder::changeResolution() { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); - std::optional<struct v4l2_format> format = getFormatInfo(); + const std::optional<struct v4l2_format> format = getFormatInfo(); std::optional<size_t> numOutputBuffers = getNumOutputBuffers(); if (!format || !numOutputBuffers) { return false; } - mCodedSize.SetSize(format->fmt.pix_mp.width, format->fmt.pix_mp.height); + const ui::Size codedSize(format->fmt.pix_mp.width, format->fmt.pix_mp.height); + if (!setupOutputFormat(codedSize)) { + return false; + } + + const std::optional<struct v4l2_format> adjustedFormat = getFormatInfo(); + if (!adjustedFormat) { + return false; + } + mCodedSize.set(adjustedFormat->fmt.pix_mp.width, adjustedFormat->fmt.pix_mp.height); mVisibleRect = getVisibleRect(mCodedSize); ALOGI("Need %zu output buffers. coded size: %s, visible rect: %s", *numOutputBuffers, - mCodedSize.ToString().c_str(), mVisibleRect.ToString().c_str()); - if (mCodedSize.IsEmpty()) { + toString(mCodedSize).c_str(), toString(mVisibleRect).c_str()); + if (isEmpty(mCodedSize)) { ALOGE("Failed to get resolution from V4L2 driver."); return false; } - mOutputQueue->Streamoff(); - mOutputQueue->DeallocateBuffers(); + mOutputQueue->streamoff(); + mOutputQueue->deallocateBuffers(); mFrameAtDevice.clear(); mBlockIdToV4L2Id.clear(); - if (mOutputQueue->AllocateBuffers(*numOutputBuffers, V4L2_MEMORY_DMABUF) == 0) { + if (mOutputQueue->allocateBuffers(*numOutputBuffers, V4L2_MEMORY_DMABUF) == 0) { ALOGE("Failed to allocate output buffer."); return false; } - if (!mOutputQueue->Streamon()) { + if (!mOutputQueue->streamon()) { ALOGE("Failed to streamon output queue."); return false; } - // Always use fexible pixel 420 format YCBCR_420_888 in Android. - mGetPoolCb.Run(&mVideoFramePool, mCodedSize, HalPixelFormat::YCBCR_420_888, *numOutputBuffers); + // Release the previous VideoFramePool before getting a new one to guarantee only one pool + // exists at the same time. + mVideoFramePool.reset(); + // Always use flexible pixel 420 format YCBCR_420_888 in Android. + mVideoFramePool = mGetPoolCb.Run(mCodedSize, HalPixelFormat::YCBCR_420_888, *numOutputBuffers); if (!mVideoFramePool) { - ALOGE("Failed to get block pool with size: %s", mCodedSize.ToString().c_str()); + ALOGE("Failed to get block pool with size: %s", toString(mCodedSize).c_str()); return false; } @@ -515,13 +544,36 @@ bool V4L2Decoder::changeResolution() { return true; } +bool V4L2Decoder::setupOutputFormat(const ui::Size& size) { + for (const uint32_t& pixfmt : + mDevice->enumerateSupportedPixelformats(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) { + if (std::find(kSupportedOutputFourccs.begin(), kSupportedOutputFourccs.end(), pixfmt) == + kSupportedOutputFourccs.end()) { + ALOGD("Pixel format %s is not supported, skipping...", fourccToString(pixfmt).c_str()); + continue; + } + + if (mOutputQueue->setFormat(pixfmt, size, 0) != std::nullopt) { + return true; + } + } + + ALOGE("Failed to find supported pixel format"); + return false; +} + void V4L2Decoder::tryFetchVideoFrame() { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mVideoFramePool, "mVideoFramePool is null, haven't get the instance yet?"); - if (mOutputQueue->FreeBuffersCount() == 0) { - ALOGD("No free V4L2 output buffers, ignore."); + if (!mVideoFramePool) { + ALOGE("mVideoFramePool is null, failed to get the instance after resolution change?"); + onError(); + return; + } + + if (mOutputQueue->freeBuffersCount() == 0) { + ALOGV("No free V4L2 output buffers, ignore."); return; } @@ -547,18 +599,18 @@ void V4L2Decoder::onVideoFrameReady( uint32_t blockId; std::tie(frame, blockId) = std::move(*frameWithBlockId); - ::base::Optional<media::V4L2WritableBufferRef> outputBuffer; + std::optional<V4L2WritableBufferRef> outputBuffer; // Find the V4L2 buffer that is associated with this block. auto iter = mBlockIdToV4L2Id.find(blockId); if (iter != mBlockIdToV4L2Id.end()) { // If we have met this block in the past, reuse the same V4L2 buffer. - outputBuffer = mOutputQueue->GetFreeBuffer(iter->second); - } else if (mBlockIdToV4L2Id.size() < mOutputQueue->AllocatedBuffersCount()) { + outputBuffer = mOutputQueue->getFreeBuffer(iter->second); + } else if (mBlockIdToV4L2Id.size() < mOutputQueue->allocatedBuffersCount()) { // If this is the first time we see this block, give it the next // available V4L2 buffer. const size_t v4l2BufferId = mBlockIdToV4L2Id.size(); mBlockIdToV4L2Id.emplace(blockId, v4l2BufferId); - outputBuffer = mOutputQueue->GetFreeBuffer(v4l2BufferId); + outputBuffer = mOutputQueue->getFreeBuffer(v4l2BufferId); } else { // If this happens, this is a bug in VideoFramePool. It should never // provide more blocks than we have V4L2 buffers. @@ -566,15 +618,15 @@ void V4L2Decoder::onVideoFrameReady( } if (!outputBuffer) { - ALOGE("V4L2 buffer not available."); + ALOGE("V4L2 buffer not available. blockId=%u", blockId); onError(); return; } - uint32_t v4l2Id = outputBuffer->BufferId(); + uint32_t v4l2Id = outputBuffer->bufferId(); ALOGV("QBUF to output queue, blockId=%u, V4L2Id=%u", blockId, v4l2Id); - if (!std::move(*outputBuffer).QueueDMABuf(frame->getFDs())) { + if (!std::move(*outputBuffer).queueDMABuf(frame->getFDs())) { ALOGE("%s(): Failed to QBUF to output queue, blockId=%u, V4L2Id=%u", __func__, blockId, v4l2Id); onError(); @@ -597,7 +649,7 @@ std::optional<size_t> V4L2Decoder::getNumOutputBuffers() { struct v4l2_control ctrl; memset(&ctrl, 0, sizeof(ctrl)); ctrl.id = V4L2_CID_MIN_BUFFERS_FOR_CAPTURE; - if (mDevice->Ioctl(VIDIOC_G_CTRL, &ctrl) != 0) { + if (mDevice->ioctl(VIDIOC_G_CTRL, &ctrl) != 0) { ALOGE("ioctl() failed: VIDIOC_G_CTRL"); return std::nullopt; } @@ -613,7 +665,7 @@ std::optional<struct v4l2_format> V4L2Decoder::getFormatInfo() { struct v4l2_format format; memset(&format, 0, sizeof(format)); format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (mDevice->Ioctl(VIDIOC_G_FMT, &format) != 0) { + if (mDevice->ioctl(VIDIOC_G_FMT, &format) != 0) { ALOGE("ioctl() failed: VIDIOC_G_FMT"); return std::nullopt; } @@ -621,7 +673,7 @@ std::optional<struct v4l2_format> V4L2Decoder::getFormatInfo() { return format; } -media::Rect V4L2Decoder::getVisibleRect(const media::Size& codedSize) { +Rect V4L2Decoder::getVisibleRect(const ui::Size& codedSize) { ALOGV("%s()", __func__); ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); @@ -631,7 +683,7 @@ media::Rect V4L2Decoder::getVisibleRect(const media::Size& codedSize) { selection_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; selection_arg.target = V4L2_SEL_TGT_COMPOSE; - if (mDevice->Ioctl(VIDIOC_G_SELECTION, &selection_arg) == 0) { + if (mDevice->ioctl(VIDIOC_G_SELECTION, &selection_arg) == 0) { ALOGV("VIDIOC_G_SELECTION is supported"); visible_rect = &selection_arg.r; } else { @@ -640,24 +692,24 @@ media::Rect V4L2Decoder::getVisibleRect(const media::Size& codedSize) { memset(&crop_arg, 0, sizeof(crop_arg)); crop_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (mDevice->Ioctl(VIDIOC_G_CROP, &crop_arg) != 0) { + if (mDevice->ioctl(VIDIOC_G_CROP, &crop_arg) != 0) { ALOGW("ioctl() VIDIOC_G_CROP failed"); - return media::Rect(codedSize); + return Rect(codedSize.width, codedSize.height); } visible_rect = &crop_arg.c; } - media::Rect rect(visible_rect->left, visible_rect->top, visible_rect->width, - visible_rect->height); - ALOGD("visible rectangle is %s", rect.ToString().c_str()); - if (!media::Rect(codedSize).Contains(rect)) { - ALOGW("visible rectangle %s is not inside coded size %s", rect.ToString().c_str(), - codedSize.ToString().c_str()); - return media::Rect(codedSize); + Rect rect(visible_rect->left, visible_rect->top, visible_rect->left + visible_rect->width, + visible_rect->top + visible_rect->height); + ALOGV("visible rectangle is %s", toString(rect).c_str()); + if (!contains(Rect(codedSize.width, codedSize.height), rect)) { + ALOGW("visible rectangle %s is not inside coded size %s", toString(rect).c_str(), + toString(codedSize).c_str()); + return Rect(codedSize.width, codedSize.height); } - if (rect.IsEmpty()) { + if (rect.isEmpty()) { ALOGW("visible size is empty"); - return media::Rect(codedSize); + return Rect(codedSize.width, codedSize.height); } return rect; @@ -670,7 +722,7 @@ bool V4L2Decoder::sendV4L2DecoderCmd(bool start) { struct v4l2_decoder_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.cmd = start ? V4L2_DEC_CMD_START : V4L2_DEC_CMD_STOP; - if (mDevice->Ioctl(VIDIOC_DECODER_CMD, &cmd) != 0) { + if (mDevice->ioctl(VIDIOC_DECODER_CMD, &cmd) != 0) { ALOGE("ioctl() VIDIOC_DECODER_CMD failed: start=%d", start); return false; } diff --git a/components/V4L2EncodeComponent.cpp b/components/V4L2EncodeComponent.cpp index ab2230e..b4bbc0e 100644 --- a/components/V4L2EncodeComponent.cpp +++ b/components/V4L2EncodeComponent.cpp @@ -18,17 +18,19 @@ #include <android/hardware/graphics/common/1.0/types.h> #include <base/bind.h> #include <base/bind_helpers.h> +#include <cutils/properties.h> #include <log/log.h> #include <media/stagefright/MediaDefs.h> #include <ui/GraphicBuffer.h> +#include <ui/Size.h> -#include <fourcc.h> -#include <h264_parser.h> -#include <rect.h> #include <v4l2_codec2/common/Common.h> #include <v4l2_codec2/common/EncodeHelpers.h> -#include <v4l2_device.h> -#include <video_pixel_format.h> +#include <v4l2_codec2/common/FormatConverter.h> +#include <v4l2_codec2/common/VideoPixelFormat.h> +#include <v4l2_codec2/components/BitstreamBuffer.h> +#include <v4l2_codec2/components/V4L2EncodeInterface.h> +#include <v4l2_codec2/components/V4L2Encoder.h> using android::hardware::graphics::common::V1_0::BufferUsage; @@ -36,12 +38,12 @@ namespace android { namespace { -const media::VideoPixelFormat kInputPixelFormat = media::VideoPixelFormat::PIXEL_FORMAT_NV12; +const VideoPixelFormat kInputPixelFormat = VideoPixelFormat::NV12; // Get the video frame layout from the specified |inputBlock|. // TODO(dstaessens): Clean up code extracting layout from a C2GraphicBlock. std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGraphicBlock& block, - media::VideoPixelFormat* format) { + VideoPixelFormat* format) { ALOGV("%s()", __func__); // Get the C2PlanarLayout from the graphics block. The C2GraphicView returned by block.map() @@ -95,17 +97,17 @@ std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGra } if (!crcb && !semiplanar) { - *format = media::VideoPixelFormat::PIXEL_FORMAT_I420; + *format = VideoPixelFormat::I420; } else if (!crcb && semiplanar) { - *format = media::VideoPixelFormat::PIXEL_FORMAT_NV12; + *format = VideoPixelFormat::NV12; } else if (crcb && !semiplanar) { // HACK: pretend YV12 is I420 now since VEA only accepts I420. (YV12 will be used // for input byte-buffer mode). // TODO(dstaessens): Is this hack still necessary now we're not using the VEA directly? - //format = media::VideoPixelFormat::PIXEL_FORMAT_YV12; - *format = media::VideoPixelFormat::PIXEL_FORMAT_I420; + //format = VideoPixelFormat::YV12; + *format = VideoPixelFormat::I420; } else { - *format = media::VideoPixelFormat::PIXEL_FORMAT_NV21; + *format = VideoPixelFormat::NV21; } break; } @@ -113,7 +115,7 @@ std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGra offsets[C2PlanarLayout::PLANE_R] = layout.planes[C2PlanarLayout::PLANE_R].offset; strides[C2PlanarLayout::PLANE_R] = static_cast<uint32_t>(layout.planes[C2PlanarLayout::PLANE_R].rowInc); - *format = media::VideoPixelFormat::PIXEL_FORMAT_ARGB; + *format = VideoPixelFormat::ARGB; break; } default: @@ -123,56 +125,89 @@ std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGra std::vector<VideoFramePlane> planes; for (uint32_t i = 0; i < layout.rootPlanes; ++i) { - planes.push_back({offsets[i], strides[i]}); + // The mSize field is not used in our case, so we can safely set it to zero. + planes.push_back({strides[i], offsets[i], 0}); } return planes; } -// The maximum size for output buffer, which is chosen empirically for a 1080p video. -constexpr size_t kMaxBitstreamBufferSizeInBytes = 2 * 1024 * 1024; // 2MB -// The frame size for 1080p (FHD) video in pixels. -constexpr int k1080PSizeInPixels = 1920 * 1080; -// The frame size for 1440p (QHD) video in pixels. -constexpr int k1440PSizeInPixels = 2560 * 1440; - -// Use quadruple size of kMaxBitstreamBufferSizeInBytes when the input frame size is larger than -// 1440p, double if larger than 1080p. This is chosen empirically for some 4k encoding use cases and -// the Android CTS VideoEncoderTest (crbug.com/927284). -size_t GetMaxOutputBufferSize(const media::Size& size) { - if (size.GetArea() > k1440PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 4; - if (size.GetArea() > k1080PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 2; - return kMaxBitstreamBufferSizeInBytes; -} +// Get the video frame stride for the specified |format| and |size|. +std::optional<uint32_t> getVideoFrameStride(VideoPixelFormat format, ui::Size size) { + // Fetch a graphic block from the pool to determine the stride. + std::shared_ptr<C2BlockPool> pool; + c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool); + if (status != C2_OK) { + ALOGE("Failed to get basic graphic block pool (err=%d)", status); + return std::nullopt; + } -// These are rather subjectively tuned. -constexpr size_t kInputBufferCount = 2; -constexpr size_t kOutputBufferCount = 2; + // Android HAL format doesn't have I420, we use YV12 instead and swap the U and V planes when + // converting to NV12. YCBCR_420_888 will allocate NV12 by minigbm. + HalPixelFormat halFormat = (format == VideoPixelFormat::I420) ? HalPixelFormat::YV12 + : HalPixelFormat::YCBCR_420_888; + + std::shared_ptr<C2GraphicBlock> block; + status = pool->fetchGraphicBlock(size.width, size.height, static_cast<uint32_t>(halFormat), + C2MemoryUsage(C2MemoryUsage::CPU_READ), &block); + if (status != C2_OK) { + ALOGE("Failed to fetch graphic block (err=%d)", status); + return std::nullopt; + } -// Define V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR control code if not present in header files. -#ifndef V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR -#define V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR (V4L2_CID_MPEG_BASE + 388) -#endif + const C2ConstGraphicBlock constBlock = block->share(C2Rect(size.width, size.height), C2Fence()); + VideoPixelFormat pixelFormat; + std::optional<std::vector<VideoFramePlane>> planes = + getVideoFrameLayout(constBlock, &pixelFormat); + if (!planes || planes.value().empty()) { + ALOGE("Failed to get video frame layout from block"); + return std::nullopt; + } -} // namespace + return planes.value()[0].mStride; +} + +// Create an input frame from the specified graphic block. +std::unique_ptr<V4L2Encoder::InputFrame> CreateInputFrame(const C2ConstGraphicBlock& block, + uint64_t index, int64_t timestamp) { + VideoPixelFormat format; + std::optional<std::vector<VideoFramePlane>> planes = getVideoFrameLayout(block, &format); + if (!planes) { + ALOGE("Failed to get input block's layout"); + return nullptr; + } -// static -std::unique_ptr<V4L2EncodeComponent::InputFrame> V4L2EncodeComponent::InputFrame::Create( - const C2ConstGraphicBlock& block) { std::vector<int> fds; const C2Handle* const handle = block.handle(); for (int i = 0; i < handle->numFds; i++) { fds.emplace_back(handle->data[i]); } - return std::unique_ptr<InputFrame>(new InputFrame(std::move(fds))); + return std::make_unique<V4L2Encoder::InputFrame>(std::move(fds), std::move(planes.value()), + format, index, timestamp); } +} // namespace + +// static +std::atomic<int32_t> V4L2EncodeComponent::sConcurrentInstances = 0; + // static std::shared_ptr<C2Component> V4L2EncodeComponent::create( C2String name, c2_node_id_t id, std::shared_ptr<C2ReflectorHelper> helper, C2ComponentFactory::ComponentDeleter deleter) { ALOGV("%s(%s)", __func__, name.c_str()); + static const int32_t kMaxConcurrentInstances = + property_get_int32("debug.v4l2_codec2.encode.concurrent-instances", -1); + + static std::mutex mutex; + std::lock_guard<std::mutex> lock(mutex); + if (kMaxConcurrentInstances >= 0 && sConcurrentInstances.load() >= kMaxConcurrentInstances) { + ALOGW("Cannot create additional encoder, maximum number of instances reached: %d", + kMaxConcurrentInstances); + return nullptr; + } + auto interface = std::make_shared<V4L2EncodeInterface>(name, std::move(helper)); if (interface->status() != C2_OK) { ALOGE("Component interface initialization failed (error code %d)", interface->status()); @@ -190,6 +225,8 @@ V4L2EncodeComponent::V4L2EncodeComponent(C2String name, c2_node_id_t id, mInterface(std::move(interface)), mComponentState(ComponentState::LOADED) { ALOGV("%s(%s)", __func__, name.c_str()); + + sConcurrentInstances.fetch_add(1, std::memory_order_relaxed); } V4L2EncodeComponent::~V4L2EncodeComponent() { @@ -205,6 +242,8 @@ V4L2EncodeComponent::~V4L2EncodeComponent() { &mWeakThisFactory)); mEncoderThread.Stop(); } + + sConcurrentInstances.fetch_sub(1, std::memory_order_relaxed); ALOGV("%s(): done", __func__); } @@ -391,7 +430,6 @@ std::shared_ptr<C2ComponentInterface> V4L2EncodeComponent::intf() { void V4L2EncodeComponent::startTask(bool* success, ::base::WaitableEvent* done) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED); *success = initializeEncoder(); done->Signal(); @@ -401,128 +439,148 @@ void V4L2EncodeComponent::stopTask(::base::WaitableEvent* done) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - // Flushing the encoder will abort all pending work and stop polling and streaming on the V4L2 - // device queues. + // Flushing the encoder will abort all pending work. flush(); - // Deallocate all V4L2 device input and output buffers. - destroyInputBuffers(); - destroyOutputBuffers(); + mInputFormatConverter.reset(); + + mEncoder.reset(); + mOutputBlockPool.reset(); // Invalidate all weak pointers so no more functions will be executed on the encoder thread. mWeakThisFactory.InvalidateWeakPtrs(); - setEncoderState(EncoderState::UNINITIALIZED); done->Signal(); } void V4L2EncodeComponent::queueTask(std::unique_ptr<C2Work> work) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED); + ALOG_ASSERT(mEncoder); + + // Currently only a single worklet per work item is supported. An input buffer should always be + // supplied unless this is a drain or CSD request. + ALOG_ASSERT(work->input.buffers.size() <= 1u && work->worklets.size() == 1u); + + // Set the default values for the output worklet. + work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0); + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.ordinal = work->input.ordinal; - // If we're in the error state we can immediately return, freeing all buffers in the work item. - if (mEncoderState == EncoderState::ERROR) { + uint64_t index = work->input.ordinal.frameIndex.peeku(); + int64_t timestamp = static_cast<int64_t>(work->input.ordinal.timestamp.peeku()); + bool endOfStream = work->input.flags & C2FrameData::FLAG_END_OF_STREAM; + ALOGV("Queuing next encode (index: %" PRIu64 ", timestamp: %" PRId64 ", EOS: %d)", index, + timestamp, endOfStream); + + // The codec 2.0 framework might queue an empty CSD request, but this is currently not + // supported. We will return the CSD with the first encoded buffer work. + if (work->input.buffers.empty() && !endOfStream) { + ALOGV("Discarding empty CSD request"); + reportWork(std::move(work)); return; } - ALOGV("Queued work item (index: %llu, timestamp: %llu, EOS: %d)", - work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull(), - work->input.flags & C2FrameData::FLAG_END_OF_STREAM); + // By the time we get an input buffer, the output block pool should be configured. + if (!mOutputBlockPool && !getBlockPool()) { + reportError(C2_CORRUPTED); + return; + } - mInputWorkQueue.push(std::move(work)); + // If conversion is required but no free buffers are available we queue the work item. + if (mInputFormatConverter && !mInputFormatConverter->isReady()) { + ALOGV("Input format convertor ran out of buffers"); + mInputConverterQueue.push(std::move(work)); + return; + } - // If we were waiting for work, start encoding again. - if (mEncoderState == EncoderState::WAITING_FOR_INPUT) { - setEncoderState(EncoderState::ENCODING); - mEncoderTaskRunner->PostTask( - FROM_HERE, - ::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis)); + // If we have data to encode send it to the encoder. If conversion is required we will first + // convert the data to the requested pixel format. + if (!work->input.buffers.empty()) { + C2ConstGraphicBlock inputBlock = + work->input.buffers.front()->data().graphicBlocks().front(); + if (mInputFormatConverter) { + ALOGV("Converting input block (index: %" PRIu64 ")", index); + c2_status_t status = C2_CORRUPTED; + inputBlock = mInputFormatConverter->convertBlock(index, inputBlock, &status); + if (status != C2_OK) { + ALOGE("Failed to convert input block (index: %" PRIu64 ")", index); + reportError(status); + return; + } + } + if (!encode(inputBlock, index, timestamp)) { + return; + } + } + + mWorkQueue.push_back(std::move(work)); + if (endOfStream) { + mEncoder->drain(); } } -// TODO(dstaessens): Investigate improving drain logic after draining the virtio device is fixed. void V4L2EncodeComponent::drainTask(drain_mode_t /*drainMode*/) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - // We can only start draining if all the work in our input queue has been queued on the V4L2 - // device input queue, so we mark the last item in the input queue as EOS. - if (!mInputWorkQueue.empty()) { - ALOGV("Marking last item in input work queue as EOS"); - mInputWorkQueue.back()->input.flags = static_cast<C2FrameData::flags_t>( - mInputWorkQueue.back()->input.flags | C2FrameData::FLAG_END_OF_STREAM); + // We can only start draining if all work has been queued in the encoder, so we mark the last + // item waiting for conversion as EOS if required. + if (!mInputConverterQueue.empty()) { + C2Work* work = mInputConverterQueue.back().get(); + work->input.flags = static_cast<C2FrameData::flags_t>(work->input.flags | + C2FrameData::FLAG_END_OF_STREAM); return; } - // If the input queue is empty and there is only a single empty EOS work item in the output - // queue we can immediately consider flushing done. - if ((mOutputWorkQueue.size() == 1) && mOutputWorkQueue.back()->input.buffers.empty()) { - ALOG_ASSERT(mOutputWorkQueue.back()->input.flags & C2FrameData::FLAG_END_OF_STREAM); - setEncoderState(EncoderState::DRAINING); - mEncoderTaskRunner->PostTask( - FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::onDrainDone, mWeakThis, true)); - return; - } - - // If the input queue is empty all work that needs to be drained has already been queued in the - // V4L2 device, so we can immediately request a drain. - if (!mOutputWorkQueue.empty()) { - // Mark the last item in the output work queue as EOS, so we will only report it as - // finished after draining has completed. + // Mark the last item in the output work queue as EOS, so we will only report it as finished + // after draining has completed. + if (!mWorkQueue.empty()) { ALOGV("Starting drain and marking last item in output work queue as EOS"); - mOutputWorkQueue.back()->input.flags = C2FrameData::FLAG_END_OF_STREAM; - drain(); + C2Work* work = mWorkQueue.back().get(); + work->input.flags = static_cast<C2FrameData::flags_t>(work->input.flags | + C2FrameData::FLAG_END_OF_STREAM); + mEncoder->drain(); } } -void V4L2EncodeComponent::onDrainDone(bool done) { +void V4L2EncodeComponent::onDrainDone(bool success) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::DRAINING || mEncoderState == EncoderState::ERROR); + ALOG_ASSERT(!mWorkQueue.empty()); - if (mEncoderState == EncoderState::ERROR) { - return; - } - - if (!done) { + if (!success) { ALOGE("draining the encoder failed"); reportError(C2_CORRUPTED); return; } - // The last work item in the output work queue should be an EOS request. - if (mOutputWorkQueue.empty() || - !(mOutputWorkQueue.back()->input.flags & C2FrameData::FLAG_END_OF_STREAM)) { - ALOGE("The last item in the output work queue should be marked EOS"); - reportError(C2_CORRUPTED); + // Find the first work item marked as EOS. This might not be the first item in the queue, as + // previous buffers in the queue might still be waiting for their associated input buffers. + auto it = std::find_if( + mWorkQueue.cbegin(), mWorkQueue.cend(), [](const std::unique_ptr<C2Work>& work) { + return ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) && + !(work->worklets.back()->output.flags & C2FrameData::FLAG_END_OF_STREAM)); + }); + if (it == mWorkQueue.end()) { + ALOGW("No EOS work item found in queue"); return; } - // Mark the last item in the output work queue as EOS done. - C2Work* eosWork = mOutputWorkQueue.back().get(); + // Mark the item in the output work queue as EOS done. + C2Work* eosWork = it->get(); eosWork->worklets.back()->output.flags = C2FrameData::FLAG_END_OF_STREAM; // Draining is done which means all buffers on the device output queue have been returned, but // not all buffers on the device input queue might have been returned yet. - if ((mOutputWorkQueue.size() > 1) || !isWorkDone(*eosWork)) { + if ((eosWork != mWorkQueue.front().get()) || !isWorkDone(*eosWork)) { ALOGV("Draining done, waiting for input buffers to be returned"); return; } ALOGV("Draining done"); - reportWork(std::move(mOutputWorkQueue.front())); - mOutputWorkQueue.pop_front(); - - // Draining the encoder is now done, we can start encoding again. - if (!mInputWorkQueue.empty()) { - setEncoderState(EncoderState::ENCODING); - mEncoderTaskRunner->PostTask( - FROM_HERE, - ::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis)); - } else { - setEncoderState(EncoderState::WAITING_FOR_INPUT); - } + reportWork(std::move(mWorkQueue.front())); + mWorkQueue.pop_front(); } void V4L2EncodeComponent::flushTask(::base::WaitableEvent* done, @@ -532,11 +590,11 @@ void V4L2EncodeComponent::flushTask(::base::WaitableEvent* done, // Move all work that can immediately be aborted to flushedWork, and notify the caller. if (flushedWork) { - while (!mInputWorkQueue.empty()) { - std::unique_ptr<C2Work> work = std::move(mInputWorkQueue.front()); + while (!mInputConverterQueue.empty()) { + std::unique_ptr<C2Work> work = std::move(mInputConverterQueue.front()); work->input.buffers.clear(); flushedWork->push_back(std::move(work)); - mInputWorkQueue.pop(); + mInputConverterQueue.pop(); } } done->Signal(); @@ -556,290 +614,54 @@ void V4L2EncodeComponent::setListenerTask(const std::shared_ptr<Listener>& liste bool V4L2EncodeComponent::initializeEncoder() { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED); + ALOG_ASSERT(!mInputFormatConverter); + ALOG_ASSERT(!mEncoder); - mVisibleSize = mInterface->getInputVisibleSize(); - mKeyFramePeriod = mInterface->getKeyFramePeriod(); - mKeyFrameCounter = 0; mCSDSubmitted = false; - // Open the V4L2 device for encoding to the requested output format. - // TODO(dstaessens): Do we need to close the device first if already opened? - // TODO(dstaessens): Avoid conversion to VideoCodecProfile and use C2Config::profile_t directly. - media::VideoCodecProfile outputProfile = - c2ProfileToVideoCodecProfile(mInterface->getOutputProfile()); - uint32_t outputPixelFormat = - media::V4L2Device::VideoCodecProfileToV4L2PixFmt(outputProfile, false); - if (!outputPixelFormat) { - ALOGE("Invalid output profile %s", media::GetProfileName(outputProfile).c_str()); - return false; - } - - mDevice = media::V4L2Device::Create(); - if (!mDevice) { - ALOGE("Failed to create V4L2 device"); - return false; - } - - if (!mDevice->Open(media::V4L2Device::Type::kEncoder, outputPixelFormat)) { - ALOGE("Failed to open device for profile %s (%s)", - media::GetProfileName(outputProfile).c_str(), - media::FourccToString(outputPixelFormat).c_str()); - return false; - } - - // Make sure the device has all required capabilities (multi-planar Memory-To-Memory and - // streaming I/O), and whether flushing is supported. - if (!mDevice->HasCapabilities(V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING)) { - ALOGE("Device doesn't have the required capabilities"); - return false; - } - if (!mDevice->IsCommandSupported(V4L2_ENC_CMD_STOP)) { - ALOGE("Device does not support flushing (V4L2_ENC_CMD_STOP)"); - return false; - } - - // Get input/output queues so we can send encode request to the device and get back the results. - mInputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); - mOutputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); - if (!mInputQueue || !mOutputQueue) { - ALOGE("Failed to get V4L2 device queues"); - return false; - } - - // First try to configure the specified output format, as changing the output format can affect - // the configured input format. - if (!configureOutputFormat(outputProfile)) return false; - - // Configure the input format. If the device doesn't support the specified format we'll use one - // of the device's preferred formats in combination with an input format convertor. - if (!configureInputFormat(kInputPixelFormat)) return false; - - // Create input and output buffers. - // TODO(dstaessens): Avoid allocating output buffers, encode directly into blockpool buffers. - if (!createInputBuffers() || !createOutputBuffers()) return false; - - // Configure the device, setting all required controls. - uint8_t level = c2LevelToLevelIDC(mInterface->getOutputLevel()); - if (!configureDevice(outputProfile, level)) return false; - - // We're ready to start encoding now. - setEncoderState(EncoderState::WAITING_FOR_INPUT); - - // As initialization is asynchronous work might have already be queued. - if (!mInputWorkQueue.empty()) { - setEncoderState(EncoderState::ENCODING); - mEncoderTaskRunner->PostTask( - FROM_HERE, ::base::Bind(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis)); - } - return true; -} - -bool V4L2EncodeComponent::configureInputFormat(media::VideoPixelFormat inputFormat) { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED); - ALOG_ASSERT(!mInputQueue->IsStreaming()); - ALOG_ASSERT(!mVisibleSize.IsEmpty()); - ALOG_ASSERT(!mInputFormatConverter); - - // First try to use the requested pixel format directly. - ::base::Optional<struct v4l2_format> format; - auto fourcc = media::Fourcc::FromVideoPixelFormat(inputFormat, false); - if (fourcc) { - format = mInputQueue->SetFormat(fourcc->ToV4L2PixFmt(), mVisibleSize, 0); - } - - // If the device doesn't support the requested input format we'll try the device's preferred - // input pixel formats and use a format convertor. We need to try all formats as some formats - // might not be supported for the configured output format. - if (!format) { - std::vector<uint32_t> preferredFormats = - mDevice->PreferredInputFormat(media::V4L2Device::Type::kEncoder); - for (uint32_t i = 0; !format && i < preferredFormats.size(); ++i) { - format = mInputQueue->SetFormat(preferredFormats[i], mVisibleSize, 0); - } - } + // Get the requested profile and level. + C2Config::profile_t outputProfile = mInterface->getOutputProfile(); - if (!format) { - ALOGE("Failed to set input format to %s", - media::VideoPixelFormatToString(inputFormat).c_str()); - return false; + std::optional<uint8_t> h264Level; + if (outputProfile >= C2Config::PROFILE_AVC_BASELINE && + outputProfile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) { + h264Level = c2LevelToV4L2Level(mInterface->getOutputLevel()); } - // Check whether the negotiated input format is valid. The coded size might be adjusted to match - // encoder minimums, maximums and alignment requirements of the currently selected formats. - auto layout = media::V4L2Device::V4L2FormatToVideoFrameLayout(*format); - if (!layout) { - ALOGE("Invalid input layout"); + // Get the stride used by the C2 framework, as this might be different from the stride used by + // the V4L2 encoder. + std::optional<uint32_t> stride = + getVideoFrameStride(kInputPixelFormat, mInterface->getInputVisibleSize()); + if (!stride) { + ALOGE("Failed to get video frame stride"); + reportError(C2_CORRUPTED); return false; } - mInputLayout = layout.value(); - if (!media::Rect(mInputLayout->coded_size()).Contains(media::Rect(mVisibleSize))) { - ALOGE("Input size %s exceeds encoder capability, encoder can handle %s", - mVisibleSize.ToString().c_str(), mInputLayout->coded_size().ToString().c_str()); + mEncoder = V4L2Encoder::create( + outputProfile, h264Level, mInterface->getInputVisibleSize(), *stride, + mInterface->getKeyFramePeriod(), + ::base::BindRepeating(&V4L2EncodeComponent::fetchOutputBlock, mWeakThis), + ::base::BindRepeating(&V4L2EncodeComponent::onInputBufferDone, mWeakThis), + ::base::BindRepeating(&V4L2EncodeComponent::onOutputBufferDone, mWeakThis), + ::base::BindRepeating(&V4L2EncodeComponent::onDrainDone, mWeakThis), + ::base::BindRepeating(&V4L2EncodeComponent::reportError, mWeakThis, C2_CORRUPTED), + mEncoderTaskRunner); + if (!mEncoder) { + ALOGE("Failed to create V4L2Encoder (profile: %s)", profileToString(outputProfile)); return false; } - // Calculate the input coded size from the format. - // TODO(dstaessens): How is this different from mInputLayout->coded_size()? - mInputCodedSize = media::V4L2Device::AllocatedSizeFromV4L2Format(*format); - // Add an input format convertor if the device doesn't support the requested input format. - // Note: The amount of input buffers in the convertor should match the amount of buffers on the - // device input queue, to simplify logic. - // TODO(dstaessens): Currently an input format convertor is always required. Mapping an input - // buffer always seems to fail unless we copy it into a new a buffer first. As a temporary - // workaround the line below is commented, but this should be undone once the issue is fixed. - //if (mInputLayout->format() != inputFormat) { ALOGV("Creating input format convertor (%s)", - media::VideoPixelFormatToString(mInputLayout->format()).c_str()); + videoPixelFormatToString(mEncoder->inputFormat()).c_str()); mInputFormatConverter = - FormatConverter::Create(inputFormat, mVisibleSize, kInputBufferCount, mInputCodedSize); + FormatConverter::Create(mEncoder->inputFormat(), mEncoder->visibleSize(), + V4L2Encoder::kInputBufferCount, mEncoder->codedSize()); if (!mInputFormatConverter) { ALOGE("Failed to created input format convertor"); return false; } - //} - - // The coded input size might be different from the visible size due to alignment requirements, - // So we need to specify the visible rectangle. Note that this rectangle might still be adjusted - // due to hardware limitations. - // TODO(dstaessens): Overwrite mVisibleSize with the adapted visible size here? - media::Rect visibleRectangle(mVisibleSize.width(), mVisibleSize.height()); - - struct v4l2_rect rect; - rect.left = visibleRectangle.x(); - rect.top = visibleRectangle.y(); - rect.width = visibleRectangle.width(); - rect.height = visibleRectangle.height(); - - // Try to adjust the visible rectangle using the VIDIOC_S_SELECTION command. If this is not - // supported we'll try to use the VIDIOC_S_CROP command instead. The visible rectangle might be - // adjusted to conform to hardware limitations (e.g. round to closest horizontal and vertical - // offsets, width and height). - struct v4l2_selection selection_arg; - memset(&selection_arg, 0, sizeof(selection_arg)); - selection_arg.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; - selection_arg.target = V4L2_SEL_TGT_CROP; - selection_arg.r = rect; - if (mDevice->Ioctl(VIDIOC_S_SELECTION, &selection_arg) == 0) { - visibleRectangle = media::Rect(selection_arg.r.left, selection_arg.r.top, - selection_arg.r.width, selection_arg.r.height); - } else { - struct v4l2_crop crop; - memset(&crop, 0, sizeof(v4l2_crop)); - crop.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - crop.c = rect; - if (mDevice->Ioctl(VIDIOC_S_CROP, &crop) != 0 || - mDevice->Ioctl(VIDIOC_G_CROP, &crop) != 0) { - ALOGE("Failed to crop to specified visible rectangle"); - return false; - } - visibleRectangle = media::Rect(crop.c.left, crop.c.top, crop.c.width, crop.c.height); - } - - ALOGV("Input format set to %s (size: %s, adjusted size: %dx%d, coded size: %s)", - media::VideoPixelFormatToString(mInputLayout->format()).c_str(), - mVisibleSize.ToString().c_str(), visibleRectangle.width(), visibleRectangle.height(), - mInputCodedSize.ToString().c_str()); - - mVisibleSize.SetSize(visibleRectangle.width(), visibleRectangle.height()); - return true; -} - -bool V4L2EncodeComponent::configureOutputFormat(media::VideoCodecProfile outputProfile) { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED); - ALOG_ASSERT(!mOutputQueue->IsStreaming()); - ALOG_ASSERT(!mVisibleSize.IsEmpty()); - - auto format = mOutputQueue->SetFormat( - media::V4L2Device::VideoCodecProfileToV4L2PixFmt(outputProfile, false), mVisibleSize, - GetMaxOutputBufferSize(mVisibleSize)); - if (!format) { - ALOGE("Failed to set output format to %s", media::GetProfileName(outputProfile).c_str()); - return false; - } - - // The device might adjust the requested output buffer size to match hardware requirements. - mOutputBufferSize = ::base::checked_cast<size_t>(format->fmt.pix_mp.plane_fmt[0].sizeimage); - - ALOGV("Output format set to %s (buffer size: %u)", media::GetProfileName(outputProfile).c_str(), - mOutputBufferSize); - return true; -} - -bool V4L2EncodeComponent::configureDevice(media::VideoCodecProfile outputProfile, - std::optional<const uint8_t> outputH264Level) { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - - // Enable frame-level bitrate control. This is the only mandatory general control. - if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, - {media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, 1)})) { - ALOGW("Failed enabling bitrate control"); - // TODO(b/161508368): V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE is currently not supported yet, - // assume the operation was successful for now. - } - - // Additional optional controls: - // - Enable macroblock-level bitrate control. - // - Set GOP length to 0 to disable periodic key frames. - mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, - {media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE, 1), - media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0)}); - - // All controls below are H.264-specific, so we can return here if the profile is not H.264. - if (outputProfile < media::H264PROFILE_MIN || outputProfile > media::H264PROFILE_MAX) { - return true; - } - - // When encoding H.264 we want to prepend SPS and PPS to each IDR for resilience. Some - // devices support this through the V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR control. - // TODO(b/161495502): V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR is currently not supported - // yet, just log a warning if the operation was unsuccessful for now. - if (mDevice->IsCtrlExposed(V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR)) { - if (!mDevice->SetExtCtrls( - V4L2_CTRL_CLASS_MPEG, - {media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR, 1)})) { - ALOGE("Failed to configure device to prepend SPS and PPS to each IDR"); - return false; - } - ALOGV("Device supports prepending SPS and PPS to each IDR"); - } else { - ALOGW("Device doesn't support prepending SPS and PPS to IDR"); - } - - std::vector<media::V4L2ExtCtrl> h264Ctrls; - - // No B-frames, for lowest decoding latency. - h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_B_FRAMES, 0); - // Quantization parameter maximum value (for variable bitrate control). - h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 51); - - // Set H.264 profile. - int32_t profile = media::V4L2Device::VideoCodecProfileToV4L2H264Profile(outputProfile); - if (profile < 0) { - ALOGE("Trying to set invalid H.264 profile"); - return false; - } - h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_PROFILE, profile); - - // Set H.264 output level. Use Level 4.0 as fallback default. - // TODO(dstaessens): Investigate code added by hiroh@ recently to select level in Chrome VEA. - uint8_t h264Level = outputH264Level.value_or(media::H264SPS::kLevelIDC4p0); - h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_LEVEL, - media::V4L2Device::H264LevelIdcToV4L2H264Level(h264Level)); - - // Ask not to put SPS and PPS into separate bitstream buffers. - h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_HEADER_MODE, - V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME); - - // Ignore return value as these controls are optional. - mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, std::move(h264Ctrls)); return true; } @@ -864,30 +686,22 @@ bool V4L2EncodeComponent::updateEncodingParameters() { if (mBitrate != bitrate) { ALOG_ASSERT(bitrate > 0u); ALOGV("Setting bitrate to %u", bitrate); - if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, - {media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE, bitrate)})) { - // TODO(b/161495749): V4L2_CID_MPEG_VIDEO_BITRATE is currently not supported yet, assume - // the operation was successful for now. - ALOGW("Requesting bitrate change failed"); + if (!mEncoder->setBitrate(bitrate)) { + reportError(C2_CORRUPTED); + return false; } mBitrate = bitrate; } // Ask device to change framerate if it's different from the currently configured framerate. - // TODO(dstaessens): Move IOCTL to device and use helper function. uint32_t framerate = static_cast<uint32_t>(std::round(framerateInfo.value)); if (mFramerate != framerate) { ALOG_ASSERT(framerate > 0u); ALOGV("Setting framerate to %u", framerate); - struct v4l2_streamparm parms; - memset(&parms, 0, sizeof(v4l2_streamparm)); - parms.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - parms.parm.output.timeperframe.numerator = 1; - parms.parm.output.timeperframe.denominator = framerate; - if (mDevice->Ioctl(VIDIOC_S_PARM, &parms) != 0) { - // TODO(b/161499573): VIDIOC_S_PARM is currently not supported yet, assume the operation - // was successful for now. - ALOGW("Requesting framerate change failed"); + if (!mEncoder->setFramerate(framerate)) { + ALOGE("Requesting framerate change failed"); + reportError(C2_CORRUPTED); + return false; } mFramerate = framerate; } @@ -902,7 +716,7 @@ bool V4L2EncodeComponent::updateEncodingParameters() { return false; } if (requestKeyFrame.value == C2_TRUE) { - mKeyFrameCounter = 0; + mEncoder->requestKeyframe(); requestKeyFrame.value = C2_FALSE; std::vector<std::unique_ptr<C2SettingResult>> failures; status = mInterface->config({&requestKeyFrame}, C2_MAY_BLOCK, &failures); @@ -913,132 +727,22 @@ bool V4L2EncodeComponent::updateEncodingParameters() { } } - // Request the next frame to be a key frame each time the counter reaches 0. - if (mKeyFrameCounter == 0) { - if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, - {media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME)})) { - // TODO(b/161498590): V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME is currently not supported - // yet, assume the operation was successful for now. - ALOGW("Failed requesting key frame"); - } - } - return true; } -void V4L2EncodeComponent::scheduleNextEncodeTask() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::ENCODING || mEncoderState == EncoderState::ERROR); - - // If we're in the error state we can immediately return. - if (mEncoderState == EncoderState::ERROR) { - return; - } - - // Get the next work item. Currently only a single worklet per work item is supported. An input - // buffer should always be supplied unless this is a drain or CSD request. - ALOG_ASSERT(!mInputWorkQueue.empty()); - C2Work* work = mInputWorkQueue.front().get(); - ALOG_ASSERT(work->input.buffers.size() <= 1u && work->worklets.size() == 1u); - - // Set the default values for the output worklet. - work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0); - work->worklets.front()->output.buffers.clear(); - work->worklets.front()->output.ordinal = work->input.ordinal; - - uint64_t index = work->input.ordinal.frameIndex.peeku(); - int64_t timestamp = static_cast<int64_t>(work->input.ordinal.timestamp.peeku()); - bool endOfStream = work->input.flags & C2FrameData::FLAG_END_OF_STREAM; - ALOGV("Scheduling next encode (index: %" PRIu64 ", timestamp: %" PRId64 ", EOS: %d)", index, - timestamp, endOfStream); - - if (!work->input.buffers.empty()) { - // Check if the device has free input buffers available. If not we'll switch to the - // WAITING_FOR_INPUT_BUFFERS state, and resume encoding once we're notified buffers are - // available in the onInputBufferDone() task. Note: The input buffers are not copied into - // the device's input buffers, but rather a memory pointer is imported. We still have to - // throttle the number of enqueues queued simultaneously on the device however. - if (mInputQueue->FreeBuffersCount() == 0) { - ALOGV("Waiting for device to return input buffers"); - setEncoderState(EncoderState::WAITING_FOR_INPUT_BUFFERS); - return; - } - - C2ConstGraphicBlock inputBlock = - work->input.buffers.front()->data().graphicBlocks().front(); - - // If encoding fails, we'll wait for an event (e.g. input buffers available) to start - // encoding again. - if (!encode(inputBlock, index, timestamp)) { - return; - } - } - - // The codec 2.0 framework might queue an empty CSD request, but this is currently not - // supported. We will return the CSD with the first encoded buffer work. - // TODO(dstaessens): Avoid doing this, store CSD request work at start of output queue. - if (work->input.buffers.empty() && !endOfStream) { - ALOGV("Discarding empty CSD request"); - reportWork(std::move(mInputWorkQueue.front())); - } else { - mOutputWorkQueue.push_back(std::move(mInputWorkQueue.front())); - } - mInputWorkQueue.pop(); - - // Drain the encoder if required. - if (endOfStream) { - drainTask(C2Component::DRAIN_COMPONENT_WITH_EOS); - } - - if (mEncoderState == EncoderState::DRAINING) { - return; - } else if (mInputWorkQueue.empty()) { - setEncoderState(EncoderState::WAITING_FOR_INPUT); - return; - } - - // Queue the next work item to be encoded. - mEncoderTaskRunner->PostTask( - FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis)); -} - bool V4L2EncodeComponent::encode(C2ConstGraphicBlock block, uint64_t index, int64_t timestamp) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState == EncoderState::ENCODING); - - // Update dynamic encoding parameters (bitrate, framerate, key frame) if requested. - if (!updateEncodingParameters()) return false; - - mKeyFrameCounter = (mKeyFrameCounter + 1) % mKeyFramePeriod; - - // If required convert the data to the V4L2 device's configured input pixel format. We - // allocate the same amount of buffers on the device input queue and the format convertor, - // so we should never run out of conversion buffers if there are free buffers in the input - // queue. - if (mInputFormatConverter) { - if (!mInputFormatConverter->isReady()) { - ALOGE("Input format convertor ran out of buffers"); - reportError(C2_CORRUPTED); - return false; - } - - ALOGV("Converting input block (index: %" PRIu64 ")", index); - c2_status_t status = C2_CORRUPTED; - block = mInputFormatConverter->convertBlock(index, block, &status); - if (status != C2_OK) { - ALOGE("Failed to convert input block (index: %" PRIu64 ")", index); - reportError(status); - return false; - } - } + ALOG_ASSERT(mEncoder); ALOGV("Encoding input block (index: %" PRIu64 ", timestamp: %" PRId64 ", size: %dx%d)", index, timestamp, block.width(), block.height()); - // Create a video frame from the graphic block. - std::unique_ptr<InputFrame> frame = InputFrame::Create(block); + // Update dynamic encoding parameters (bitrate, framerate, key frame) if requested. + if (!updateEncodingParameters()) return false; + + // Create an input frame from the graphic block. + std::unique_ptr<V4L2Encoder::InputFrame> frame = CreateInputFrame(block, index, timestamp); if (!frame) { ALOGE("Failed to create video frame from input block (index: %" PRIu64 ", timestamp: %" PRId64 ")", @@ -1047,141 +751,71 @@ bool V4L2EncodeComponent::encode(C2ConstGraphicBlock block, uint64_t index, int6 return false; } - // Get the video frame layout and pixel format from the graphic block. - // TODO(dstaessens) Integrate getVideoFrameLayout() into InputFrame::Create() - media::VideoPixelFormat format; - std::optional<std::vector<VideoFramePlane>> planes = getVideoFrameLayout(block, &format); - if (!planes) { - ALOGE("Failed to get input block's layout"); - reportError(C2_CORRUPTED); - return false; - } - - if (!enqueueInputBuffer(std::move(frame), format, *planes, index, timestamp)) { - ALOGE("Failed to enqueue video frame (index: %" PRIu64 ", timestamp: %" PRId64 ")", index, - timestamp); - reportError(C2_CORRUPTED); + if (!mEncoder->encode(std::move(frame))) { return false; } - // Start streaming on the input and output queue if required. - if (!mInputQueue->IsStreaming()) { - ALOG_ASSERT(!mOutputQueue->IsStreaming()); - if (!mOutputQueue->Streamon() || !mInputQueue->Streamon()) { - ALOGE("Failed to start streaming on input and output queue"); - reportError(C2_CORRUPTED); - return false; - } - // Start polling on the V4L2 device. - startDevicePoll(); - } - - // Queue all buffers on the output queue. These buffers will be used to store the encoded - // bitstreams. - while (mOutputQueue->FreeBuffersCount() > 0) { - if (!enqueueOutputBuffer()) return false; - } - return true; } -void V4L2EncodeComponent::drain() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - - if (mEncoderState == EncoderState::DRAINING || mEncoderState == EncoderState::ERROR) { - return; - } - - ALOG_ASSERT(mInputQueue->IsStreaming() && mOutputQueue->IsStreaming()); - ALOG_ASSERT(!mOutputWorkQueue.empty()); - - // TODO(dstaessens): Move IOCTL to device class. - struct v4l2_encoder_cmd cmd; - memset(&cmd, 0, sizeof(v4l2_encoder_cmd)); - cmd.cmd = V4L2_ENC_CMD_STOP; - if (mDevice->Ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) { - ALOGE("Failed to stop encoder"); - onDrainDone(false); - return; - } - ALOGV("%s(): Sent STOP command to encoder", __func__); - - setEncoderState(EncoderState::DRAINING); -} - void V4L2EncodeComponent::flush() { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - // Stop the device poll thread. - stopDevicePoll(); - - // Stop streaming on the V4L2 device, which stops all currently queued work and releases all - // buffers currently in use by the device. - // TODO(b/160540027): Calling streamoff currently results in a bug. - for (auto& queue : {mInputQueue, mOutputQueue}) { - if (queue && queue->IsStreaming() && !queue->Streamoff()) { - ALOGE("Failed to stop streaming on the device queue"); - reportError(C2_CORRUPTED); - } - } - - // Return all buffers to the input format convertor and clear all references to graphic blocks - // in the input queue. We don't need to clear the output map as those buffers will still be - // used. - for (auto& it : mInputBuffersMap) { - if (mInputFormatConverter && it.second) { - mInputFormatConverter->returnBlock(it.first); - } - it.second = nullptr; - } + mEncoder->flush(); // Report all queued work items as aborted. std::list<std::unique_ptr<C2Work>> abortedWorkItems; - while (!mInputWorkQueue.empty()) { - std::unique_ptr<C2Work> work = std::move(mInputWorkQueue.front()); + while (!mInputConverterQueue.empty()) { + std::unique_ptr<C2Work> work = std::move(mInputConverterQueue.front()); work->result = C2_NOT_FOUND; work->input.buffers.clear(); abortedWorkItems.push_back(std::move(work)); - mInputWorkQueue.pop(); + mInputConverterQueue.pop(); } - while (!mOutputWorkQueue.empty()) { - std::unique_ptr<C2Work> work = std::move(mOutputWorkQueue.front()); + while (!mWorkQueue.empty()) { + std::unique_ptr<C2Work> work = std::move(mWorkQueue.front()); + // Return buffer to the input format convertor if required. + if (mInputFormatConverter && work->input.buffers.empty()) { + mInputFormatConverter->returnBlock(work->input.ordinal.frameIndex.peeku()); + } work->result = C2_NOT_FOUND; work->input.buffers.clear(); abortedWorkItems.push_back(std::move(work)); - mOutputWorkQueue.pop_front(); + mWorkQueue.pop_front(); } - if (!abortedWorkItems.empty()) + if (!abortedWorkItems.empty()) { mListener->onWorkDone_nb(shared_from_this(), std::move(abortedWorkItems)); - - // Streaming and polling on the V4L2 device input and output queues will be resumed once new - // encode work is queued. + } } -std::shared_ptr<C2LinearBlock> V4L2EncodeComponent::fetchOutputBlock() { - // TODO(dstaessens): fetchLinearBlock() might be blocking. - ALOGV("Fetching linear block (size: %u)", mOutputBufferSize); - std::shared_ptr<C2LinearBlock> outputBlock; +void V4L2EncodeComponent::fetchOutputBlock(uint32_t size, + std::unique_ptr<BitstreamBuffer>* buffer) { + ALOGV("Fetching linear block (size: %u)", size); + std::shared_ptr<C2LinearBlock> block; c2_status_t status = mOutputBlockPool->fetchLinearBlock( - mOutputBufferSize, + size, C2MemoryUsage(C2MemoryUsage::CPU_READ | static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)), - &outputBlock); + &block); if (status != C2_OK) { ALOGE("Failed to fetch linear block (error: %d)", status); reportError(status); - return nullptr; } - return outputBlock; + // Store a reference to the block to keep the fds alive. + int fd = block->handle()->data[0]; + ALOG_ASSERT(!mOutputBuffersMap[fd]); + mOutputBuffersMap[fd] = std::move(block); + + // TODO(dstaessens) Store the C2LinearBlock directly into the BitstreamBuffer. + *buffer = std::make_unique<BitstreamBuffer>(fd, fd, 0, size); } void V4L2EncodeComponent::onInputBufferDone(uint64_t index) { ALOGV("%s(): Input buffer done (index: %" PRIu64 ")", __func__, index); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED); + ALOG_ASSERT(mEncoder); // There are no guarantees the input buffers are returned in order, so we need to find the work // item which this buffer belongs to. @@ -1192,53 +826,45 @@ void V4L2EncodeComponent::onInputBufferDone(uint64_t index) { return; } - // We're done using the input block, release reference to return the block to the client. If - // using an input format convertor, we also need to return the block to the convertor. + // We're done using the input block, release reference to return the block to the client. LOG_ASSERT(!work->input.buffers.empty()); work->input.buffers.front().reset(); + + // Return the block to the convertor if required. If we have buffers awaiting conversion, we can + // now attempt to convert and encode them again. if (mInputFormatConverter) { c2_status_t status = mInputFormatConverter->returnBlock(index); if (status != C2_OK) { reportError(status); return; } + while (!mInputConverterQueue.empty() && mInputFormatConverter->isReady()) { + std::unique_ptr<C2Work> work = std::move(mInputConverterQueue.front()); + mInputConverterQueue.pop(); + queueTask(std::move(work)); + } } // Return all completed work items. The work item might have been waiting for it's input buffer // to be returned, in which case we can report it as completed now. As input buffers are not // necessarily returned in order we might be able to return multiple ready work items now. - while (!mOutputWorkQueue.empty() && isWorkDone(*mOutputWorkQueue.front())) { - reportWork(std::move(mOutputWorkQueue.front())); - mOutputWorkQueue.pop_front(); - } - - // We might have been waiting for input buffers to be returned after draining finished. - if (mEncoderState == EncoderState::DRAINING && mOutputWorkQueue.empty()) { - ALOGV("Draining done"); - mEncoderState = EncoderState::WAITING_FOR_INPUT_BUFFERS; - } - - // If we previously used up all input queue buffers we can start encoding again now. - if ((mEncoderState == EncoderState::WAITING_FOR_INPUT_BUFFERS) && !mInputWorkQueue.empty()) { - setEncoderState(EncoderState::ENCODING); - mEncoderTaskRunner->PostTask( - FROM_HERE, - ::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis)); + while (!mWorkQueue.empty() && isWorkDone(*mWorkQueue.front())) { + reportWork(std::move(mWorkQueue.front())); + mWorkQueue.pop_front(); } } -void V4L2EncodeComponent::onOutputBufferDone(uint32_t payloadSize, bool keyFrame, int64_t timestamp, - std::shared_ptr<C2LinearBlock> outputBlock) { - ALOGV("%s(): output buffer done (timestamp: %" PRId64 ", size: %u, key frame: %d)", __func__, - timestamp, payloadSize, keyFrame); +void V4L2EncodeComponent::onOutputBufferDone(size_t dataSize, int64_t timestamp, bool keyFrame, + std::unique_ptr<BitstreamBuffer> buffer) { + ALOGV("%s(): output buffer done (timestamp: %" PRId64 ", size: %zu, keyframe: %d)", __func__, + timestamp, dataSize, keyFrame); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - if (mEncoderState == EncoderState::ERROR) { - return; - } + std::shared_ptr<C2LinearBlock> outputBlock = std::move(mOutputBuffersMap[buffer->id]); + mOutputBuffersMap.erase(buffer->id); + ALOG_ASSERT(outputBlock); - C2ConstLinearBlock constBlock = - outputBlock->share(outputBlock->offset(), payloadSize, C2Fence()); + C2ConstLinearBlock constBlock = outputBlock->share(outputBlock->offset(), dataSize, C2Fence()); // If no CSD (content-specific-data, e.g. SPS for H.264) has been submitted yet, we expect this // output block to contain CSD. We only submit the CSD once, even if it's attached to each key @@ -1255,8 +881,8 @@ void V4L2EncodeComponent::onOutputBufferDone(uint32_t payloadSize, bool keyFrame } // Attach the CSD to the first item in our output work queue. - LOG_ASSERT(!mOutputWorkQueue.empty()); - C2Work* work = mOutputWorkQueue.front().get(); + LOG_ASSERT(!mWorkQueue.empty()); + C2Work* work = mWorkQueue.front().get(); work->worklets.front()->output.configUpdate.push_back(std::move(csd)); mCSDSubmitted = true; } @@ -1266,26 +892,31 @@ void V4L2EncodeComponent::onOutputBufferDone(uint32_t payloadSize, bool keyFrame if (!work) { // It's possible we got an empty CSD request with timestamp 0, which we currently just // discard. - // TODO(dstaessens): Investigate handling empty CSD requests. if (timestamp != 0) { reportError(C2_CORRUPTED); } return; } - std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateLinearBuffer(std::move(constBlock)); + std::shared_ptr<C2Buffer> linearBuffer = C2Buffer::CreateLinearBuffer(std::move(constBlock)); + if (!linearBuffer) { + ALOGE("Failed to create linear buffer from block"); + reportError(C2_CORRUPTED); + return; + } + if (keyFrame) { - buffer->setInfo( + linearBuffer->setInfo( std::make_shared<C2StreamPictureTypeMaskInfo::output>(0u, C2Config::SYNC_FRAME)); } - work->worklets.front()->output.buffers.emplace_back(buffer); + work->worklets.front()->output.buffers.emplace_back(std::move(linearBuffer)); // We can report the work item as completed if its associated input buffer has also been // released. As output buffers are not necessarily returned in order we might be able to return // multiple ready work items now. - while (!mOutputWorkQueue.empty() && isWorkDone(*mOutputWorkQueue.front())) { - reportWork(std::move(mOutputWorkQueue.front())); - mOutputWorkQueue.pop_front(); + while (!mWorkQueue.empty() && isWorkDone(*mWorkQueue.front())) { + reportWork(std::move(mWorkQueue.front())); + mWorkQueue.pop_front(); } } @@ -1293,11 +924,11 @@ C2Work* V4L2EncodeComponent::getWorkByIndex(uint64_t index) { ALOGV("%s(): getting work item (index: %" PRIu64 ")", __func__, index); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - auto it = std::find_if(mOutputWorkQueue.begin(), mOutputWorkQueue.end(), + auto it = std::find_if(mWorkQueue.begin(), mWorkQueue.end(), [index](const std::unique_ptr<C2Work>& w) { return w->input.ordinal.frameIndex.peeku() == index; }); - if (it == mOutputWorkQueue.end()) { + if (it == mWorkQueue.end()) { ALOGE("Failed to find work (index: %" PRIu64 ")", index); return nullptr; } @@ -1312,13 +943,12 @@ C2Work* V4L2EncodeComponent::getWorkByTimestamp(int64_t timestamp) { // Find the work with specified timestamp by looping over the output work queue. This should be // very fast as the output work queue will never be longer then a few items. Ignore empty work // items that are marked as EOS, as their timestamp might clash with other work items. - auto it = std::find_if(mOutputWorkQueue.begin(), mOutputWorkQueue.end(), - [timestamp](const std::unique_ptr<C2Work>& w) { - return !(w->input.flags & C2FrameData::FLAG_END_OF_STREAM) && - w->input.ordinal.timestamp.peeku() == - static_cast<uint64_t>(timestamp); - }); - if (it == mOutputWorkQueue.end()) { + auto it = std::find_if( + mWorkQueue.begin(), mWorkQueue.end(), [timestamp](const std::unique_ptr<C2Work>& w) { + return !(w->input.flags & C2FrameData::FLAG_END_OF_STREAM) && + w->input.ordinal.timestamp.peeku() == static_cast<uint64_t>(timestamp); + }); + if (it == mWorkQueue.end()) { ALOGE("Failed to find work (timestamp: %" PRIu64 ")", timestamp); return nullptr; } @@ -1366,321 +996,27 @@ void V4L2EncodeComponent::reportWork(std::unique_ptr<C2Work> work) { mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorkList)); } -bool V4L2EncodeComponent::startDevicePoll() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - - if (!mDevice->StartPolling( - ::base::BindRepeating(&V4L2EncodeComponent::serviceDeviceTask, mWeakThis), - ::base::BindRepeating(&V4L2EncodeComponent::onPollError, mWeakThis))) { - ALOGE("Device poll thread failed to start"); - reportError(C2_CORRUPTED); - return false; - } - - ALOGV("Device poll started"); - return true; -} - -bool V4L2EncodeComponent::stopDevicePoll() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - - if (!mDevice->StopPolling()) { - ALOGE("Failed to stop polling on the device"); - reportError(C2_CORRUPTED); - return false; - } - - ALOGV("Device poll stopped"); - return true; -} - -void V4L2EncodeComponent::onPollError() { - ALOGV("%s()", __func__); - reportError(C2_CORRUPTED); -} - -void V4L2EncodeComponent::serviceDeviceTask(bool /*event*/) { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED); - - if (mEncoderState == EncoderState::ERROR) { - return; - } - - // Dequeue completed input (VIDEO_OUTPUT) buffers, and recycle to the free list. - while (mInputQueue->QueuedBuffersCount() > 0) { - if (!dequeueInputBuffer()) break; - } - - // Dequeue completed output (VIDEO_CAPTURE) buffers, and recycle to the free list. - while (mOutputQueue->QueuedBuffersCount() > 0) { - if (!dequeueOutputBuffer()) break; - } - - ALOGV("%s() - done", __func__); -} - -bool V4L2EncodeComponent::enqueueInputBuffer(std::unique_ptr<InputFrame> frame, - media::VideoPixelFormat format, - const std::vector<VideoFramePlane>& planes, - int64_t index, int64_t timestamp) { - ALOGV("%s(): queuing input buffer (index: %" PRId64 ")", __func__, index); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mInputQueue->FreeBuffersCount() > 0); - ALOG_ASSERT(mEncoderState == EncoderState::ENCODING); - ALOG_ASSERT(mInputLayout->format() == format); - ALOG_ASSERT(mInputLayout->planes().size() == planes.size()); - - auto buffer = mInputQueue->GetFreeBuffer(); - if (!buffer) { - ALOGE("Failed to get free buffer from device input queue"); - return false; - } - - // Mark the buffer with the frame's timestamp so we can identify the associated output buffers. - buffer->SetTimeStamp( - {.tv_sec = static_cast<time_t>(timestamp / ::base::Time::kMicrosecondsPerSecond), - .tv_usec = static_cast<time_t>(timestamp % ::base::Time::kMicrosecondsPerSecond)}); - size_t bufferId = buffer->BufferId(); - - for (size_t i = 0; i < planes.size(); ++i) { - // Single-buffer input format may have multiple color planes, so bytesUsed of the single - // buffer should be sum of each color planes' size. - size_t bytesUsed = 0; - if (planes.size() == 1) { - bytesUsed = media::VideoFrame::AllocationSize(format, mInputLayout->coded_size()); - } else { - bytesUsed = ::base::checked_cast<size_t>( - media::VideoFrame::PlaneSize(format, i, mInputLayout->coded_size()).GetArea()); - } - - // TODO(crbug.com/901264): The way to pass an offset within a DMA-buf is not defined - // in V4L2 specification, so we abuse data_offset for now. Fix it when we have the - // right interface, including any necessary validation and potential alignment. - buffer->SetPlaneDataOffset(i, planes[i].mOffset); - bytesUsed += planes[i].mOffset; - // Workaround: filling length should not be needed. This is a bug of videobuf2 library. - buffer->SetPlaneSize(i, mInputLayout->planes()[i].size + planes[i].mOffset); - buffer->SetPlaneBytesUsed(i, bytesUsed); - } - - std::move(*buffer).QueueDMABuf(frame->getFDs()); - - ALOGV("Queued buffer in input queue (index: %" PRId64 ", timestamp: %" PRId64 - ", bufferId: %zu)", - index, timestamp, bufferId); - - mInputBuffersMap[bufferId] = {index, std::move(frame)}; - - return true; -} - -bool V4L2EncodeComponent::enqueueOutputBuffer() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mOutputQueue->FreeBuffersCount() > 0); - - auto buffer = mOutputQueue->GetFreeBuffer(); - if (!buffer) { - ALOGE("Failed to get free buffer from device output queue"); - reportError(C2_CORRUPTED); - return false; - } - - std::shared_ptr<C2LinearBlock> outputBlock = fetchOutputBlock(); - if (!outputBlock) { - ALOGE("Failed to fetch output block"); - reportError(C2_CORRUPTED); - return false; - } - - size_t bufferId = buffer->BufferId(); - - std::vector<int> fds; - fds.push_back(outputBlock->handle()->data[0]); - if (!std::move(*buffer).QueueDMABuf(fds)) { - ALOGE("Failed to queue output buffer using QueueDMABuf"); - reportError(C2_CORRUPTED); - return false; - } - - ALOG_ASSERT(!mOutputBuffersMap[bufferId]); - mOutputBuffersMap[bufferId] = std::move(outputBlock); - ALOGV("%s(): Queued buffer in output queue (bufferId: %zu)", __func__, bufferId); - return true; -} - -bool V4L2EncodeComponent::dequeueInputBuffer() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED); - ALOG_ASSERT(mInputQueue->QueuedBuffersCount() > 0); - - std::pair<bool, media::V4L2ReadableBufferRef> result = mInputQueue->DequeueBuffer(); - if (!result.first) { - ALOGE("Failed to dequeue buffer from input queue"); - reportError(C2_CORRUPTED); - return false; - } - if (!result.second) { - // No more buffers ready to be dequeued in input queue. - return false; - } - - const media::V4L2ReadableBufferRef buffer = std::move(result.second); - uint64_t index = mInputBuffersMap[buffer->BufferId()].first; - int64_t timestamp = buffer->GetTimeStamp().tv_usec + - buffer->GetTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond; - ALOGV("Dequeued buffer from input queue (index: %" PRId64 ", timestamp: %" PRId64 - ", bufferId: %zu)", - index, timestamp, buffer->BufferId()); - - mInputBuffersMap[buffer->BufferId()].second = nullptr; - onInputBufferDone(index); - - return true; -} - -bool V4L2EncodeComponent::dequeueOutputBuffer() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED); - ALOG_ASSERT(mOutputQueue->QueuedBuffersCount() > 0); - - std::pair<bool, media::V4L2ReadableBufferRef> result = mOutputQueue->DequeueBuffer(); - if (!result.first) { - ALOGE("Failed to dequeue buffer from output queue"); - reportError(C2_CORRUPTED); - return false; - } - if (!result.second) { - // No more buffers ready to be dequeued in output queue. - return false; - } - - media::V4L2ReadableBufferRef buffer = std::move(result.second); - size_t encodedDataSize = buffer->GetPlaneBytesUsed(0) - buffer->GetPlaneDataOffset(0); - ::base::TimeDelta timestamp = ::base::TimeDelta::FromMicroseconds( - buffer->GetTimeStamp().tv_usec + - buffer->GetTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond); - - ALOGV("Dequeued buffer from output queue (timestamp: %" PRId64 - ", bufferId: %zu, data size: %zu, EOS: %d)", - timestamp.InMicroseconds(), buffer->BufferId(), encodedDataSize, buffer->IsLast()); - - if (!mOutputBuffersMap[buffer->BufferId()]) { - ALOGE("Failed to find output block associated with output buffer"); - reportError(C2_CORRUPTED); - return false; - } - - std::shared_ptr<C2LinearBlock> block = std::move(mOutputBuffersMap[buffer->BufferId()]); - if (encodedDataSize > 0) { - onOutputBufferDone(encodedDataSize, buffer->IsKeyframe(), timestamp.InMicroseconds(), - std::move(block)); - } - - // If the buffer is marked as last and we were flushing the encoder, flushing is now done. - if ((mEncoderState == EncoderState::DRAINING) && buffer->IsLast()) { - onDrainDone(true); - - // Start the encoder again. - struct v4l2_encoder_cmd cmd; - memset(&cmd, 0, sizeof(v4l2_encoder_cmd)); - cmd.cmd = V4L2_ENC_CMD_START; - if (mDevice->Ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) { - ALOGE("Failed to restart encoder after flushing (V4L2_ENC_CMD_START)"); - reportError(C2_CORRUPTED); - return false; - } - } - - // Queue a new output buffer to replace the one we dequeued. - buffer = nullptr; - enqueueOutputBuffer(); - - return true; -} - -bool V4L2EncodeComponent::createInputBuffers() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(!mInputQueue->IsStreaming()); - ALOG_ASSERT(mInputBuffersMap.empty()); - - // No memory is allocated here, we just generate a list of buffers on the input queue, which - // will hold memory handles to the real buffers. - if (mInputQueue->AllocateBuffers(kInputBufferCount, V4L2_MEMORY_DMABUF) < kInputBufferCount) { - ALOGE("Failed to create V4L2 input buffers."); - return false; - } - - mInputBuffersMap.resize(mInputQueue->AllocatedBuffersCount()); - return true; -} - -bool V4L2EncodeComponent::createOutputBuffers() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(!mOutputQueue->IsStreaming()); - ALOG_ASSERT(mOutputBuffersMap.empty()); - - // Fetch the output block pool. +bool V4L2EncodeComponent::getBlockPool() { C2BlockPool::local_id_t poolId = mInterface->getBlockPoolId(); + if (poolId == C2BlockPool::BASIC_LINEAR) { + ALOGW("Using unoptimized linear block pool"); + } c2_status_t status = GetCodec2BlockPool(poolId, shared_from_this(), &mOutputBlockPool); if (status != C2_OK || !mOutputBlockPool) { ALOGE("Failed to get output block pool, error: %d", status); return false; } - - // No memory is allocated here, we just generate a list of buffers on the output queue, which - // will hold memory handles to the real buffers. - if (mOutputQueue->AllocateBuffers(kOutputBufferCount, V4L2_MEMORY_DMABUF) < - kOutputBufferCount) { - ALOGE("Failed to create V4L2 output buffers."); - return false; - } - - mOutputBuffersMap.resize(mOutputQueue->AllocatedBuffersCount()); return true; } -void V4L2EncodeComponent::destroyInputBuffers() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(!mInputQueue->IsStreaming()); - - if (!mInputQueue || mInputQueue->AllocatedBuffersCount() == 0) return; - mInputQueue->DeallocateBuffers(); - mInputBuffersMap.clear(); -} - -void V4L2EncodeComponent::destroyOutputBuffers() { - ALOGV("%s()", __func__); - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - ALOG_ASSERT(!mOutputQueue->IsStreaming()); - - if (!mOutputQueue || mOutputQueue->AllocatedBuffersCount() == 0) return; - mOutputQueue->DeallocateBuffers(); - mOutputBuffersMap.clear(); - mOutputBlockPool.reset(); -} - void V4L2EncodeComponent::reportError(c2_status_t error) { ALOGV("%s()", __func__); ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - { - std::lock_guard<std::mutex> lock(mComponentLock); - setComponentState(ComponentState::ERROR); - } - // TODO(dstaessens): Report all pending work items as finished upon failure. - if (mEncoderState != EncoderState::ERROR) { - setEncoderState(EncoderState::ERROR); + std::lock_guard<std::mutex> lock(mComponentLock); + if (mComponentState != ComponentState::ERROR) { + setComponentState(ComponentState::ERROR); mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error)); } } @@ -1708,39 +1044,6 @@ void V4L2EncodeComponent::setComponentState(ComponentState state) { mComponentState = state; } -void V4L2EncodeComponent::setEncoderState(EncoderState state) { - ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence()); - - // Check whether the state change is valid. - switch (state) { - case EncoderState::UNINITIALIZED: - // TODO(dstaessens): Check all valid state changes. - break; - case EncoderState::WAITING_FOR_INPUT: - ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED || - mEncoderState == EncoderState::ENCODING || - mEncoderState == EncoderState::DRAINING); - break; - case EncoderState::WAITING_FOR_INPUT_BUFFERS: - ALOG_ASSERT(mEncoderState == EncoderState::ENCODING); - break; - case EncoderState::ENCODING: - ALOG_ASSERT(mEncoderState == EncoderState::WAITING_FOR_INPUT || - mEncoderState == EncoderState::WAITING_FOR_INPUT_BUFFERS || - mEncoderState == EncoderState::DRAINING); - break; - case EncoderState::DRAINING: - ALOG_ASSERT(mEncoderState == EncoderState::ENCODING); - break; - case EncoderState::ERROR: - break; - } - - ALOGV("Changed encoder state from %s to %s", encoderStateToString(mEncoderState), - encoderStateToString(state)); - mEncoderState = state; -} - const char* V4L2EncodeComponent::componentStateToString(V4L2EncodeComponent::ComponentState state) { switch (state) { case ComponentState::UNLOADED: @@ -1754,21 +1057,4 @@ const char* V4L2EncodeComponent::componentStateToString(V4L2EncodeComponent::Com } } -const char* V4L2EncodeComponent::encoderStateToString(V4L2EncodeComponent::EncoderState state) { - switch (state) { - case EncoderState::UNINITIALIZED: - return "UNINITIALIZED"; - case EncoderState::WAITING_FOR_INPUT: - return "WAITING_FOR_INPUT"; - case EncoderState::WAITING_FOR_INPUT_BUFFERS: - return "WAITING_FOR_INPUT_BUFFERS"; - case EncoderState::ENCODING: - return "ENCODING"; - case EncoderState::DRAINING: - return "Draining"; - case EncoderState::ERROR: - return "ERROR"; - } -} - } // namespace android diff --git a/components/V4L2EncodeInterface.cpp b/components/V4L2EncodeInterface.cpp index 9e6b556..7f0fb39 100644 --- a/components/V4L2EncodeInterface.cpp +++ b/components/V4L2EncodeInterface.cpp @@ -8,6 +8,7 @@ #include <v4l2_codec2/components/V4L2EncodeInterface.h> #include <inttypes.h> +#include <algorithm> #include <C2PlatformSupport.h> #include <SimpleC2Interface.h> @@ -15,9 +16,9 @@ #include <media/stagefright/MediaDefs.h> #include <utils/Log.h> -#include <v4l2_device.h> #include <v4l2_codec2/common/V4L2ComponentCommon.h> -#include <video_codecs.h> +#include <v4l2_codec2/common/V4L2Device.h> +#include <v4l2_codec2/common/VideoTypes.h> using android::hardware::graphics::common::V1_0::BufferUsage; @@ -41,76 +42,50 @@ constexpr uint32_t kDefaultBitrate = 64000; // TODO: increase this in the future for supporting higher level/resolution encoding. constexpr uint32_t kMaxBitrate = 50000000; -// The frame size of 1080p video. -constexpr uint32_t kFrameSize1080P = 1920 * 1080; - -C2Config::profile_t videoCodecProfileToC2Profile(media::VideoCodecProfile profile) { - switch (profile) { - case media::VideoCodecProfile::H264PROFILE_BASELINE: - return C2Config::PROFILE_AVC_BASELINE; - case media::VideoCodecProfile::H264PROFILE_MAIN: - return C2Config::PROFILE_AVC_MAIN; - case media::VideoCodecProfile::H264PROFILE_EXTENDED: - return C2Config::PROFILE_AVC_EXTENDED; - case media::VideoCodecProfile::H264PROFILE_HIGH: - return C2Config::PROFILE_AVC_HIGH; - case media::VideoCodecProfile::H264PROFILE_HIGH10PROFILE: - return C2Config::PROFILE_AVC_HIGH_10; - case media::VideoCodecProfile::H264PROFILE_HIGH422PROFILE: - return C2Config::PROFILE_AVC_HIGH_422; - case media::VideoCodecProfile::H264PROFILE_HIGH444PREDICTIVEPROFILE: - return C2Config::PROFILE_AVC_HIGH_444_PREDICTIVE; - case media::VideoCodecProfile::H264PROFILE_SCALABLEBASELINE: - return C2Config::PROFILE_AVC_SCALABLE_BASELINE; - case media::VideoCodecProfile::H264PROFILE_SCALABLEHIGH: - return C2Config::PROFILE_AVC_SCALABLE_HIGH; - case media::VideoCodecProfile::H264PROFILE_STEREOHIGH: - return C2Config::PROFILE_AVC_STEREO_HIGH; - case media::VideoCodecProfile::H264PROFILE_MULTIVIEWHIGH: - return C2Config::PROFILE_AVC_MULTIVIEW_HIGH; - default: - ALOGE("Unrecognizable profile (value = %d)...", profile); - return C2Config::PROFILE_UNUSED; - } -} - -std::optional<media::VideoCodec> getCodecFromComponentName(const std::string& name) { - if (name == V4L2ComponentName::kH264Encoder) - return media::VideoCodec::kCodecH264; +std::optional<VideoCodec> getCodecFromComponentName(const std::string& name) { + if (name == V4L2ComponentName::kH264Encoder) return VideoCodec::H264; + if (name == V4L2ComponentName::kVP8Encoder) return VideoCodec::VP8; + if (name == V4L2ComponentName::kVP9Encoder) return VideoCodec::VP9; ALOGE("Unknown name: %s", name.c_str()); return std::nullopt; } +// Check whether the specified profile is a valid profile for the specified codec. +bool IsValidProfileForCodec(VideoCodec codec, C2Config::profile_t profile) { + switch (codec) { + case VideoCodec::H264: + return ((profile >= C2Config::PROFILE_AVC_BASELINE) && + (profile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH)); + case VideoCodec::VP8: + return ((profile >= C2Config::PROFILE_VP8_0) && (profile <= C2Config::PROFILE_VP8_3)); + case VideoCodec::VP9: + return ((profile >= C2Config::PROFILE_VP9_0) && (profile <= C2Config::PROFILE_VP9_3)); + default: + return false; + } +} + } // namespace // static -C2R V4L2EncodeInterface::ProfileLevelSetter(bool mayBlock, - C2P<C2StreamProfileLevelInfo::output>& info, - const C2P<C2StreamPictureSizeInfo::input>& videoSize, - const C2P<C2StreamFrameRateInfo::output>& frameRate, - const C2P<C2StreamBitrateInfo::output>& bitrate) { - (void)mayBlock; - +C2R V4L2EncodeInterface::H264ProfileLevelSetter( + bool /*mayBlock*/, C2P<C2StreamProfileLevelInfo::output>& info, + const C2P<C2StreamPictureSizeInfo::input>& videoSize, + const C2P<C2StreamFrameRateInfo::output>& frameRate, + const C2P<C2StreamBitrateInfo::output>& bitrate) { static C2Config::level_t lowestConfigLevel = C2Config::LEVEL_UNUSED; - // Use at least PROFILE_AVC_MAIN as default for 1080p input video and up. - // TODO (b/114332827): Find root cause of bad quality of Baseline encoding. - C2Config::profile_t defaultMinProfile = C2Config::PROFILE_AVC_BASELINE; - if (videoSize.v.width * videoSize.v.height >= kFrameSize1080P) { - defaultMinProfile = C2Config::PROFILE_AVC_MAIN; - } - // Adopt default minimal profile instead if the requested profile is not supported, or lower // than the default minimal one. - if (!info.F(info.v.profile).supportsAtAll(info.v.profile) || - info.v.profile < defaultMinProfile) { - if (info.F(info.v.profile).supportsAtAll(defaultMinProfile)) { - ALOGV("Set profile to default (%u) instead.", defaultMinProfile); - info.set().profile = defaultMinProfile; + constexpr C2Config::profile_t minProfile = C2Config::PROFILE_AVC_BASELINE; + if (!info.F(info.v.profile).supportsAtAll(info.v.profile) || info.v.profile < minProfile) { + if (info.F(info.v.profile).supportsAtAll(minProfile)) { + ALOGV("Set profile to default (%u) instead.", minProfile); + info.set().profile = minProfile; } else { ALOGE("Unable to set either requested profile (%u) or default profile (%u).", - info.v.profile, defaultMinProfile); + info.v.profile, minProfile); return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile))); } } @@ -210,6 +185,29 @@ C2R V4L2EncodeInterface::ProfileLevelSetter(bool mayBlock, return C2R::Ok(); } +C2R V4L2EncodeInterface::VP9ProfileLevelSetter( + bool /*mayBlock*/, C2P<C2StreamProfileLevelInfo::output>& info, + const C2P<C2StreamPictureSizeInfo::input>& /*videoSize*/, + const C2P<C2StreamFrameRateInfo::output>& /*frameRate*/, + const C2P<C2StreamBitrateInfo::output>& /*bitrate*/) { + // Adopt default minimal profile instead if the requested profile is not supported, or lower + // than the default minimal one. + constexpr C2Config::profile_t defaultMinProfile = C2Config::PROFILE_VP9_0; + if (!info.F(info.v.profile).supportsAtAll(info.v.profile) || + info.v.profile < defaultMinProfile) { + if (info.F(info.v.profile).supportsAtAll(defaultMinProfile)) { + ALOGV("Set profile to default (%u) instead.", defaultMinProfile); + info.set().profile = defaultMinProfile; + } else { + ALOGE("Unable to set either requested profile (%u) or default profile (%u).", + info.v.profile, defaultMinProfile); + return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile))); + } + } + + return C2R::Ok(); +} + // static C2R V4L2EncodeInterface::SizeSetter(bool mayBlock, C2P<C2StreamPictureSizeInfo::input>& videoSize) { (void)mayBlock; @@ -233,8 +231,8 @@ C2R V4L2EncodeInterface::IntraRefreshPeriodSetter(bool mayBlock, return C2R::Ok(); } -V4L2EncodeInterface::V4L2EncodeInterface( - const C2String& name, std::shared_ptr<C2ReflectorHelper> helper) +V4L2EncodeInterface::V4L2EncodeInterface(const C2String& name, + std::shared_ptr<C2ReflectorHelper> helper) : C2InterfaceHelper(std::move(helper)) { ALOGV("%s(%s)", __func__, name.c_str()); @@ -244,48 +242,57 @@ V4L2EncodeInterface::V4L2EncodeInterface( } void V4L2EncodeInterface::Initialize(const C2String& name) { - scoped_refptr<media::V4L2Device> device = media::V4L2Device::Create(); + scoped_refptr<V4L2Device> device = V4L2Device::create(); if (!device) { ALOGE("Failed to create V4L2 device"); mInitStatus = C2_CORRUPTED; return; } - // Use type=unsigned int here, otherwise it will cause compile error in - // C2F(mProfileLevel, profile).oneOf(profiles) since std::vector<C2Config::profile_t> cannot - // convert to std::vector<unsigned int>. + auto codec = getCodecFromComponentName(name); + if (!codec) { + ALOGE("Invalid component name"); + mInitStatus = C2_BAD_VALUE; + return; + } + + V4L2Device::SupportedEncodeProfiles supported_profiles = device->getSupportedEncodeProfiles(); + + // Compile the list of supported profiles. + // Note: unsigned int is used here, since std::vector<C2Config::profile_t> cannot convert to + // std::vector<unsigned int> required by the c2 framework below. std::vector<unsigned int> profiles; - media::Size maxSize; - for (const auto& supportedProfile : device->GetSupportedEncodeProfiles()) { - C2Config::profile_t profile = videoCodecProfileToC2Profile(supportedProfile.profile); - if (profile == C2Config::PROFILE_UNUSED) { - continue; // neglect unrecognizable profile + ui::Size maxSize; + for (const auto& supportedProfile : supported_profiles) { + if (!IsValidProfileForCodec(codec.value(), supportedProfile.profile)) { + continue; // Ignore unrecognizable or unsupported profiles. } - ALOGV("Queried c2_profile = 0x%x : max_size = %d x %d", profile, - supportedProfile.max_resolution.width(), supportedProfile.max_resolution.height()); - profiles.push_back(static_cast<unsigned int>(profile)); - maxSize.set_width(std::max(maxSize.width(), supportedProfile.max_resolution.width())); - maxSize.set_height(std::max(maxSize.height(), supportedProfile.max_resolution.height())); + ALOGV("Queried c2_profile = 0x%x : max_size = %d x %d", supportedProfile.profile, + supportedProfile.max_resolution.width, supportedProfile.max_resolution.height); + profiles.push_back(static_cast<unsigned int>(supportedProfile.profile)); + maxSize.setWidth(std::max(maxSize.width, supportedProfile.max_resolution.width)); + maxSize.setHeight(std::max(maxSize.height, supportedProfile.max_resolution.height)); } if (profiles.empty()) { - ALOGD("No supported profiles"); + ALOGE("No supported profiles"); mInitStatus = C2_BAD_VALUE; return; } - C2Config::profile_t minProfile = - static_cast<C2Config::profile_t>(*std::min_element(profiles.begin(), profiles.end())); - // Special note: the order of addParameter matters if your setters are dependent on other // parameters. Please make sure the dependent parameters are added prior to the // one needs the setter dependency. + addParameter(DefineParam(mKind, C2_PARAMKEY_COMPONENT_KIND) + .withConstValue(new C2ComponentKindSetting(C2Component::KIND_ENCODER)) + .build()); + addParameter(DefineParam(mInputVisibleSize, C2_PARAMKEY_PICTURE_SIZE) .withDefault(new C2StreamPictureSizeInfo::input(0u, 320, 240)) .withFields({ - C2F(mInputVisibleSize, width).inRange(2, maxSize.width(), 2), - C2F(mInputVisibleSize, height).inRange(2, maxSize.height(), 2), + C2F(mInputVisibleSize, width).inRange(2, maxSize.width, 2), + C2F(mInputVisibleSize, height).inRange(2, maxSize.height, 2), }) .withSetter(SizeSetter) .build()); @@ -304,8 +311,10 @@ void V4L2EncodeInterface::Initialize(const C2String& name) { .build()); std::string outputMime; - if (getCodecFromComponentName(name) == media::VideoCodec::kCodecH264) { + if (getCodecFromComponentName(name) == VideoCodec::H264) { outputMime = MEDIA_MIMETYPE_VIDEO_AVC; + C2Config::profile_t minProfile = static_cast<C2Config::profile_t>( + *std::min_element(profiles.begin(), profiles.end())); addParameter( DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile, @@ -322,10 +331,38 @@ void V4L2EncodeInterface::Initialize(const C2String& name) { C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4, C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1})}) - .withSetter(ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate) + .withSetter(H264ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate) + .build()); + } else if (getCodecFromComponentName(name) == VideoCodec::VP8) { + outputMime = MEDIA_MIMETYPE_VIDEO_VP8; + // VP8 doesn't have conventional profiles, we'll use profile0 if the VP8 codec is requested. + addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withConstValue(new C2StreamProfileLevelInfo::output( + 0u, C2Config::PROFILE_VP8_0, C2Config::LEVEL_UNUSED)) + .build()); + } else if (getCodecFromComponentName(name) == VideoCodec::VP9) { + outputMime = MEDIA_MIMETYPE_VIDEO_VP9; + C2Config::profile_t minProfile = static_cast<C2Config::profile_t>( + *std::min_element(profiles.begin(), profiles.end())); + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile, + C2Config::LEVEL_VP9_1)) + .withFields( + {C2F(mProfileLevel, profile).oneOf(profiles), + C2F(mProfileLevel, level) + // TODO(dstaessens) query supported levels from adaptor. + .oneOf({C2Config::LEVEL_VP9_1, C2Config::LEVEL_VP9_1_1, + C2Config::LEVEL_VP9_2, C2Config::LEVEL_VP9_2_1, + C2Config::LEVEL_VP9_3, C2Config::LEVEL_VP9_3_1, + C2Config::LEVEL_VP9_4, C2Config::LEVEL_VP9_4_1, + C2Config::LEVEL_VP9_5, C2Config::LEVEL_VP9_5_1, + C2Config::LEVEL_VP9_5_2, C2Config::LEVEL_VP9_6, + C2Config::LEVEL_VP9_6_1, + C2Config::LEVEL_VP9_6_2})}) + .withSetter(VP9ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate) .build()); } else { - // TODO(johnylin): implement VP8/VP9 encoder in the future. ALOGE("Unsupported component name: %s", name.c_str()); mInitStatus = C2_BAD_VALUE; return; diff --git a/components/V4L2Encoder.cpp b/components/V4L2Encoder.cpp new file mode 100644 index 0000000..cdd2d89 --- /dev/null +++ b/components/V4L2Encoder.cpp @@ -0,0 +1,1016 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#define LOG_NDEBUG 0 +#define LOG_TAG "V4L2Encoder" + +#include <v4l2_codec2/components/V4L2Encoder.h> + +#include <stdint.h> +#include <optional> +#include <vector> + +#include <base/bind.h> +#include <base/files/scoped_file.h> +#include <base/memory/ptr_util.h> +#include <log/log.h> +#include <ui/Rect.h> + +#include <v4l2_codec2/common/Fourcc.h> +#include <v4l2_codec2/common/V4L2Device.h> +#include <v4l2_codec2/components/BitstreamBuffer.h> + +namespace android { + +namespace { + +const VideoPixelFormat kInputPixelFormat = VideoPixelFormat::NV12; + +// The maximum size for output buffer, which is chosen empirically for a 1080p video. +constexpr size_t kMaxBitstreamBufferSizeInBytes = 2 * 1024 * 1024; // 2MB +// The frame size for 1080p (FHD) video in pixels. +constexpr int k1080PSizeInPixels = 1920 * 1080; +// The frame size for 1440p (QHD) video in pixels. +constexpr int k1440PSizeInPixels = 2560 * 1440; + +// Use quadruple size of kMaxBitstreamBufferSizeInBytes when the input frame size is larger than +// 1440p, double if larger than 1080p. This is chosen empirically for some 4k encoding use cases and +// the Android CTS VideoEncoderTest (crbug.com/927284). +size_t GetMaxOutputBufferSize(const ui::Size& size) { + if (getArea(size) > k1440PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 4; + if (getArea(size) > k1080PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 2; + return kMaxBitstreamBufferSizeInBytes; +} + +// Define V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR control code if not present in header files. +#ifndef V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR +#define V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644) +#endif + +} // namespace + +// static +std::unique_ptr<VideoEncoder> V4L2Encoder::create( + C2Config::profile_t outputProfile, std::optional<uint8_t> level, + const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod, + FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB inputBufferDoneCb, + OutputBufferDoneCB outputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb, + scoped_refptr<::base::SequencedTaskRunner> taskRunner) { + ALOGV("%s()", __func__); + + std::unique_ptr<V4L2Encoder> encoder = ::base::WrapUnique<V4L2Encoder>(new V4L2Encoder( + std::move(taskRunner), std::move(fetchOutputBufferCb), std::move(inputBufferDoneCb), + std::move(outputBufferDoneCb), std::move(drainDoneCb), std::move(errorCb))); + if (!encoder->initialize(outputProfile, level, visibleSize, stride, keyFramePeriod)) { + return nullptr; + } + return encoder; +} + +V4L2Encoder::V4L2Encoder(scoped_refptr<::base::SequencedTaskRunner> taskRunner, + FetchOutputBufferCB fetchOutputBufferCb, + InputBufferDoneCB inputBufferDoneCb, OutputBufferDoneCB outputBufferDoneCb, + DrainDoneCB drainDoneCb, ErrorCB errorCb) + : mFetchOutputBufferCb(fetchOutputBufferCb), + mInputBufferDoneCb(inputBufferDoneCb), + mOutputBufferDoneCb(outputBufferDoneCb), + mDrainDoneCb(std::move(drainDoneCb)), + mErrorCb(std::move(errorCb)), + mTaskRunner(std::move(taskRunner)) { + ALOGV("%s()", __func__); + + mWeakThis = mWeakThisFactory.GetWeakPtr(); +} + +V4L2Encoder::~V4L2Encoder() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + mWeakThisFactory.InvalidateWeakPtrs(); + + // Flushing the encoder will stop polling and streaming on the V4L2 device queues. + flush(); + + // Deallocate all V4L2 device input and output buffers. + destroyInputBuffers(); + destroyOutputBuffers(); +} + +bool V4L2Encoder::encode(std::unique_ptr<InputFrame> frame) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState != State::UNINITIALIZED); + + // If we're in the error state we can immediately return, freeing the input buffer. + if (mState == State::ERROR) { + return false; + } + + if (!frame) { + ALOGW("Empty encode request scheduled"); + return false; + } + + mEncodeRequests.push(EncodeRequest(std::move(frame))); + + // If we were waiting for encode requests, start encoding again. + if (mState == State::WAITING_FOR_INPUT_FRAME) { + setState(State::ENCODING); + mTaskRunner->PostTask(FROM_HERE, + ::base::BindOnce(&V4L2Encoder::handleEncodeRequest, mWeakThis)); + } + + return true; +} + +void V4L2Encoder::drain() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + // We can only start draining if all the requests in our input queue has been queued on the V4L2 + // device input queue, so we mark the last item in the input queue as EOS. + if (!mEncodeRequests.empty()) { + ALOGV("Marking last item (index: %" PRIu64 ") in encode request queue as EOS", + mEncodeRequests.back().video_frame->index()); + mEncodeRequests.back().end_of_stream = true; + return; + } + + // Start a drain operation on the device. If no buffers are currently queued the device will + // return an empty buffer with the V4L2_BUF_FLAG_LAST flag set. + handleDrainRequest(); +} + +void V4L2Encoder::flush() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + handleFlushRequest(); +} + +bool V4L2Encoder::setBitrate(uint32_t bitrate) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE, bitrate)})) { + ALOGE("Setting bitrate to %u failed", bitrate); + return false; + } + return true; +} + +bool V4L2Encoder::setFramerate(uint32_t framerate) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + struct v4l2_streamparm parms; + memset(&parms, 0, sizeof(v4l2_streamparm)); + parms.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + parms.parm.output.timeperframe.numerator = 1; + parms.parm.output.timeperframe.denominator = framerate; + if (mDevice->ioctl(VIDIOC_S_PARM, &parms) != 0) { + ALOGE("Setting framerate to %u failed", framerate); + return false; + } + return true; +} + +void V4L2Encoder::requestKeyframe() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + mKeyFrameCounter = 0; +} + +VideoPixelFormat V4L2Encoder::inputFormat() const { + return mInputLayout ? mInputLayout.value().mFormat : VideoPixelFormat::UNKNOWN; +} + +bool V4L2Encoder::initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level, + const ui::Size& visibleSize, uint32_t stride, + uint32_t keyFramePeriod) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(keyFramePeriod > 0); + + mVisibleSize = visibleSize; + mKeyFramePeriod = keyFramePeriod; + mKeyFrameCounter = 0; + + // Open the V4L2 device for encoding to the requested output format. + // TODO(dstaessens): Avoid conversion to VideoCodecProfile and use C2Config::profile_t directly. + uint32_t outputPixelFormat = V4L2Device::C2ProfileToV4L2PixFmt(outputProfile, false); + if (!outputPixelFormat) { + ALOGE("Invalid output profile %s", profileToString(outputProfile)); + return false; + } + + mDevice = V4L2Device::create(); + if (!mDevice) { + ALOGE("Failed to create V4L2 device"); + return false; + } + + if (!mDevice->open(V4L2Device::Type::kEncoder, outputPixelFormat)) { + ALOGE("Failed to open device for profile %s (%s)", profileToString(outputProfile), + fourccToString(outputPixelFormat).c_str()); + return false; + } + + // Make sure the device has all required capabilities (multi-planar Memory-To-Memory and + // streaming I/O), and whether flushing is supported. + if (!mDevice->hasCapabilities(V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING)) { + ALOGE("Device doesn't have the required capabilities"); + return false; + } + if (!mDevice->isCommandSupported(V4L2_ENC_CMD_STOP)) { + ALOGE("Device does not support flushing (V4L2_ENC_CMD_STOP)"); + return false; + } + + // Get input/output queues so we can send encode request to the device and get back the results. + mInputQueue = mDevice->getQueue(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + mOutputQueue = mDevice->getQueue(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + if (!mInputQueue || !mOutputQueue) { + ALOGE("Failed to get V4L2 device queues"); + return false; + } + + // First try to configure the specified output format, as changing the output format can affect + // the configured input format. + if (!configureOutputFormat(outputProfile)) return false; + + // Configure the input format. If the device doesn't support the specified format we'll use one + // of the device's preferred formats in combination with an input format convertor. + if (!configureInputFormat(kInputPixelFormat, stride)) return false; + + // Create input and output buffers. + if (!createInputBuffers() || !createOutputBuffers()) return false; + + // Configure the device, setting all required controls. + if (!configureDevice(outputProfile, level)) return false; + + // We're ready to start encoding now. + setState(State::WAITING_FOR_INPUT_FRAME); + return true; +} + +void V4L2Encoder::handleEncodeRequest() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState == State::ENCODING || mState == State::ERROR); + + // If we're in the error state we can immediately return. + if (mState == State::ERROR) { + return; + } + + // It's possible we flushed the encoder since this function was scheduled. + if (mEncodeRequests.empty()) { + return; + } + + // Get the next encode request from the queue. + EncodeRequest& encodeRequest = mEncodeRequests.front(); + + // Check if the device has free input buffers available. If not we'll switch to the + // WAITING_FOR_INPUT_BUFFERS state, and resume encoding once we've dequeued an input buffer. + // Note: The input buffers are not copied into the device's input buffers, but rather a memory + // pointer is imported. We still have to throttle the number of enqueues queued simultaneously + // on the device however. + if (mInputQueue->freeBuffersCount() == 0) { + ALOGV("Waiting for device to return input buffers"); + setState(State::WAITING_FOR_V4L2_BUFFER); + return; + } + + // Request the next frame to be a key frame each time the counter reaches 0. + if (mKeyFrameCounter == 0) { + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME)})) { + ALOGE("Failed requesting key frame"); + onError(); + return; + } + } + mKeyFrameCounter = (mKeyFrameCounter + 1) % mKeyFramePeriod; + + // Enqueue the input frame in the V4L2 device. + uint64_t index = encodeRequest.video_frame->index(); + uint64_t timestamp = encodeRequest.video_frame->timestamp(); + bool end_of_stream = encodeRequest.end_of_stream; + if (!enqueueInputBuffer(std::move(encodeRequest.video_frame))) { + ALOGE("Failed to enqueue input frame (index: %" PRIu64 ", timestamp: %" PRId64 ")", index, + timestamp); + onError(); + return; + } + mEncodeRequests.pop(); + + // Start streaming and polling on the input and output queue if required. + if (!mInputQueue->isStreaming()) { + ALOG_ASSERT(!mOutputQueue->isStreaming()); + if (!mOutputQueue->streamon() || !mInputQueue->streamon()) { + ALOGE("Failed to start streaming on input and output queue"); + onError(); + return; + } + startDevicePoll(); + } + + // Queue buffers on output queue. These buffers will be used to store the encoded bitstream. + while (mOutputQueue->freeBuffersCount() > 0) { + if (!enqueueOutputBuffer()) return; + } + + // Drain the encoder if requested. + if (end_of_stream) { + handleDrainRequest(); + return; + } + + if (mEncodeRequests.empty()) { + setState(State::WAITING_FOR_INPUT_FRAME); + return; + } + + // Schedule the next buffer to be encoded. + mTaskRunner->PostTask(FROM_HERE, + ::base::BindOnce(&V4L2Encoder::handleEncodeRequest, mWeakThis)); +} + +void V4L2Encoder::handleFlushRequest() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + // Stop the device poll thread. + stopDevicePoll(); + + // Stop streaming on the V4L2 device, which stops all currently queued encode operations and + // releases all buffers currently in use by the device. + for (auto& queue : {mInputQueue, mOutputQueue}) { + if (queue && queue->isStreaming() && !queue->streamoff()) { + ALOGE("Failed to stop streaming on the device queue"); + onError(); + } + } + + // Clear all outstanding encode requests and references to input and output queue buffers. + while (!mEncodeRequests.empty()) { + mEncodeRequests.pop(); + } + for (auto& buf : mInputBuffers) { + buf = nullptr; + } + for (auto& buf : mOutputBuffers) { + buf = nullptr; + } + + // Streaming and polling on the V4L2 device input and output queues will be resumed once new + // encode work is queued. + if (mState != State::ERROR) { + setState(State::WAITING_FOR_INPUT_FRAME); + } +} + +void V4L2Encoder::handleDrainRequest() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (mState == State::DRAINING || mState == State::ERROR) { + return; + } + + setState(State::DRAINING); + + // If we're not streaming we can consider the request completed immediately. + if (!mInputQueue->isStreaming()) { + onDrainDone(true); + return; + } + + struct v4l2_encoder_cmd cmd; + memset(&cmd, 0, sizeof(v4l2_encoder_cmd)); + cmd.cmd = V4L2_ENC_CMD_STOP; + if (mDevice->ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) { + ALOGE("Failed to stop encoder"); + onDrainDone(false); + return; + } + ALOGV("%s(): Sent STOP command to encoder", __func__); +} + +void V4L2Encoder::onDrainDone(bool done) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState == State::DRAINING || mState == State::ERROR); + + if (mState == State::ERROR) { + return; + } + + if (!done) { + ALOGE("draining the encoder failed"); + mDrainDoneCb.Run(false); + onError(); + return; + } + + ALOGV("Draining done"); + mDrainDoneCb.Run(true); + + // Draining the encoder is done, we can now start encoding again. + if (!mEncodeRequests.empty()) { + setState(State::ENCODING); + mTaskRunner->PostTask(FROM_HERE, + ::base::BindOnce(&V4L2Encoder::handleEncodeRequest, mWeakThis)); + } else { + setState(State::WAITING_FOR_INPUT_FRAME); + } +} + +bool V4L2Encoder::configureInputFormat(VideoPixelFormat inputFormat, uint32_t stride) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState == State::UNINITIALIZED); + ALOG_ASSERT(!mInputQueue->isStreaming()); + ALOG_ASSERT(!isEmpty(mVisibleSize)); + + // First try to use the requested pixel format directly. + std::optional<struct v4l2_format> format; + auto fourcc = Fourcc::fromVideoPixelFormat(inputFormat, false); + if (fourcc) { + format = mInputQueue->setFormat(fourcc->toV4L2PixFmt(), mVisibleSize, 0, stride); + } + + // If the device doesn't support the requested input format we'll try the device's preferred + // input pixel formats and use a format convertor. We need to try all formats as some formats + // might not be supported for the configured output format. + if (!format) { + std::vector<uint32_t> preferredFormats = + mDevice->preferredInputFormat(V4L2Device::Type::kEncoder); + for (uint32_t i = 0; !format && i < preferredFormats.size(); ++i) { + format = mInputQueue->setFormat(preferredFormats[i], mVisibleSize, 0, stride); + } + } + + if (!format) { + ALOGE("Failed to set input format to %s", videoPixelFormatToString(inputFormat).c_str()); + return false; + } + + // Check whether the negotiated input format is valid. The coded size might be adjusted to match + // encoder minimums, maximums and alignment requirements of the currently selected formats. + auto layout = V4L2Device::v4L2FormatToVideoFrameLayout(*format); + if (!layout) { + ALOGE("Invalid input layout"); + return false; + } + + mInputLayout = layout.value(); + if (!contains(Rect(mInputLayout->mCodedSize.width, mInputLayout->mCodedSize.height), + Rect(mVisibleSize.width, mVisibleSize.height))) { + ALOGE("Input size %s exceeds encoder capability, encoder can handle %s", + toString(mVisibleSize).c_str(), toString(mInputLayout->mCodedSize).c_str()); + return false; + } + + // Calculate the input coded size from the format. + // TODO(dstaessens): How is this different from mInputLayout->coded_size()? + mInputCodedSize = V4L2Device::allocatedSizeFromV4L2Format(*format); + + // Configuring the input format might cause the output buffer size to change. + auto outputFormat = mOutputQueue->getFormat(); + if (!outputFormat.first) { + ALOGE("Failed to get output format (errno: %i)", outputFormat.second); + return false; + } + uint32_t AdjustedOutputBufferSize = outputFormat.first->fmt.pix_mp.plane_fmt[0].sizeimage; + if (mOutputBufferSize != AdjustedOutputBufferSize) { + mOutputBufferSize = AdjustedOutputBufferSize; + ALOGV("Output buffer size adjusted to: %u", mOutputBufferSize); + } + + // The coded input size might be different from the visible size due to alignment requirements, + // So we need to specify the visible rectangle. Note that this rectangle might still be adjusted + // due to hardware limitations. + Rect visibleRectangle(mVisibleSize.width, mVisibleSize.height); + + struct v4l2_rect rect; + memset(&rect, 0, sizeof(rect)); + rect.left = visibleRectangle.left; + rect.top = visibleRectangle.top; + rect.width = visibleRectangle.width(); + rect.height = visibleRectangle.height(); + + // Try to adjust the visible rectangle using the VIDIOC_S_SELECTION command. If this is not + // supported we'll try to use the VIDIOC_S_CROP command instead. The visible rectangle might be + // adjusted to conform to hardware limitations (e.g. round to closest horizontal and vertical + // offsets, width and height). + struct v4l2_selection selection_arg; + memset(&selection_arg, 0, sizeof(selection_arg)); + selection_arg.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + selection_arg.target = V4L2_SEL_TGT_CROP; + selection_arg.r = rect; + if (mDevice->ioctl(VIDIOC_S_SELECTION, &selection_arg) == 0) { + visibleRectangle = Rect(selection_arg.r.left, selection_arg.r.top, + selection_arg.r.left + selection_arg.r.width, + selection_arg.r.top + selection_arg.r.height); + } else { + struct v4l2_crop crop; + memset(&crop, 0, sizeof(v4l2_crop)); + crop.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + crop.c = rect; + if (mDevice->ioctl(VIDIOC_S_CROP, &crop) != 0 || + mDevice->ioctl(VIDIOC_G_CROP, &crop) != 0) { + ALOGE("Failed to crop to specified visible rectangle"); + return false; + } + visibleRectangle = Rect(crop.c.left, crop.c.top, crop.c.left + crop.c.width, + crop.c.top + crop.c.height); + } + + ALOGV("Input format set to %s (size: %s, adjusted size: %dx%d, coded size: %s)", + videoPixelFormatToString(mInputLayout->mFormat).c_str(), toString(mVisibleSize).c_str(), + visibleRectangle.width(), visibleRectangle.height(), toString(mInputCodedSize).c_str()); + + mVisibleSize.set(visibleRectangle.width(), visibleRectangle.height()); + return true; +} + +bool V4L2Encoder::configureOutputFormat(C2Config::profile_t outputProfile) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState == State::UNINITIALIZED); + ALOG_ASSERT(!mOutputQueue->isStreaming()); + ALOG_ASSERT(!isEmpty(mVisibleSize)); + + auto format = mOutputQueue->setFormat(V4L2Device::C2ProfileToV4L2PixFmt(outputProfile, false), + mVisibleSize, GetMaxOutputBufferSize(mVisibleSize)); + if (!format) { + ALOGE("Failed to set output format to %s", profileToString(outputProfile)); + return false; + } + + // The device might adjust the requested output buffer size to match hardware requirements. + mOutputBufferSize = format->fmt.pix_mp.plane_fmt[0].sizeimage; + + ALOGV("Output format set to %s (buffer size: %u)", profileToString(outputProfile), + mOutputBufferSize); + return true; +} + +bool V4L2Encoder::configureDevice(C2Config::profile_t outputProfile, + std::optional<const uint8_t> outputH264Level) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + // Enable frame-level bitrate control. This is the only mandatory general control. + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, 1)})) { + ALOGW("Failed enabling bitrate control"); + // TODO(b/161508368): V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE is currently not supported yet, + // assume the operation was successful for now. + } + + // Additional optional controls: + // - Enable macroblock-level bitrate control. + // - Set GOP length to 0 to disable periodic key frames. + mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE, 1), + V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0)}); + + // All controls below are H.264-specific, so we can return here if the profile is not H.264. + if (outputProfile >= C2Config::PROFILE_AVC_BASELINE || + outputProfile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) { + return configureH264(outputProfile, outputH264Level); + } + + return true; +} + +bool V4L2Encoder::configureH264(C2Config::profile_t outputProfile, + std::optional<const uint8_t> outputH264Level) { + // When encoding H.264 we want to prepend SPS and PPS to each IDR for resilience. Some + // devices support this through the V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR control. + // TODO(b/161495502): V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR is currently not supported + // yet, just log a warning if the operation was unsuccessful for now. + if (mDevice->isCtrlExposed(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR)) { + if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, + {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR, 1)})) { + ALOGE("Failed to configure device to prepend SPS and PPS to each IDR"); + return false; + } + ALOGV("Device supports prepending SPS and PPS to each IDR"); + } else { + ALOGW("Device doesn't support prepending SPS and PPS to IDR"); + } + + std::vector<V4L2ExtCtrl> h264Ctrls; + + // No B-frames, for lowest decoding latency. + h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_B_FRAMES, 0); + // Quantization parameter maximum value (for variable bitrate control). + h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 51); + + // Set H.264 profile. + int32_t profile = V4L2Device::c2ProfileToV4L2H264Profile(outputProfile); + if (profile < 0) { + ALOGE("Trying to set invalid H.264 profile"); + return false; + } + h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_PROFILE, profile); + + // Set H.264 output level. Use Level 4.0 as fallback default. + int32_t h264Level = + static_cast<int32_t>(outputH264Level.value_or(V4L2_MPEG_VIDEO_H264_LEVEL_4_0)); + h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_LEVEL, h264Level); + + // Ask not to put SPS and PPS into separate bitstream buffers. + h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_HEADER_MODE, + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME); + + // Ignore return value as these controls are optional. + mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG, std::move(h264Ctrls)); + + return true; +} + +bool V4L2Encoder::startDevicePoll() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (!mDevice->startPolling(::base::BindRepeating(&V4L2Encoder::serviceDeviceTask, mWeakThis), + ::base::BindRepeating(&V4L2Encoder::onPollError, mWeakThis))) { + ALOGE("Device poll thread failed to start"); + onError(); + return false; + } + + ALOGV("Device poll started"); + return true; +} + +bool V4L2Encoder::stopDevicePoll() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (!mDevice->stopPolling()) { + ALOGE("Failed to stop polling on the device"); + onError(); + return false; + } + + ALOGV("Device poll stopped"); + return true; +} + +void V4L2Encoder::onPollError() { + ALOGV("%s()", __func__); + onError(); +} + +void V4L2Encoder::serviceDeviceTask(bool /*event*/) { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState != State::UNINITIALIZED); + + if (mState == State::ERROR) { + return; + } + + // Dequeue completed input (VIDEO_OUTPUT) buffers, and recycle to the free list. + while (mInputQueue->queuedBuffersCount() > 0) { + if (!dequeueInputBuffer()) break; + } + + // Dequeue completed output (VIDEO_CAPTURE) buffers, and recycle to the free list. + while (mOutputQueue->queuedBuffersCount() > 0) { + if (!dequeueOutputBuffer()) break; + } + + ALOGV("%s() - done", __func__); +} + +bool V4L2Encoder::enqueueInputBuffer(std::unique_ptr<InputFrame> frame) { + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mInputQueue->freeBuffersCount() > 0); + ALOG_ASSERT(mState == State::ENCODING); + ALOG_ASSERT(frame); + ALOG_ASSERT(mInputLayout->mFormat == frame->pixelFormat()); + ALOG_ASSERT(mInputLayout->mPlanes.size() == frame->planes().size()); + + auto format = frame->pixelFormat(); + auto planes = frame->planes(); + auto index = frame->index(); + auto timestamp = frame->timestamp(); + + ALOGV("%s(): queuing input buffer (index: %" PRId64 ")", __func__, index); + + auto buffer = mInputQueue->getFreeBuffer(); + if (!buffer) { + ALOGE("Failed to get free buffer from device input queue"); + return false; + } + + // Mark the buffer with the frame's timestamp so we can identify the associated output buffers. + buffer->setTimeStamp( + {.tv_sec = static_cast<time_t>(timestamp / ::base::Time::kMicrosecondsPerSecond), + .tv_usec = static_cast<time_t>(timestamp % ::base::Time::kMicrosecondsPerSecond)}); + size_t bufferId = buffer->bufferId(); + + for (size_t i = 0; i < planes.size(); ++i) { + // Single-buffer input format may have multiple color planes, so bytesUsed of the single + // buffer should be sum of each color planes' size. + size_t bytesUsed = 0; + if (planes.size() == 1) { + bytesUsed = allocationSize(format, mInputLayout->mCodedSize); + } else { + bytesUsed = ::base::checked_cast<size_t>( + getArea(planeSize(format, i, mInputLayout->mCodedSize)).value()); + } + + // TODO(crbug.com/901264): The way to pass an offset within a DMA-buf is not defined + // in V4L2 specification, so we abuse data_offset for now. Fix it when we have the + // right interface, including any necessary validation and potential alignment. + buffer->setPlaneDataOffset(i, planes[i].mOffset); + bytesUsed += planes[i].mOffset; + // Workaround: filling length should not be needed. This is a bug of videobuf2 library. + buffer->setPlaneSize(i, mInputLayout->mPlanes[i].mSize + planes[i].mOffset); + buffer->setPlaneBytesUsed(i, bytesUsed); + } + + if (!std::move(*buffer).queueDMABuf(frame->fds())) { + ALOGE("Failed to queue input buffer using QueueDMABuf"); + onError(); + return false; + } + + ALOGV("Queued buffer in input queue (index: %" PRId64 ", timestamp: %" PRId64 + ", bufferId: %zu)", + index, timestamp, bufferId); + + ALOG_ASSERT(!mInputBuffers[bufferId]); + mInputBuffers[bufferId] = std::move(frame); + + return true; +} + +bool V4L2Encoder::enqueueOutputBuffer() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mOutputQueue->freeBuffersCount() > 0); + + auto buffer = mOutputQueue->getFreeBuffer(); + if (!buffer) { + ALOGE("Failed to get free buffer from device output queue"); + onError(); + return false; + } + + std::unique_ptr<BitstreamBuffer> bitstreamBuffer; + mFetchOutputBufferCb.Run(mOutputBufferSize, &bitstreamBuffer); + if (!bitstreamBuffer) { + ALOGE("Failed to fetch output block"); + onError(); + return false; + } + + size_t bufferId = buffer->bufferId(); + + std::vector<int> fds; + fds.push_back(bitstreamBuffer->dmabuf_fd); + if (!std::move(*buffer).queueDMABuf(fds)) { + ALOGE("Failed to queue output buffer using QueueDMABuf"); + onError(); + return false; + } + + ALOG_ASSERT(!mOutputBuffers[bufferId]); + mOutputBuffers[bufferId] = std::move(bitstreamBuffer); + ALOGV("%s(): Queued buffer in output queue (bufferId: %zu)", __func__, bufferId); + return true; +} + +bool V4L2Encoder::dequeueInputBuffer() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState != State::UNINITIALIZED); + ALOG_ASSERT(mInputQueue->queuedBuffersCount() > 0); + + if (mState == State::ERROR) { + return false; + } + + bool success; + V4L2ReadableBufferRef buffer; + std::tie(success, buffer) = mInputQueue->dequeueBuffer(); + if (!success) { + ALOGE("Failed to dequeue buffer from input queue"); + onError(); + return false; + } + if (!buffer) { + // No more buffers ready to be dequeued in input queue. + return false; + } + + uint64_t index = mInputBuffers[buffer->bufferId()]->index(); + int64_t timestamp = buffer->getTimeStamp().tv_usec + + buffer->getTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond; + ALOGV("Dequeued buffer from input queue (index: %" PRId64 ", timestamp: %" PRId64 + ", bufferId: %zu)", + index, timestamp, buffer->bufferId()); + + mInputBuffers[buffer->bufferId()] = nullptr; + + mInputBufferDoneCb.Run(index); + + // If we previously used up all input queue buffers we can start encoding again now. + if ((mState == State::WAITING_FOR_V4L2_BUFFER) && !mEncodeRequests.empty()) { + setState(State::ENCODING); + mTaskRunner->PostTask(FROM_HERE, + ::base::BindOnce(&V4L2Encoder::handleEncodeRequest, mWeakThis)); + } + + return true; +} + +bool V4L2Encoder::dequeueOutputBuffer() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(mState != State::UNINITIALIZED); + ALOG_ASSERT(mOutputQueue->queuedBuffersCount() > 0); + + if (mState == State::ERROR) { + return false; + } + + bool success; + V4L2ReadableBufferRef buffer; + std::tie(success, buffer) = mOutputQueue->dequeueBuffer(); + if (!success) { + ALOGE("Failed to dequeue buffer from output queue"); + onError(); + return false; + } + if (!buffer) { + // No more buffers ready to be dequeued in output queue. + return false; + } + + size_t encodedDataSize = buffer->getPlaneBytesUsed(0) - buffer->getPlaneDataOffset(0); + ::base::TimeDelta timestamp = ::base::TimeDelta::FromMicroseconds( + buffer->getTimeStamp().tv_usec + + buffer->getTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond); + + ALOGV("Dequeued buffer from output queue (timestamp: %" PRId64 + ", bufferId: %zu, data size: %zu, EOS: %d)", + timestamp.InMicroseconds(), buffer->bufferId(), encodedDataSize, buffer->isLast()); + + if (!mOutputBuffers[buffer->bufferId()]) { + ALOGE("Failed to find output block associated with output buffer"); + onError(); + return false; + } + + std::unique_ptr<BitstreamBuffer> bitstream_buffer = + std::move(mOutputBuffers[buffer->bufferId()]); + if (encodedDataSize > 0) { + mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), buffer->isKeyframe(), + std::move(bitstream_buffer)); + } + + // If the buffer is marked as last and we were flushing the encoder, flushing is now done. + if ((mState == State::DRAINING) && buffer->isLast()) { + onDrainDone(true); + // Start the encoder again. + struct v4l2_encoder_cmd cmd; + memset(&cmd, 0, sizeof(v4l2_encoder_cmd)); + cmd.cmd = V4L2_ENC_CMD_START; + if (mDevice->ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) { + ALOGE("Failed to restart encoder after draining (V4L2_ENC_CMD_START)"); + onError(); + return false; + } + } + + // Queue a new output buffer to replace the one we dequeued. + buffer = nullptr; + enqueueOutputBuffer(); + + return true; +} + +bool V4L2Encoder::createInputBuffers() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(!mInputQueue->isStreaming()); + ALOG_ASSERT(mInputBuffers.empty()); + + // No memory is allocated here, we just generate a list of buffers on the input queue, which + // will hold memory handles to the real buffers. + if (mInputQueue->allocateBuffers(kInputBufferCount, V4L2_MEMORY_DMABUF) < kInputBufferCount) { + ALOGE("Failed to create V4L2 input buffers."); + return false; + } + + mInputBuffers.resize(mInputQueue->allocatedBuffersCount()); + return true; +} + +bool V4L2Encoder::createOutputBuffers() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(!mOutputQueue->isStreaming()); + ALOG_ASSERT(mOutputBuffers.empty()); + + // No memory is allocated here, we just generate a list of buffers on the output queue, which + // will hold memory handles to the real buffers. + if (mOutputQueue->allocateBuffers(kOutputBufferCount, V4L2_MEMORY_DMABUF) < + kOutputBufferCount) { + ALOGE("Failed to create V4L2 output buffers."); + return false; + } + + mOutputBuffers.resize(mOutputQueue->allocatedBuffersCount()); + return true; +} + +void V4L2Encoder::destroyInputBuffers() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(!mInputQueue->isStreaming()); + + if (!mInputQueue || mInputQueue->allocatedBuffersCount() == 0) return; + mInputQueue->deallocateBuffers(); + mInputBuffers.clear(); +} + +void V4L2Encoder::destroyOutputBuffers() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + ALOG_ASSERT(!mOutputQueue->isStreaming()); + + if (!mOutputQueue || mOutputQueue->allocatedBuffersCount() == 0) return; + mOutputQueue->deallocateBuffers(); + mOutputBuffers.clear(); +} + +void V4L2Encoder::onError() { + ALOGV("%s()", __func__); + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + if (mState != State::ERROR) { + setState(State::ERROR); + mErrorCb.Run(); + } +} + +void V4L2Encoder::setState(State state) { + ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence()); + + // Check whether the state change is valid. + switch (state) { + case State::UNINITIALIZED: + break; + case State::WAITING_FOR_INPUT_FRAME: + ALOG_ASSERT(mState != State::ERROR); + break; + case State::WAITING_FOR_V4L2_BUFFER: + ALOG_ASSERT(mState == State::ENCODING); + break; + case State::ENCODING: + ALOG_ASSERT(mState == State::WAITING_FOR_INPUT_FRAME || + mState == State::WAITING_FOR_V4L2_BUFFER || mState == State::DRAINING); + break; + case State::DRAINING: + ALOG_ASSERT(mState == State::ENCODING || mState == State::WAITING_FOR_INPUT_FRAME); + break; + case State::ERROR: + break; + } + + ALOGV("Changed encoder state from %s to %s", stateToString(mState), stateToString(state)); + mState = state; +} + +const char* V4L2Encoder::stateToString(State state) { + switch (state) { + case State::UNINITIALIZED: + return "UNINITIALIZED"; + case State::WAITING_FOR_INPUT_FRAME: + return "WAITING_FOR_INPUT_FRAME"; + case State::WAITING_FOR_V4L2_BUFFER: + return "WAITING_FOR_V4L2_BUFFER"; + case State::ENCODING: + return "ENCODING"; + case State::DRAINING: + return "DRAINING"; + case State::ERROR: + return "ERROR"; + } +} + +} // namespace android diff --git a/components/VideoEncoder.cpp b/components/VideoEncoder.cpp new file mode 100644 index 0000000..e3e19c2 --- /dev/null +++ b/components/VideoEncoder.cpp @@ -0,0 +1,18 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <v4l2_codec2/components/VideoEncoder.h> + +namespace android { + +VideoEncoder::InputFrame::InputFrame(std::vector<int>&& fds, std::vector<VideoFramePlane>&& planes, + VideoPixelFormat pixelFormat, uint64_t index, + int64_t timestamp) + : mFds(std::move(fds)), + mPlanes(std::move(planes)), + mPixelFormat(pixelFormat), + mIndex(index), + mTimestamp(timestamp) {} + +} // namespace android diff --git a/components/VideoFrame.cpp b/components/VideoFrame.cpp index cb5efb7..b7481ad 100644 --- a/components/VideoFrame.cpp +++ b/components/VideoFrame.cpp @@ -34,11 +34,11 @@ const std::vector<int>& VideoFrame::getFDs() const { return mFds; } -void VideoFrame::setVisibleRect(const media::Rect& visibleRect) { +void VideoFrame::setVisibleRect(const Rect& visibleRect) { mVisibleRect = visibleRect; } -const media::Rect& VideoFrame::getVisibleRect() const { +const Rect& VideoFrame::getVisibleRect() const { return mVisibleRect; } diff --git a/components/VideoFramePool.cpp b/components/VideoFramePool.cpp index b6bbfab..665ff73 100644 --- a/components/VideoFramePool.cpp +++ b/components/VideoFramePool.cpp @@ -26,14 +26,15 @@ using android::hardware::graphics::common::V1_0::BufferUsage; namespace android { // static -std::optional<uint32_t> VideoFramePool::getBufferIdFromGraphicBlock(const C2BlockPool& blockPool, +std::optional<uint32_t> VideoFramePool::getBufferIdFromGraphicBlock(C2BlockPool& blockPool, const C2Block2D& block) { ALOGV("%s() blockPool.getAllocatorId() = %u", __func__, blockPool.getAllocatorId()); if (blockPool.getAllocatorId() == android::V4L2AllocatorId::V4L2_BUFFERPOOL) { return C2VdaPooledBlockPool::getBufferIdFromGraphicBlock(block); } else if (blockPool.getAllocatorId() == C2PlatformAllocatorStore::BUFFERQUEUE) { - return C2VdaBqBlockPool::getBufferIdFromGraphicBlock(block); + C2VdaBqBlockPool* bqPool = static_cast<C2VdaBqBlockPool*>(&blockPool); + return bqPool->getBufferIdFromGraphicBlock(block); } ALOGE("%s(): unknown allocator ID: %u", __func__, blockPool.getAllocatorId()); @@ -41,7 +42,9 @@ std::optional<uint32_t> VideoFramePool::getBufferIdFromGraphicBlock(const C2Bloc } // static -c2_status_t VideoFramePool::requestNewBufferSet(C2BlockPool& blockPool, int32_t bufferCount) { +c2_status_t VideoFramePool::requestNewBufferSet(C2BlockPool& blockPool, int32_t bufferCount, + const ui::Size& size, uint32_t format, + C2MemoryUsage usage) { ALOGV("%s() blockPool.getAllocatorId() = %u", __func__, blockPool.getAllocatorId()); if (blockPool.getAllocatorId() == android::V4L2AllocatorId::V4L2_BUFFERPOOL) { @@ -49,7 +52,7 @@ c2_status_t VideoFramePool::requestNewBufferSet(C2BlockPool& blockPool, int32_t return bpPool->requestNewBufferSet(bufferCount); } else if (blockPool.getAllocatorId() == C2PlatformAllocatorStore::BUFFERQUEUE) { C2VdaBqBlockPool* bqPool = static_cast<C2VdaBqBlockPool*>(&blockPool); - return bqPool->requestNewBufferSet(bufferCount); + return bqPool->requestNewBufferSet(bufferCount, size.width, size.height, format, usage); } ALOGE("%s(): unknown allocator ID: %u", __func__, blockPool.getAllocatorId()); @@ -69,31 +72,40 @@ bool VideoFramePool::setNotifyBlockAvailableCb(C2BlockPool& blockPool, ::base::O // static std::unique_ptr<VideoFramePool> VideoFramePool::Create( - std::shared_ptr<C2BlockPool> blockPool, const size_t numBuffers, const media::Size& size, + std::shared_ptr<C2BlockPool> blockPool, const size_t numBuffers, const ui::Size& size, HalPixelFormat pixelFormat, bool isSecure, scoped_refptr<::base::SequencedTaskRunner> taskRunner) { ALOG_ASSERT(blockPool != nullptr); - if (requestNewBufferSet(*blockPool, numBuffers) != C2_OK) { + uint64_t usage = static_cast<uint64_t>(BufferUsage::VIDEO_DECODER); + if (isSecure) { + usage |= C2MemoryUsage::READ_PROTECTED; + } else if (blockPool->getAllocatorId() == android::V4L2AllocatorId::V4L2_BUFFERPOOL) { + // CPU access to buffers is only required in byte buffer mode. + usage |= C2MemoryUsage::CPU_READ; + } + const C2MemoryUsage memoryUsage(usage); + + if (requestNewBufferSet(*blockPool, numBuffers, size, static_cast<uint32_t>(pixelFormat), + memoryUsage) != C2_OK) { return nullptr; } std::unique_ptr<VideoFramePool> pool = ::base::WrapUnique(new VideoFramePool( - std::move(blockPool), size, pixelFormat, isSecure, std::move(taskRunner))); + std::move(blockPool), size, pixelFormat, memoryUsage, std::move(taskRunner))); if (!pool->initialize()) return nullptr; return pool; } -VideoFramePool::VideoFramePool(std::shared_ptr<C2BlockPool> blockPool, const media::Size& size, - HalPixelFormat pixelFormat, bool isSecure, +VideoFramePool::VideoFramePool(std::shared_ptr<C2BlockPool> blockPool, const ui::Size& size, + HalPixelFormat pixelFormat, C2MemoryUsage memoryUsage, scoped_refptr<::base::SequencedTaskRunner> taskRunner) : mBlockPool(std::move(blockPool)), mSize(size), mPixelFormat(pixelFormat), - mMemoryUsage(isSecure ? C2MemoryUsage::READ_PROTECTED : C2MemoryUsage::CPU_READ, - static_cast<uint64_t>(BufferUsage::VIDEO_DECODER)), + mMemoryUsage(memoryUsage), mClientTaskRunner(std::move(taskRunner)) { - ALOGV("%s(size=%dx%d)", __func__, size.width(), size.height()); + ALOGV("%s(size=%dx%d)", __func__, size.width, size.height); ALOG_ASSERT(mClientTaskRunner->RunsTasksInCurrentSequence()); DCHECK(mBlockPool); DCHECK(mClientTaskRunner); @@ -168,9 +180,8 @@ void VideoFramePool::getVideoFrameTask() { static size_t sDelay = kFetchRetryDelayInit; std::shared_ptr<C2GraphicBlock> block; - c2_status_t err = mBlockPool->fetchGraphicBlock(mSize.width(), mSize.height(), - static_cast<uint32_t>(mPixelFormat), - mMemoryUsage, &block); + c2_status_t err = mBlockPool->fetchGraphicBlock( + mSize.width, mSize.height, static_cast<uint32_t>(mPixelFormat), mMemoryUsage, &block); if (err == C2_TIMED_OUT || err == C2_BLOCKING) { if (setNotifyBlockAvailableCb(*mBlockPool, ::base::BindOnce(&VideoFramePool::getVideoFrameTaskThunk, diff --git a/components/include/v4l2_codec2/components/BitstreamBuffer.h b/components/include/v4l2_codec2/components/BitstreamBuffer.h index ec8a917..d61e4f9 100644 --- a/components/include/v4l2_codec2/components/BitstreamBuffer.h +++ b/components/include/v4l2_codec2/components/BitstreamBuffer.h @@ -7,8 +7,6 @@ #include <stdint.h> -#include <base/files/scoped_file.h> - namespace android { // The BitstreamBuffer class can be used to store encoded video data. diff --git a/components/include/v4l2_codec2/components/V4L2ComponentFactory.h b/components/include/v4l2_codec2/components/V4L2ComponentFactory.h new file mode 100644 index 0000000..fc6abea --- /dev/null +++ b/components/include/v4l2_codec2/components/V4L2ComponentFactory.h @@ -0,0 +1,39 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_FACTORY_H +#define ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_FACTORY_H + +#include <memory> +#include <string> + +#include <C2ComponentFactory.h> +#include <util/C2InterfaceHelper.h> + +namespace android { + +class V4L2ComponentFactory : public C2ComponentFactory { +public: + static std::unique_ptr<V4L2ComponentFactory> create( + const std::string& componentName, std::shared_ptr<C2ReflectorHelper> reflector); + V4L2ComponentFactory(const std::string& componentName, bool isEncoder, + std::shared_ptr<C2ReflectorHelper> reflector); + ~V4L2ComponentFactory() override = default; + + // Implementation of C2ComponentFactory. + c2_status_t createComponent(c2_node_id_t id, std::shared_ptr<C2Component>* const component, + ComponentDeleter deleter) override; + c2_status_t createInterface(c2_node_id_t id, + std::shared_ptr<C2ComponentInterface>* const interface, + InterfaceDeleter deleter) override; + +private: + const std::string mComponentName; + const bool mIsEncoder; + std::shared_ptr<C2ReflectorHelper> mReflector; +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_FACTORY_H diff --git a/store/include/v4l2_codec2/store/V4L2ComponentStore.h b/components/include/v4l2_codec2/components/V4L2ComponentStore.h index 8013f55..bfec407 100644 --- a/store/include/v4l2_codec2/store/V4L2ComponentStore.h +++ b/components/include/v4l2_codec2/components/V4L2ComponentStore.h @@ -2,15 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef ANDROID_V4L2_CODEC2_STORE_V4L2_COMPONENT_STORE_H -#define ANDROID_V4L2_CODEC2_STORE_V4L2_COMPONENT_STORE_H +#ifndef ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_STORE_H +#define ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_STORE_H #include <map> #include <mutex> -#include <android-base/thread_annotations.h> #include <C2Component.h> #include <C2ComponentFactory.h> +#include <android-base/thread_annotations.h> #include <util/C2InterfaceHelper.h> namespace android { @@ -41,27 +41,21 @@ public: std::vector<C2FieldSupportedValuesQuery>& fields) const override; private: - using CreateV4L2FactoryFunc = ::C2ComponentFactory* (*)(const char* /* componentName */); - using DestroyV4L2FactoryFunc = void (*)(::C2ComponentFactory*); - - V4L2ComponentStore(void* libHandle, CreateV4L2FactoryFunc createFactoryFunc, - DestroyV4L2FactoryFunc destroyFactoryFunc); + V4L2ComponentStore(); ::C2ComponentFactory* GetFactory(const C2String& name); std::shared_ptr<const C2Component::Traits> GetTraits(const C2String& name); - void* mLibHandle; - CreateV4L2FactoryFunc mCreateFactoryFunc; - DestroyV4L2FactoryFunc mDestroyFactoryFunc; - std::shared_ptr<C2ReflectorHelper> mReflector; std::mutex mCachedFactoriesLock; - std::map<C2String, ::C2ComponentFactory*> mCachedFactories GUARDED_BY(mCachedFactoriesLock); + std::map<C2String, std::unique_ptr<::C2ComponentFactory>> mCachedFactories + GUARDED_BY(mCachedFactoriesLock); std::mutex mCachedTraitsLock; - std::map<C2String, std::shared_ptr<const C2Component::Traits>> mCachedTraits GUARDED_BY(mCachedTraitsLock); + std::map<C2String, std::shared_ptr<const C2Component::Traits>> mCachedTraits + GUARDED_BY(mCachedTraitsLock); }; } // namespace android -#endif // ANDROID_V4L2_CODEC2_STORE_V4L2_COMPONENT_STORE_H +#endif // ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_COMPONENT_STORE_H diff --git a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h index 37da866..1e98118 100644 --- a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h +++ b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h @@ -5,6 +5,7 @@ #ifndef ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_DECODE_COMPONENT_H #define ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_DECODE_COMPONENT_H +#include <atomic> #include <memory> #include <C2Component.h> @@ -19,7 +20,6 @@ #include <v4l2_codec2/components/V4L2DecodeInterface.h> #include <v4l2_codec2/components/VideoDecoder.h> #include <v4l2_codec2/components/VideoFramePool.h> -#include <v4l2_device.h> namespace android { @@ -59,9 +59,9 @@ private: static const char* ComponentStateToString(ComponentState state); // Handle C2Component's public methods on |mDecoderTaskRunner|. - void destroyTask(); - void startTask(c2_status_t* status); + void startTask(c2_status_t* status, ::base::WaitableEvent* done); void stopTask(); + void releaseTask(); void queueTask(std::unique_ptr<C2Work> work); void flushTask(); void drainTask(); @@ -70,8 +70,9 @@ private: // Try to process pending works at |mPendingWorks|. Paused when |mIsDraining| is set. void pumpPendingWorks(); // Get the buffer pool. - void getVideoFramePool(std::unique_ptr<VideoFramePool>* pool, const media::Size& size, - HalPixelFormat pixelFormat, size_t numBuffers); + std::unique_ptr<VideoFramePool> getVideoFramePool(const ui::Size& size, + HalPixelFormat pixelFormat, + size_t numBuffers); // Detect and report works with no-show frame, only used at VP8 and VP9. void detectNoShowFrameWorksAndReportIfFinished(const C2WorkOrdinalStruct& currOrdinal); @@ -91,6 +92,8 @@ private: // Report error when any error occurs. void reportError(c2_status_t error); + static std::atomic<int32_t> sConcurrentInstances; + // The pointer of component interface implementation. std::shared_ptr<V4L2DecodeInterface> mIntfImpl; // The pointer of component interface. @@ -119,7 +122,6 @@ private: // The mutex lock to synchronize start/stop/reset/release calls. std::mutex mStartStopLock; - ::base::WaitableEvent mStartStopDone; // The color aspects parameter for current decoded output buffers. std::shared_ptr<C2StreamColorAspectsInfo::output> mCurrentColorAspects; @@ -138,6 +140,9 @@ private: ::base::Thread mDecoderThread{"V4L2DecodeComponentDecoderThread"}; scoped_refptr<::base::SequencedTaskRunner> mDecoderTaskRunner; + // Hold a weak_ptr of |*this| when |mDecoderThread| is running. + std::weak_ptr<V4L2DecodeComponent> mStdWeakThis; + ::base::WeakPtrFactory<V4L2DecodeComponent> mWeakThisFactory{this}; ::base::WeakPtr<V4L2DecodeComponent> mWeakThis; }; diff --git a/components/include/v4l2_codec2/components/V4L2DecodeInterface.h b/components/include/v4l2_codec2/components/V4L2DecodeInterface.h index b57f6c1..f2ab898 100644 --- a/components/include/v4l2_codec2/components/V4L2DecodeInterface.h +++ b/components/include/v4l2_codec2/components/V4L2DecodeInterface.h @@ -9,9 +9,9 @@ #include <string> #include <C2Config.h> +#include <ui/Size.h> #include <util/C2InterfaceHelper.h> -#include <size.h> #include <v4l2_codec2/common/VideoTypes.h> namespace android { @@ -27,8 +27,8 @@ public: c2_status_t status() const { return mInitStatus; } C2BlockPool::local_id_t getBlockPoolId() const { return mOutputBlockPoolIds->m.values[0]; } std::optional<VideoCodec> getVideoCodec() const { return mVideoCodec; } - media::Size getMaxSize() const { return mMaxSize; } - media::Size getMinSize() const { return mMinSize; } + + static uint32_t getOutputDelay(VideoCodec codec); size_t getInputBufferSize() const; c2_status_t queryColorAspects( @@ -50,6 +50,8 @@ private: const C2P<C2StreamColorAspectsTuning::output>& def, const C2P<C2StreamColorAspectsInfo::input>& coded); + // The kind of the component; should be C2Component::KIND_DECODER. + std::shared_ptr<C2ComponentKindSetting> mKind; // The input format kind; should be C2FormatCompressed. std::shared_ptr<C2StreamBufferTypeSetting::input> mInputFormat; // The memory usage flag of input buffer; should be BufferUsage::VIDEO_DECODER. @@ -94,8 +96,6 @@ private: c2_status_t mInitStatus; std::optional<VideoCodec> mVideoCodec; - media::Size mMinSize; - media::Size mMaxSize; }; } // namespace android diff --git a/components/include/v4l2_codec2/components/V4L2Decoder.h b/components/include/v4l2_codec2/components/V4L2Decoder.h index bdddc7f..b65bd49 100644 --- a/components/include/v4l2_codec2/components/V4L2Decoder.h +++ b/components/include/v4l2_codec2/components/V4L2Decoder.h @@ -13,13 +13,13 @@ #include <base/callback.h> #include <base/memory/weak_ptr.h> -#include <rect.h> -#include <size.h> +#include <ui/Rect.h> +#include <ui/Size.h> +#include <v4l2_codec2/common/V4L2Device.h> #include <v4l2_codec2/common/VideoTypes.h> #include <v4l2_codec2/components/VideoDecoder.h> #include <v4l2_codec2/components/VideoFrame.h> #include <v4l2_codec2/components/VideoFramePool.h> -#include <v4l2_device.h> namespace android { @@ -49,7 +49,6 @@ private: : buffer(std::move(buffer)), decodeCb(std::move(decodeCb)) {} DecodeRequest(DecodeRequest&&) = default; ~DecodeRequest() = default; - DecodeRequest& operator=(DecodeRequest&&); std::unique_ptr<BitstreamBuffer> buffer; // nullptr means Drain DecodeCB decodeCb; @@ -64,13 +63,14 @@ private: void serviceDeviceTask(bool event); bool dequeueResolutionChangeEvent(); bool changeResolution(); + bool setupOutputFormat(const ui::Size& size); void tryFetchVideoFrame(); void onVideoFrameReady(std::optional<VideoFramePool::FrameWithBlockId> frameWithBlockId); std::optional<size_t> getNumOutputBuffers(); std::optional<struct v4l2_format> getFormatInfo(); - media::Rect getVisibleRect(const media::Size& codedSize); + Rect getVisibleRect(const ui::Size& codedSize); bool sendV4L2DecoderCmd(bool start); void setState(State newState); @@ -78,9 +78,9 @@ private: std::unique_ptr<VideoFramePool> mVideoFramePool; - scoped_refptr<media::V4L2Device> mDevice; - scoped_refptr<media::V4L2Queue> mInputQueue; - scoped_refptr<media::V4L2Queue> mOutputQueue; + scoped_refptr<V4L2Device> mDevice; + scoped_refptr<V4L2Queue> mInputQueue; + scoped_refptr<V4L2Queue> mOutputQueue; std::queue<DecodeRequest> mDecodeRequests; std::map<int32_t, DecodeCB> mPendingDecodeCbs; @@ -90,8 +90,8 @@ private: DecodeCB mDrainCb; ErrorCB mErrorCb; - media::Size mCodedSize; - media::Rect mVisibleRect; + ui::Size mCodedSize; + Rect mVisibleRect; std::map<size_t, std::unique_ptr<VideoFrame>> mFrameAtDevice; diff --git a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h index 4a61e05..4665ffa 100644 --- a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h +++ b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h @@ -8,6 +8,7 @@ #include <atomic> #include <memory> #include <optional> +#include <unordered_map> #include <C2Component.h> #include <C2ComponentFactory.h> @@ -22,20 +23,12 @@ #include <base/threading/thread.h> #include <util/C2InterfaceHelper.h> -#include <size.h> -#include <v4l2_codec2/common/FormatConverter.h> -#include <v4l2_codec2/components/V4L2EncodeInterface.h> -#include <video_frame_layout.h> - -namespace media { -class V4L2Device; -class V4L2ReadableBuffer; -class V4L2Queue; -} // namespace media - namespace android { -struct VideoFramePlane; +struct BitstreamBuffer; +class FormatConverter; +class VideoEncoder; +class V4L2EncodeInterface; class V4L2EncodeComponent : public C2Component, public std::enable_shared_from_this<V4L2EncodeComponent> { @@ -62,19 +55,6 @@ public: std::shared_ptr<C2ComponentInterface> intf() override; private: - class InputFrame { - public: - // Create an input frame from a C2GraphicBlock. - static std::unique_ptr<InputFrame> Create(const C2ConstGraphicBlock& block); - ~InputFrame() = default; - - const std::vector<int>& getFDs() const { return mFds; } - - private: - InputFrame(std::vector<int> fds) : mFds(std::move(fds)) {} - const std::vector<int> mFds; - }; - // Possible component states. enum class ComponentState { UNLOADED, // Initial state of component. @@ -83,16 +63,6 @@ private: ERROR, // An error occurred. }; - // Possible encoder states. - enum class EncoderState { - UNINITIALIZED, // Not initialized yet or initialization failed. - WAITING_FOR_INPUT, // Waiting for work to be queued. - WAITING_FOR_INPUT_BUFFERS, // Waiting for V4L2 input queue buffers. - ENCODING, // Queuing input buffers. - DRAINING, // Flushing encoder. - ERROR, // encoder encountered an error. - }; - V4L2EncodeComponent(C2String name, c2_node_id_t id, std::shared_ptr<V4L2EncodeInterface> interface); @@ -109,7 +79,7 @@ private: // scheduled work and mark the last item as EOS, before processing any new work. void drainTask(drain_mode_t drainMode); // Called on the encoder thread when a drain is completed. - void onDrainDone(bool done); + void onDrainDone(bool success); // Flush all currently scheduled work on the encoder thread. The encoder will abort all // scheduled work items, work that can be immediately aborted will be placed in |flushedWork|. void flushTask(::base::WaitableEvent* done, @@ -119,13 +89,6 @@ private: // Initialize the V4L2 device for encoding with the requested configuration. bool initializeEncoder(); - // Configure input format on the V4L2 device. - bool configureInputFormat(media::VideoPixelFormat inputFormat); - // Configure output format on the V4L2 device. - bool configureOutputFormat(media::VideoCodecProfile outputProfile); - // Configure required and optional controls on the V4L2 device. - bool configureDevice(media::VideoCodecProfile outputProfile, - std::optional<const uint8_t> outputH264Level); // Update the |mBitrate| and |mFramerate| currently configured on the V4L2 device, to match the // values requested by the codec 2.0 framework. bool updateEncodingParameters(); @@ -134,19 +97,17 @@ private: void scheduleNextEncodeTask(); // Encode the specified |block| with corresponding |index| and |timestamp|. bool encode(C2ConstGraphicBlock block, uint64_t index, int64_t timestamp); - // Drain the encoder. - void drain(); // Flush the encoder. void flush(); - // Fetch a new output buffer from the output block pool. - std::shared_ptr<C2LinearBlock> fetchOutputBlock(); + // Fetch a new output buffer from the output block pool with specified |size|. + void fetchOutputBlock(uint32_t size, std::unique_ptr<BitstreamBuffer>* buffer); // Called on the encoder thread when the encoder is done using an input buffer. void onInputBufferDone(uint64_t index); // Called on the encoder thread when an output buffer is ready. - void onOutputBufferDone(uint32_t payloadSize, bool keyFrame, int64_t timestamp, - std::shared_ptr<C2LinearBlock> outputBlock); + void onOutputBufferDone(size_t dataSize, int64_t timestamp, bool keyFrame, + std::unique_ptr<BitstreamBuffer> buffer); // Helper function to find a work item in the output work queue by index. C2Work* getWorkByIndex(uint64_t index); @@ -157,52 +118,22 @@ private: // Notify the listener the specified |work| item is finished. void reportWork(std::unique_ptr<C2Work> work); - // Attempt to start the V4L2 device poller. - bool startDevicePoll(); - // Attempt to stop the V4L2 device poller. - bool stopDevicePoll(); - // Called by the V4L2 device poller on the |mEncoderTaskRunner| whenever an error occurred. - void onPollError(); - // Service I/O on the V4L2 device, called by the V4L2 device poller on the |mEncoderTaskRunner|. - void serviceDeviceTask(bool event); - - // Enqueue an input buffer to be encoded on the device input queue. Returns whether the - // operation was successful. - bool enqueueInputBuffer(std::unique_ptr<InputFrame> frame, media::VideoPixelFormat format, - const std::vector<VideoFramePlane>& planes, int64_t index, - int64_t timestamp); - // Enqueue an output buffer to store the encoded bitstream on the device output queue. Returns - // whether the operation was successful. - bool enqueueOutputBuffer(); - // Dequeue an input buffer the V4L2 device has finished encoding on the device input queue. - // Returns whether a buffer could be dequeued. - bool dequeueInputBuffer(); - // Dequeue an output buffer containing the encoded bitstream from the device output queue. The - // bitstream is copied into another buffer that is sent to the client, after which the output - // buffer is returned to the queue. Returns whether the operation was successful. - bool dequeueOutputBuffer(); - - // Create input buffers on the V4L2 device input queue. - bool createInputBuffers(); - // Create output buffers on the V4L2 device output queue. - bool createOutputBuffers(); - // Destroy the input buffers on the V4L2 device input queue. - void destroyInputBuffers(); - // Destroy the output buffers on the V4L2 device output queue. - void destroyOutputBuffers(); + // Configure the c2 block pool that will be used to create output buffers. + bool getBlockPool(); // Notify the client an error occurred and switch to the error state. void reportError(c2_status_t error); // Change the state of the component. void setComponentState(ComponentState state); - // Change the state of the encoder, only called on the encoder thread. - void setEncoderState(EncoderState state); // Get the specified component |state| as string. static const char* componentStateToString(ComponentState state); - // Get the specified encoder |state| as string. - static const char* encoderStateToString(EncoderState state); + // The underlying V4L2 encoder. + std::unique_ptr<VideoEncoder> mEncoder; + + // The number of concurrent encoder instances currently created. + static std::atomic<int32_t> sConcurrentInstances; // The component's registered name. const C2String mName; // The component's id, provided by the C2 framework upon initialization. @@ -217,53 +148,30 @@ private: // The component's listener to be notified when events occur, only accessed on encoder thread. std::shared_ptr<Listener> mListener; - // The V4L2 device used to interact with the driver, only accessed on encoder thread. - scoped_refptr<media::V4L2Device> mDevice; - scoped_refptr<media::V4L2Queue> mInputQueue; - scoped_refptr<media::V4L2Queue> mOutputQueue; - - // The video stream's visible size. - media::Size mVisibleSize; - // The video stream's coded size. - media::Size mInputCodedSize; - // The input layout configured on the V4L2 device. - std::optional<media::VideoFrameLayout> mInputLayout; + // The queue of encode work items waiting for free buffers in the input convertor. + std::queue<std::unique_ptr<C2Work>> mInputConverterQueue; // An input format convertor will be used if the device doesn't support the video's format. std::unique_ptr<FormatConverter> mInputFormatConverter; - // Required output buffer byte size. - uint32_t mOutputBufferSize = 0; // The bitrate currently configured on the v4l2 device. uint32_t mBitrate = 0; // The framerate currently configured on the v4l2 device. uint32_t mFramerate = 0; - // How often we want to request the V4L2 device to create a key frame. - uint32_t mKeyFramePeriod = 0; - // Key frame counter, a key frame will be requested each time it reaches zero. - uint32_t mKeyFrameCounter = 0; - // Whether we extracted and submitted CSD (codec-specific data, e.g. H.264 SPS) to the framework. bool mCSDSubmitted = false; - // The queue of encode work items to be processed. - std::queue<std::unique_ptr<C2Work>> mInputWorkQueue; // The queue of encode work items currently being processed. - std::deque<std::unique_ptr<C2Work>> mOutputWorkQueue; + std::deque<std::unique_ptr<C2Work>> mWorkQueue; - // List of work item indices and frames associated with each buffer in the device input queue. - std::vector<std::pair<int64_t, std::unique_ptr<InputFrame>>> mInputBuffersMap; + // Map of buffer ids and associated C2LinearBlock buffers. The buffer's fds are used as id. + std::unordered_map<int32_t, std::shared_ptr<C2LinearBlock>> mOutputBuffersMap; - // Map of buffer indices and output blocks associated with each buffer in the output queue. This - // map keeps the C2LinearBlock buffers alive so we can avoid duplicated fds. - std::vector<std::shared_ptr<C2LinearBlock>> mOutputBuffersMap; // The output block pool. std::shared_ptr<C2BlockPool> mOutputBlockPool; // The component state, accessible from any thread as C2Component interface is not thread-safe. std::atomic<ComponentState> mComponentState; - // The current state of the encoder, only accessed on the encoder thread. - EncoderState mEncoderState = EncoderState::UNINITIALIZED; // The encoder thread on which all interaction with the V4L2 device is performed. ::base::Thread mEncoderThread{"V4L2EncodeComponentThread"}; diff --git a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h index f480d25..2efbfcc 100644 --- a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h +++ b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h @@ -11,11 +11,10 @@ #include <C2.h> #include <C2Buffer.h> #include <C2Config.h> +#include <ui/Size.h> #include <util/C2InterfaceHelper.h> -#include <size.h> #include <v4l2_codec2/common/EncodeHelpers.h> -#include <video_codecs.h> namespace media { class V4L2Device; @@ -33,10 +32,11 @@ public: // Note: these getters are not thread-safe. For dynamic parameters, component should use // formal query API for C2ComponentInterface instead. c2_status_t status() const { return mInitStatus; } + const char* getOutputMediaType() const { return mOutputMediaType->m.value; } C2Config::profile_t getOutputProfile() const { return mProfileLevel->profile; } C2Config::level_t getOutputLevel() const { return mProfileLevel->level; } - const media::Size getInputVisibleSize() const { - return media::Size(mInputVisibleSize->width, mInputVisibleSize->height); + const ui::Size getInputVisibleSize() const { + return ui::Size(mInputVisibleSize->width, mInputVisibleSize->height); } C2BlockPool::local_id_t getBlockPoolId() const { return mOutputBlockPoolIds->m.values[0]; } // Get sync key-frame period in frames. @@ -46,10 +46,14 @@ protected: void Initialize(const C2String& name); // Configurable parameter setters. - static C2R ProfileLevelSetter(bool mayBlock, C2P<C2StreamProfileLevelInfo::output>& info, - const C2P<C2StreamPictureSizeInfo::input>& videosize, - const C2P<C2StreamFrameRateInfo::output>& frameRate, - const C2P<C2StreamBitrateInfo::output>& bitrate); + static C2R H264ProfileLevelSetter(bool mayBlock, C2P<C2StreamProfileLevelInfo::output>& info, + const C2P<C2StreamPictureSizeInfo::input>& videosize, + const C2P<C2StreamFrameRateInfo::output>& frameRate, + const C2P<C2StreamBitrateInfo::output>& bitrate); + static C2R VP9ProfileLevelSetter(bool mayBlock, C2P<C2StreamProfileLevelInfo::output>& info, + const C2P<C2StreamPictureSizeInfo::input>& videosize, + const C2P<C2StreamFrameRateInfo::output>& frameRate, + const C2P<C2StreamBitrateInfo::output>& bitrate); static C2R SizeSetter(bool mayBlock, C2P<C2StreamPictureSizeInfo::input>& videoSize); @@ -58,6 +62,8 @@ protected: // Constant parameters + // The kind of the component; should be C2Component::KIND_ENCODER. + std::shared_ptr<C2ComponentKindSetting> mKind; // The input format kind; should be C2FormatVideo. std::shared_ptr<C2StreamBufferTypeSetting::input> mInputFormat; // The memory usage flag of input buffer; should be BufferUsage::VIDEO_ENCODER. diff --git a/components/include/v4l2_codec2/components/V4L2Encoder.h b/components/include/v4l2_codec2/components/V4L2Encoder.h new file mode 100644 index 0000000..5abee8f --- /dev/null +++ b/components/include/v4l2_codec2/components/V4L2Encoder.h @@ -0,0 +1,189 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_ENCODER_H +#define ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_ENCODER_H + +#include <stdint.h> +#include <memory> +#include <optional> +#include <queue> +#include <vector> + +#include <base/memory/weak_ptr.h> +#include <base/sequenced_task_runner.h> +#include <ui/Size.h> + +#include <v4l2_codec2/common/Common.h> +#include <v4l2_codec2/components/VideoEncoder.h> + +namespace android { + +struct BitstreamBuffer; +struct VideoFramePlane; +class V4L2Device; +class V4L2Queue; + +class V4L2Encoder : public VideoEncoder { +public: + // Number of buffers on V4L2 device queues. + static constexpr size_t kInputBufferCount = 2; + static constexpr size_t kOutputBufferCount = 2; + + static std::unique_ptr<VideoEncoder> create( + C2Config::profile_t profile, std::optional<uint8_t> level, const ui::Size& visibleSize, + uint32_t stride, uint32_t keyFramePeriod, FetchOutputBufferCB fetchOutputBufferCb, + InputBufferDoneCB inputBufferDoneCb, OutputBufferDoneCB outputBufferDoneCb, + DrainDoneCB drainDoneCb, ErrorCB errorCb, + scoped_refptr<::base::SequencedTaskRunner> taskRunner); + ~V4L2Encoder() override; + + bool encode(std::unique_ptr<InputFrame> frame) override; + void drain() override; + void flush() override; + + bool setBitrate(uint32_t bitrate) override; + bool setFramerate(uint32_t framerate) override; + void requestKeyframe() override; + + VideoPixelFormat inputFormat() const override; + const ui::Size& visibleSize() const override { return mVisibleSize; } + const ui::Size& codedSize() const override { return mInputCodedSize; } + +private: + // Possible encoder states. + enum class State { + UNINITIALIZED, // Not initialized yet or initialization failed. + WAITING_FOR_INPUT_FRAME, // Waiting for frames to be queued. + WAITING_FOR_V4L2_BUFFER, // Waiting for V4L2 input queue buffers. + ENCODING, // Queuing input buffers. + DRAINING, // Draining encoder. + ERROR, // Encoder encountered an error. + }; + + // Contains a single encode request. + struct EncodeRequest { + EncodeRequest(std::unique_ptr<InputFrame> video_frame) + : video_frame(std::move(video_frame)) {} + ~EncodeRequest() = default; + EncodeRequest(EncodeRequest&&) = default; + EncodeRequest& operator=(EncodeRequest&&) = default; + + std::unique_ptr<InputFrame> video_frame; + bool end_of_stream = false; + }; + + V4L2Encoder(scoped_refptr<::base::SequencedTaskRunner> taskRunner, + FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB mInputBufferDoneCb, + OutputBufferDoneCB mOutputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb); + + // Initialize the V4L2 encoder for specified parameters. + bool initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level, + const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod); + + // Handle the next encode request on the queue. + void handleEncodeRequest(); + // Handle a request to flush the encoder. + void handleFlushRequest(); + // Handle a request to drain the encoder. + void handleDrainRequest(); + // Called when draining the encoder has completed. + void onDrainDone(bool done); + + // Configure input format on the V4L2 device. + bool configureInputFormat(VideoPixelFormat inputFormat, uint32_t stride); + // Configure output format on the V4L2 device. + bool configureOutputFormat(C2Config::profile_t outputProfile); + // Configure required and optional controls on the V4L2 device. + bool configureDevice(C2Config::profile_t outputProfile, + std::optional<const uint8_t> outputH264Level); + // Configure required and optional H.264 controls on the V4L2 device. + bool configureH264(C2Config::profile_t outputProfile, + std::optional<const uint8_t> outputH264Level); + + // Attempt to start the V4L2 device poller. + bool startDevicePoll(); + // Attempt to stop the V4L2 device poller. + bool stopDevicePoll(); + // Called by the V4L2 device poller whenever an error occurred. + void onPollError(); + // Service I/O on the V4L2 device, called by the V4L2 device poller. + void serviceDeviceTask(bool event); + + // Enqueue an input buffer to be encoded on the device input queue. Returns whether the + // operation was successful. + bool enqueueInputBuffer(std::unique_ptr<InputFrame> frame); + // Enqueue an output buffer to store the encoded bitstream on the device output queue. Returns + // whether the operation was successful. + bool enqueueOutputBuffer(); + // Dequeue an input buffer the V4L2 device has finished encoding on the device input queue. + // Returns whether a buffer could be dequeued. + bool dequeueInputBuffer(); + // Dequeue an output buffer containing the encoded bitstream from the device output queue. + // Returns whether the operation was successful. + bool dequeueOutputBuffer(); + + // Create input buffers on the V4L2 device input queue. + bool createInputBuffers(); + // Create output buffers on the V4L2 device output queue. + bool createOutputBuffers(); + // Destroy the input buffers on the V4L2 device input queue. + void destroyInputBuffers(); + // Destroy the output buffers on the V4L2 device output queue. + void destroyOutputBuffers(); + + // Notify the client an error occurred and switch to the error state. + void onError(); + + // Change the state of the encoder. + void setState(State state); + // Get the specified encoder |state| as string. + static const char* stateToString(State state); + + // The list of currently queued encode requests. + std::queue<EncodeRequest> mEncodeRequests; + + // The video stream's visible size. + ui::Size mVisibleSize; + // The video stream's coded size. + ui::Size mInputCodedSize; + // The input layout configured on the V4L2 device. + std::optional<VideoFrameLayout> mInputLayout; + // Required output buffer byte size. + uint32_t mOutputBufferSize = 0; + + // How often we want to request the V4L2 device to create a key frame. + uint32_t mKeyFramePeriod = 0; + // Key frame counter, a key frame will be requested each time it reaches zero. + uint32_t mKeyFrameCounter = 0; + + // The V4L2 device and associated queues used to interact with the device. + scoped_refptr<V4L2Device> mDevice; + scoped_refptr<V4L2Queue> mInputQueue; + scoped_refptr<V4L2Queue> mOutputQueue; + + // List of frames associated with each buffer in the V4L2 device input queue. + std::vector<std::unique_ptr<InputFrame>> mInputBuffers; + // List of bitstream buffers associated with each buffer in the V4L2 device output queue. + std::vector<std::unique_ptr<BitstreamBuffer>> mOutputBuffers; + + // Callbacks to be triggered on various events. + FetchOutputBufferCB mFetchOutputBufferCb; + InputBufferDoneCB mInputBufferDoneCb; + OutputBufferDoneCB mOutputBufferDoneCb; + DrainDoneCB mDrainDoneCb; + ErrorCB mErrorCb; + + // The current state of the encoder. + State mState = State::UNINITIALIZED; + + scoped_refptr<::base::SequencedTaskRunner> mTaskRunner; + + ::base::WeakPtr<V4L2Encoder> mWeakThis; + ::base::WeakPtrFactory<V4L2Encoder> mWeakThisFactory{this}; +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMPONENTS_V4L2_ENCODER_H diff --git a/components/include/v4l2_codec2/components/VideoDecoder.h b/components/include/v4l2_codec2/components/VideoDecoder.h index c737c65..9a48562 100644 --- a/components/include/v4l2_codec2/components/VideoDecoder.h +++ b/components/include/v4l2_codec2/components/VideoDecoder.h @@ -26,9 +26,8 @@ public: }; static const char* DecodeStatusToString(DecodeStatus status); - using GetPoolCB = - base::RepeatingCallback<void(std::unique_ptr<VideoFramePool>*, const media::Size& size, - HalPixelFormat pixelFormat, size_t numOutputBuffers)>; + using GetPoolCB = base::RepeatingCallback<std::unique_ptr<VideoFramePool>( + const ui::Size& size, HalPixelFormat pixelFormat, size_t numOutputBuffers)>; using DecodeCB = base::OnceCallback<void(DecodeStatus)>; using OutputCB = base::RepeatingCallback<void(std::unique_ptr<VideoFrame>)>; using ErrorCB = base::RepeatingCallback<void()>; diff --git a/components/include/v4l2_codec2/components/VideoEncoder.h b/components/include/v4l2_codec2/components/VideoEncoder.h new file mode 100644 index 0000000..46bcad1 --- /dev/null +++ b/components/include/v4l2_codec2/components/VideoEncoder.h @@ -0,0 +1,81 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_COMPONENTS_VIDEO_ENCODER_H +#define ANDROID_V4L2_CODEC2_COMPONENTS_VIDEO_ENCODER_H + +#include <stdint.h> +#include <memory> +#include <vector> + +#include <base/callback.h> +#include <ui/Size.h> + +#include <v4l2_codec2/common/Common.h> +#include <v4l2_codec2/common/VideoPixelFormat.h> +#include <v4l2_codec2/common/VideoTypes.h> + +namespace android { + +struct BitstreamBuffer; + +class VideoEncoder { +public: + // The InputFrame class can be used to store raw video frames. + // Note: The InputFrame does not take ownership of the data. The file descriptor is not + // duplicated and the caller is responsible for keeping the data alive until the buffer + // is returned by an InputBufferDoneCB() call. + class InputFrame { + public: + InputFrame(std::vector<int>&& fds, std::vector<VideoFramePlane>&& planes, + VideoPixelFormat pixelFormat, uint64_t index, int64_t timestamp); + ~InputFrame() = default; + + const std::vector<int>& fds() const { return mFds; } + const std::vector<VideoFramePlane>& planes() const { return mPlanes; } + VideoPixelFormat pixelFormat() const { return mPixelFormat; } + uint64_t index() const { return mIndex; } + int64_t timestamp() const { return mTimestamp; } + + private: + const std::vector<int> mFds; + std::vector<VideoFramePlane> mPlanes; + VideoPixelFormat mPixelFormat; + uint64_t mIndex = 0; + int64_t mTimestamp = 0; + }; + + using FetchOutputBufferCB = + base::RepeatingCallback<void(uint32_t, std::unique_ptr<BitstreamBuffer>* buffer)>; + // TODO(dstaessens): Change callbacks to OnceCallback provided when requesting encode/drain. + using InputBufferDoneCB = base::RepeatingCallback<void(uint64_t)>; + using OutputBufferDoneCB = base::RepeatingCallback<void( + size_t, int64_t, bool, std::unique_ptr<BitstreamBuffer> buffer)>; + using DrainDoneCB = base::RepeatingCallback<void(bool)>; + using ErrorCB = base::RepeatingCallback<void()>; + + virtual ~VideoEncoder() = default; + + // Encode the frame, |InputBufferDoneCB| and |OutputBufferDoneCB| will be called when done. + virtual bool encode(std::unique_ptr<InputFrame> buffer) = 0; + // Drain the encoder, |mDrainDoneCb| will be called when done. + virtual void drain() = 0; + // Flush the encoder, pending drain operations will be aborted. + virtual void flush() = 0; + + // Set the bitrate to the specified value, will affect all non-processed frames. + virtual bool setBitrate(uint32_t bitrate) = 0; + // Set the framerate to the specified value, will affect all non-processed frames. + virtual bool setFramerate(uint32_t framerate) = 0; + // Request the next frame encoded to be a key frame, will affect the next non-processed frame. + virtual void requestKeyframe() = 0; + + virtual VideoPixelFormat inputFormat() const = 0; + virtual const ui::Size& visibleSize() const = 0; + virtual const ui::Size& codedSize() const = 0; +}; + +} // namespace android + +#endif // ANDROID_V4L2_CODEC2_COMPONENTS_VIDEO_ENCODER_H diff --git a/components/include/v4l2_codec2/components/VideoFrame.h b/components/include/v4l2_codec2/components/VideoFrame.h index 395a52b..b5d7b99 100644 --- a/components/include/v4l2_codec2/components/VideoFrame.h +++ b/components/include/v4l2_codec2/components/VideoFrame.h @@ -10,11 +10,11 @@ #include <C2Buffer.h> -#include <rect.h> +#include <ui/Rect.h> namespace android { -// Wrap C2GraphicBlock and provide essiential information from C2GraphicBlock. +// Wrap C2GraphicBlock and provide essential information from C2GraphicBlock. class VideoFrame { public: // Create the instance from C2GraphicBlock. return nullptr if any error occurs. @@ -25,8 +25,8 @@ public: const std::vector<int>& getFDs() const; // Getter and setter of the visible rectangle. - void setVisibleRect(const media::Rect& visibleRect); - const media::Rect& getVisibleRect() const; + void setVisibleRect(const Rect& visibleRect); + const Rect& getVisibleRect() const; // Getter and setter of the bitstream ID of the corresponding input bitstream. void setBitstreamId(int32_t bitstreamId); @@ -40,7 +40,7 @@ private: std::shared_ptr<C2GraphicBlock> mGraphicBlock; std::vector<int> mFds; - media::Rect mVisibleRect; + Rect mVisibleRect; int32_t mBitstreamId = -1; }; diff --git a/components/include/v4l2_codec2/components/VideoFramePool.h b/components/include/v4l2_codec2/components/VideoFramePool.h index 71bfe27..2978eed 100644 --- a/components/include/v4l2_codec2/components/VideoFramePool.h +++ b/components/include/v4l2_codec2/components/VideoFramePool.h @@ -15,8 +15,8 @@ #include <base/memory/weak_ptr.h> #include <base/sequenced_task_runner.h> #include <base/threading/thread.h> +#include <ui/Size.h> -#include <size.h> #include <v4l2_codec2/common/VideoTypes.h> #include <v4l2_codec2/components/VideoFrame.h> @@ -31,8 +31,8 @@ public: using GetVideoFrameCB = ::base::OnceCallback<void(std::optional<FrameWithBlockId>)>; static std::unique_ptr<VideoFramePool> Create( - std::shared_ptr<C2BlockPool> blockPool, const size_t numBuffers, - const media::Size& size, HalPixelFormat pixelFormat, bool isSecure, + std::shared_ptr<C2BlockPool> blockPool, const size_t numBuffers, const ui::Size& size, + HalPixelFormat pixelFormat, bool isSecure, scoped_refptr<::base::SequencedTaskRunner> taskRunner); ~VideoFramePool(); @@ -48,8 +48,8 @@ private: // |pixelFormat| is the pixel format of the required graphic blocks. // |isSecure| indicates the video stream is encrypted or not. // All public methods and the callbacks should be run on |taskRunner|. - VideoFramePool(std::shared_ptr<C2BlockPool> blockPool, const media::Size& size, - HalPixelFormat pixelFormat, bool isSecure, + VideoFramePool(std::shared_ptr<C2BlockPool> blockPool, const ui::Size& size, + HalPixelFormat pixelFormat, C2MemoryUsage memoryUsage, scoped_refptr<::base::SequencedTaskRunner> taskRunner); bool initialize(); void destroyTask(); @@ -59,21 +59,21 @@ private: void getVideoFrameTask(); void onVideoFrameReady(std::optional<FrameWithBlockId> frameWithBlockId); - // Extracts buffer ID from graphic block. - // |block| is the graphic block allocated by |blockPool|. - static std::optional<uint32_t> getBufferIdFromGraphicBlock(const C2BlockPool& blockPool, - const C2Block2D& block); - // Ask |blockPool| to allocate the specified number of buffers. // |bufferCount| is the number of requested buffers. - static c2_status_t requestNewBufferSet(C2BlockPool& blockPool, int32_t bufferCount); + static c2_status_t requestNewBufferSet(C2BlockPool& blockPool, int32_t bufferCount, + const ui::Size& size, uint32_t format, + C2MemoryUsage usage); + + static std::optional<uint32_t> getBufferIdFromGraphicBlock(C2BlockPool& blockPool, + const C2Block2D& block); // Ask |blockPool| to notify when a block is available via |cb|. // Return true if |blockPool| supports notifying buffer available. static bool setNotifyBlockAvailableCb(C2BlockPool& blockPool, ::base::OnceClosure cb); std::shared_ptr<C2BlockPool> mBlockPool; - const media::Size mSize; + const ui::Size mSize; const HalPixelFormat mPixelFormat; const C2MemoryUsage mMemoryUsage; diff --git a/plugin_store/Android.bp b/plugin_store/Android.bp index 73dccaf..e358378 100644 --- a/plugin_store/Android.bp +++ b/plugin_store/Android.bp @@ -1,3 +1,12 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_v4l2_codec2_license" + // to get the below license kinds: + // SPDX-license-identifier-BSD + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + cc_library_shared { name: "libc2plugin_store", vendor: true, @@ -9,6 +18,8 @@ cc_library_shared { srcs: [ "C2VdaBqBlockPool.cpp", "C2VdaPooledBlockPool.cpp", + "DrmGrallocHelpers.cpp", + "H2BGraphicBufferProducer.cpp", "V4L2PluginStore.cpp", "VendorAllocatorLoader.cpp", ], @@ -17,12 +28,13 @@ cc_library_shared { ], header_libs: [ - "libcodec2_internal", + "libcodec2_internal", // needed for including C2BqBufferPriv.h. ], shared_libs: [ "android.hardware.graphics.bufferqueue@2.0", "libchrome", "libcutils", + "libdrm", "libhardware", "libhidlbase", "libnativewindow", diff --git a/plugin_store/C2VdaBqBlockPool.cpp b/plugin_store/C2VdaBqBlockPool.cpp index 9abc698..855f389 100644 --- a/plugin_store/C2VdaBqBlockPool.cpp +++ b/plugin_store/C2VdaBqBlockPool.cpp @@ -8,432 +8,446 @@ #include <v4l2_codec2/plugin_store/C2VdaBqBlockPool.h> #include <errno.h> +#include <string.h> #include <chrono> #include <mutex> +#include <set> +#include <sstream> #include <thread> #include <C2AllocatorGralloc.h> #include <C2BlockInternal.h> -#include <android/hardware/graphics/bufferqueue/2.0/IGraphicBufferProducer.h> +#include <C2SurfaceSyncObj.h> #include <android/hardware/graphics/bufferqueue/2.0/IProducerListener.h> #include <base/callback.h> #include <log/log.h> -#include <system/window.h> -#include <types.h> #include <ui/BufferQueueDefs.h> +#include <v4l2_codec2/plugin_store/DrmGrallocHelpers.h> +#include <v4l2_codec2/plugin_store/H2BGraphicBufferProducer.h> #include <v4l2_codec2/plugin_store/V4L2AllocatorId.h> namespace android { namespace { -// The wait time for acquire fence in milliseconds. -constexpr int kFenceWaitTimeMs = 10; -// The timeout limit of acquiring lock of timed_mutex in milliseconds. -constexpr std::chrono::milliseconds kTimedMutexTimeoutMs = std::chrono::milliseconds(500); +// The wait time for acquire fence in milliseconds. The normal display is 60Hz, +// which period is 16ms. We choose 2x period as timeout. +constexpr int kFenceWaitTimeMs = 32; + +// The default maximum dequeued buffer count of IGBP. Currently we don't use +// this value to restrict the count of allocated buffers, so we choose a huge +// enough value here. +constexpr int kMaxDequeuedBufferCount = 32u; } // namespace using namespace std::chrono_literals; -using ::android::C2AndroidMemoryUsage; -using ::android::Fence; -using ::android::GraphicBuffer; -using ::android::sp; -using ::android::status_t; +// We use the value of DRM handle as the unique ID of the graphic buffers. +using unique_id_t = uint32_t; +// Type for IGBP slot index. +using slot_t = int32_t; + using ::android::BufferQueueDefs::BUFFER_NEEDS_REALLOCATION; using ::android::BufferQueueDefs::NUM_BUFFER_SLOTS; -using ::android::BufferQueueDefs::RELEASE_ALL_BUFFERS; -using ::android::hardware::hidl_handle; using ::android::hardware::Return; - -using HBuffer = ::android::hardware::graphics::common::V1_2::HardwareBuffer; -using HStatus = ::android::hardware::graphics::bufferqueue::V2_0::Status; -using HGraphicBufferProducer = - ::android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer; using HProducerListener = ::android::hardware::graphics::bufferqueue::V2_0::IProducerListener; -using HConnectionType = hardware::graphics::bufferqueue::V2_0::ConnectionType; -using HQueueBufferOutput = - ::android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer::QueueBufferOutput; -using ::android::hardware::graphics::bufferqueue::V2_0::utils::b2h; -using ::android::hardware::graphics::bufferqueue::V2_0::utils::h2b; -using ::android::hardware::graphics::bufferqueue::V2_0::utils::HFenceWrapper; - -static c2_status_t asC2Error(int32_t err) { +static c2_status_t asC2Error(status_t err) { switch (err) { - case android::NO_ERROR: + case OK: return C2_OK; - case android::NO_INIT: + case NO_INIT: return C2_NO_INIT; - case android::BAD_VALUE: + case BAD_VALUE: return C2_BAD_VALUE; - case android::TIMED_OUT: + case TIMED_OUT: return C2_TIMED_OUT; - case android::WOULD_BLOCK: + case WOULD_BLOCK: return C2_BLOCKING; - case android::NO_MEMORY: + case NO_MEMORY: return C2_NO_MEMORY; } return C2_CORRUPTED; } -class H2BGraphicBufferProducer { +// Convert GraphicBuffer to C2GraphicAllocation and wrap producer id and slot index. +std::shared_ptr<C2GraphicAllocation> ConvertGraphicBuffer2C2Allocation( + sp<GraphicBuffer> graphicBuffer, const uint64_t igbpId, const slot_t slot, + C2Allocator* const allocator) { + ALOGV("%s(idbpId=0x%" PRIx64 ", slot=%d)", __func__, igbpId, slot); + + C2Handle* c2Handle = WrapNativeCodec2GrallocHandle( + graphicBuffer->handle, graphicBuffer->width, graphicBuffer->height, + graphicBuffer->format, graphicBuffer->usage, graphicBuffer->stride, + graphicBuffer->getGenerationNumber(), igbpId, slot); + if (!c2Handle) { + ALOGE("WrapNativeCodec2GrallocHandle() failed"); + return nullptr; + } + + std::shared_ptr<C2GraphicAllocation> allocation; + const auto err = allocator->priorGraphicAllocation(c2Handle, &allocation); + if (err != C2_OK) { + ALOGE("C2Allocator::priorGraphicAllocation() failed: %d", err); + native_handle_close(c2Handle); + native_handle_delete(c2Handle); + return nullptr; + } + + return allocation; +} + +// This class is used to notify the listener when a certain event happens. +class EventNotifier : public virtual android::RefBase { public: - explicit H2BGraphicBufferProducer(sp<HGraphicBufferProducer> base) : mBase(base) {} - ~H2BGraphicBufferProducer() = default; - - status_t requestBuffer(int slot, sp<GraphicBuffer>* buf) { - bool converted = false; - status_t status = UNKNOWN_ERROR; - Return<void> transResult = mBase->requestBuffer( - slot, [&converted, &status, buf](HStatus hStatus, HBuffer const& hBuffer, - uint32_t generationNumber) { - converted = h2b(hStatus, &status) && h2b(hBuffer, buf); - if (*buf) { - (*buf)->setGenerationNumber(generationNumber); - } - }); - - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!converted) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; - } - if (status != android::NO_ERROR) { - ALOGE("%s() failed: %d", __func__, status); + class Listener { + public: + virtual ~Listener() = default; + + // Called by EventNotifier when a certain event happens. + virtual void onEventNotified() = 0; + }; + + explicit EventNotifier(std::weak_ptr<Listener> listener) : mListener(std::move(listener)) {} + virtual ~EventNotifier() = default; + +protected: + void notify() { + ALOGV("%s()", __func__); + std::shared_ptr<Listener> listener = mListener.lock(); + if (listener) { + listener->onEventNotified(); } - return status; } - status_t setMaxDequeuedBufferCount(int maxDequeuedBuffers) { - status_t status = UNKNOWN_ERROR; - Return<HStatus> transResult = - mBase->setMaxDequeuedBufferCount(static_cast<int32_t>(maxDequeuedBuffers)); + std::weak_ptr<Listener> mListener; +}; - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!h2b(static_cast<HStatus>(transResult), &status)) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; - } - if (status != android::NO_ERROR) { - ALOGE("%s() failed: %d", __func__, status); - } - return status; - } - - status_t dequeueBuffer(uint32_t width, uint32_t height, uint32_t pixelFormat, - C2AndroidMemoryUsage androidUsage, int* slot, sp<Fence>* fence) { - using Input = HGraphicBufferProducer::DequeueBufferInput; - using Output = HGraphicBufferProducer::DequeueBufferOutput; - Input input{width, height, pixelFormat, androidUsage.asGrallocUsage()}; - - bool converted = false; - status_t status = UNKNOWN_ERROR; - Return<void> transResult = mBase->dequeueBuffer( - input, [&converted, &status, &slot, &fence](HStatus hStatus, int32_t hSlot, - Output const& hOutput) { - converted = h2b(hStatus, &status); - if (!converted || status != android::NO_ERROR) { - return; - } - - *slot = hSlot; - if (hOutput.bufferNeedsReallocation) { - status = BUFFER_NEEDS_REALLOCATION; - } - converted = h2b(hOutput.fence, fence); - }); - - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!converted) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; - } - if (status != android::NO_ERROR && status != BUFFER_NEEDS_REALLOCATION && - status != android::TIMED_OUT) { - ALOGE("%s() failed: %d", __func__, status); +// Notifies the listener when the connected IGBP releases buffers. +class BufferReleasedNotifier : public EventNotifier, public HProducerListener { +public: + using EventNotifier::EventNotifier; + ~BufferReleasedNotifier() override = default; + + // HProducerListener implementation + Return<void> onBuffersReleased(uint32_t count) override { + ALOGV("%s(%u)", __func__, count); + if (count > 0) { + notify(); } - return status; + return {}; } +}; - status_t detachBuffer(int slot) { - status_t status = UNKNOWN_ERROR; - Return<HStatus> transResult = mBase->detachBuffer(static_cast<int32_t>(slot)); +// IGBP expects its user (e.g. C2VdaBqBlockPool) to keep the mapping from dequeued slot index to +// graphic buffers. Also, C2VdaBqBlockPool guaratees to fetch N fixed set of buffers with buffer +// identifier. So this class stores the mapping from slot index to buffers and the mapping from +// buffer unique ID to buffers. +// This class also implements functionalities for buffer migration when surface switching. Buffers +// are owned by either component (i.e. local buffers) or CCodec framework (i.e. remote buffers). +// When switching surface, the ccodec framework migrates remote buffers to the new surfaces. Then +// C2VdaBqBlockPool migrates local buffers. However, some buffers might be lost during migration. +// We assume that there are enough buffers migrated to the new surface to continue the playback. +// After |NUM_BUFFER_SLOTS| amount of buffers are dequeued from new surface, all buffers should +// be dequeued at least once. Then we treat the missing buffer as lost, and attach these bufers to +// the new surface. +class TrackedGraphicBuffers { +public: + using value_type = std::tuple<slot_t, unique_id_t, std::shared_ptr<C2GraphicAllocation>>; - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!h2b(static_cast<HStatus>(transResult), &status)) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; - } - if (status != android::NO_ERROR) { - ALOGE("%s() failed: %d", __func__, status); - } - return status; + TrackedGraphicBuffers() = default; + ~TrackedGraphicBuffers() = default; + + void reset() { + mSlotId2GraphicBuffer.clear(); + mSlotId2PoolData.clear(); + mAllocationsRegistered.clear(); + mAllocationsToBeMigrated.clear(); + mMigrateLostBufferCounter = 0; + mGenerationToBeMigrated = 0; } - status_t attachBuffer(const sp<GraphicBuffer>& buffer, int* outSlot) { - HBuffer hBuffer; - uint32_t hGenerationNumber; - if (!b2h(buffer, &hBuffer, &hGenerationNumber)) { - ALOGE("%s: invalid input buffer.", __func__); - return BAD_VALUE; - } + void registerUniqueId(unique_id_t uniqueId, std::shared_ptr<C2GraphicAllocation> allocation) { + ALOGV("%s(uniqueId=%u)", __func__, uniqueId); + ALOG_ASSERT(allocation != nullptr); - bool converted = false; - status_t status = UNKNOWN_ERROR; - Return<void> transResult = mBase->attachBuffer( - hBuffer, hGenerationNumber, - [&converted, &status, outSlot](HStatus hStatus, int32_t hSlot, - bool releaseAllBuffers) { - converted = h2b(hStatus, &status); - *outSlot = static_cast<int>(hSlot); - if (converted && releaseAllBuffers && status == android::NO_ERROR) { - status = RELEASE_ALL_BUFFERS; - } - }); - - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!converted) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; - } - if (status != android::NO_ERROR) { - ALOGE("%s() failed: %d", __func__, status); - } - return status; + mAllocationsRegistered[uniqueId] = std::move(allocation); } - status_t cancelBuffer(int slot, const sp<Fence>& fence) { - HFenceWrapper hFenceWrapper; - if (!b2h(fence, &hFenceWrapper)) { - ALOGE("%s(): corrupted input fence.", __func__); - return UNKNOWN_ERROR; - } + std::shared_ptr<C2GraphicAllocation> getRegisteredAllocation(unique_id_t uniqueId) { + const auto iter = mAllocationsRegistered.find(uniqueId); + ALOG_ASSERT(iter != mAllocationsRegistered.end()); - status_t status = UNKNOWN_ERROR; - Return<HStatus> transResult = - mBase->cancelBuffer(static_cast<int32_t>(slot), hFenceWrapper.getHandle()); + return iter->second; + } - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; - } - if (!h2b(static_cast<HStatus>(transResult), &status)) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; + bool hasUniqueId(unique_id_t uniqueId) const { + return mAllocationsRegistered.find(uniqueId) != mAllocationsRegistered.end() || + mAllocationsToBeMigrated.find(uniqueId) != mAllocationsToBeMigrated.end(); + } + + void updateSlotBuffer(slot_t slotId, unique_id_t uniqueId, sp<GraphicBuffer> slotBuffer) { + ALOGV("%s(slotId=%d)", __func__, slotId); + ALOG_ASSERT(slotBuffer != nullptr); + + mSlotId2GraphicBuffer[slotId] = std::make_pair(uniqueId, std::move(slotBuffer)); + } + + std::pair<unique_id_t, sp<GraphicBuffer>> getSlotBuffer(slot_t slotId) const { + const auto iter = mSlotId2GraphicBuffer.find(slotId); + ALOG_ASSERT(iter != mSlotId2GraphicBuffer.end()); + + return iter->second; + } + + bool hasSlotId(slot_t slotId) const { + return mSlotId2GraphicBuffer.find(slotId) != mSlotId2GraphicBuffer.end(); + } + + void updatePoolData(slot_t slotId, std::weak_ptr<C2BufferQueueBlockPoolData> poolData) { + ALOGV("%s(slotId=%d)", __func__, slotId); + ALOG_ASSERT(hasSlotId(slotId)); + + mSlotId2PoolData[slotId] = std::move(poolData); + } + + bool migrateLocalBuffers(H2BGraphicBufferProducer* const producer, uint64_t producerId, + uint32_t generation, uint64_t usage) { + ALOGV("%s(producerId=%" PRIx64 ", generation=%u, usage=%" PRIx64 ")", __func__, producerId, + generation, usage); + + mGenerationToBeMigrated = generation; + mUsageToBeMigrated = usage; + + // Move all buffers to mAllocationsToBeMigrated. + for (auto& pair : mAllocationsRegistered) { + if (!mAllocationsToBeMigrated.insert(pair).second) { + ALOGE("%s() duplicated uniqueId=%u", __func__, pair.first); + return false; + } } - if (status != android::NO_ERROR) { - ALOGE("%s() failed: %d", __func__, status); + mAllocationsRegistered.clear(); + + ALOGV("%s(producerId=%" PRIx64 ", generation=%u, usage=%" PRIx64 ") before %s", __func__, + producerId, generation, usage, debugString().c_str()); + + // Migrate local buffers. + std::map<slot_t, std::pair<unique_id_t, sp<GraphicBuffer>>> newSlotId2GraphicBuffer; + std::map<slot_t, std::weak_ptr<C2BufferQueueBlockPoolData>> newSlotId2PoolData; + for (const auto& pair : mSlotId2PoolData) { + auto oldSlot = pair.first; + auto poolData = pair.second.lock(); + if (!poolData) { + continue; + } + + unique_id_t uniqueId; + sp<GraphicBuffer> slotBuffer; + std::shared_ptr<C2SurfaceSyncMemory> syncMem; + std::tie(uniqueId, slotBuffer) = getSlotBuffer(oldSlot); + slot_t newSlot = poolData->migrate(producer->getBase(), mGenerationToBeMigrated, + mUsageToBeMigrated, producerId, slotBuffer, + slotBuffer->getGenerationNumber(), + syncMem); + if (newSlot < 0) { + ALOGW("%s() Failed to migrate local buffer: uniqueId=%u, oldSlot=%d", __func__, + uniqueId, oldSlot); + continue; + } + + ALOGV("%s() migrated buffer: uniqueId=%u, oldSlot=%d, newSlot=%d", __func__, uniqueId, + oldSlot, newSlot); + newSlotId2GraphicBuffer[newSlot] = std::make_pair(uniqueId, std::move(slotBuffer)); + newSlotId2PoolData[newSlot] = std::move(poolData); + + if (!moveBufferToRegistered(uniqueId)) { + ALOGE("%s() failed to move buffer to registered, uniqueId=%u", __func__, uniqueId); + return false; + } } - return status; + mSlotId2GraphicBuffer = std::move(newSlotId2GraphicBuffer); + mSlotId2PoolData = std::move(newSlotId2PoolData); + + // Choose a big enough number to ensure all buffer should be dequeued at least once. + mMigrateLostBufferCounter = NUM_BUFFER_SLOTS; + ALOGD("%s() migrated %zu local buffers", __func__, mAllocationsRegistered.size()); + return true; + } + + bool needMigrateLostBuffers() const { + return mMigrateLostBufferCounter == 0 && !mAllocationsToBeMigrated.empty(); } - int query(int what, int* value) { - int result = 0; - Return<void> transResult = - mBase->query(static_cast<int32_t>(what), [&result, value](int32_t r, int32_t v) { - result = static_cast<int>(r); - *value = static_cast<int>(v); - }); + status_t migrateLostBuffer(C2Allocator* const allocator, + H2BGraphicBufferProducer* const producer, const uint64_t producerId, + slot_t* newSlot) { + ALOGV("%s() %s", __func__, debugString().c_str()); - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; + if (!needMigrateLostBuffers()) { + return NO_INIT; } - return result; - } - status_t allowAllocation(bool allow) { - status_t status = UNKNOWN_ERROR; - Return<HStatus> transResult = mBase->allowAllocation(allow); + auto iter = mAllocationsToBeMigrated.begin(); + const unique_id_t uniqueId = iter->first; + const C2Handle* c2Handle = iter->second->handle(); - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; + // Convert C2GraphicAllocation to GraphicBuffer, and update generation and usage. + uint32_t width, height, format, stride, igbpSlot, generation; + uint64_t usage, igbpId; + _UnwrapNativeCodec2GrallocMetadata(c2Handle, &width, &height, &format, &usage, &stride, + &generation, &igbpId, &igbpSlot); + native_handle_t* grallocHandle = UnwrapNativeCodec2GrallocHandle(c2Handle); + sp<GraphicBuffer> graphicBuffer = + new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width, height, format, + 1, mUsageToBeMigrated, stride); + native_handle_delete(grallocHandle); + if (graphicBuffer->initCheck() != android::NO_ERROR) { + ALOGE("Failed to create GraphicBuffer: %d", graphicBuffer->initCheck()); + return false; + } + graphicBuffer->setGenerationNumber(mGenerationToBeMigrated); + + // Attach GraphicBuffer to producer. + const auto attachStatus = producer->attachBuffer(graphicBuffer, newSlot); + if (attachStatus == TIMED_OUT || attachStatus == INVALID_OPERATION) { + ALOGV("%s(): No free slot yet.", __func__); + return TIMED_OUT; } - if (!h2b(static_cast<HStatus>(transResult), &status)) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; + if (attachStatus != OK) { + ALOGE("%s(): Failed to attach buffer to new producer: %d", __func__, attachStatus); + return attachStatus; } - if (status != android::NO_ERROR) { - ALOGW("%s() failed: %d", __func__, status); + ALOGD("%s(), migrated lost buffer uniqueId=%u to slot=%d", __func__, uniqueId, *newSlot); + updateSlotBuffer(*newSlot, uniqueId, graphicBuffer); + + // Wrap the new GraphicBuffer to C2GraphicAllocation and register it. + std::shared_ptr<C2GraphicAllocation> allocation = + ConvertGraphicBuffer2C2Allocation(graphicBuffer, producerId, *newSlot, allocator); + if (!allocation) { + return UNKNOWN_ERROR; } - return status; + registerUniqueId(uniqueId, std::move(allocation)); + + // Note: C2ArcProtectedGraphicAllocator releases the protected buffers if all the + // corrresponding C2GraphicAllocations are released. To prevent the protected buffer is + // released and then allocated again, we release the old C2GraphicAllocation after the new + // one has been created. + mAllocationsToBeMigrated.erase(iter); + + return OK; } - status_t getUniqueId(uint64_t* outId) const { - Return<uint64_t> transResult = mBase->getUniqueId(); + void onBufferDequeued(slot_t slotId) { + ALOGV("%s(slotId=%d)", __func__, slotId); + unique_id_t uniqueId; + std::tie(uniqueId, std::ignore) = getSlotBuffer(slotId); - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; + moveBufferToRegistered(uniqueId); + if (mMigrateLostBufferCounter > 0) { + --mMigrateLostBufferCounter; } + } - *outId = static_cast<uint64_t>(transResult); - return android::NO_ERROR; - } - - // android::IProducerListener cannot be depended by vendor library, so we use HProducerListener - // directly. - status_t connect(sp<HProducerListener> const& hListener, int32_t api, - bool producerControlledByApp) { - bool converted = false; - status_t status = UNKNOWN_ERROR; - // hack(b/146409777): We pass self-defined api, so we don't use b2h() here. - Return<void> transResult = mBase->connect( - hListener, static_cast<HConnectionType>(api), producerControlledByApp, - [&converted, &status](HStatus hStatus, HQueueBufferOutput const& /* hOutput */) { - converted = h2b(hStatus, &status); - }); - - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; + size_t size() const { return mAllocationsRegistered.size() + mAllocationsToBeMigrated.size(); } + + std::string debugString() const { + std::stringstream ss; + ss << "tracked size: " << size() << std::endl; + ss << " registered uniqueIds: "; + for (const auto& pair : mAllocationsRegistered) { + ss << pair.first << ", "; } - if (!converted) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; + ss << std::endl; + ss << " to-be-migrated uniqueIds: "; + for (const auto& pair : mAllocationsToBeMigrated) { + ss << pair.first << ", "; } - return status; + ss << std::endl; + ss << " Count down for lost buffer migration: " << mMigrateLostBufferCounter; + return ss.str(); } - status_t setDequeueTimeout(nsecs_t timeout) { - status_t status = UNKNOWN_ERROR; - Return<HStatus> transResult = mBase->setDequeueTimeout(static_cast<int64_t>(timeout)); - - if (!transResult.isOk()) { - ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); - return FAILED_TRANSACTION; +private: + bool moveBufferToRegistered(unique_id_t uniqueId) { + ALOGV("%s(uniqueId=%u)", __func__, uniqueId); + auto iter = mAllocationsToBeMigrated.find(uniqueId); + if (iter == mAllocationsToBeMigrated.end()) { + return false; } - if (!h2b(static_cast<HStatus>(transResult), &status)) { - ALOGE("%s(): corrupted transaction.", __func__); - return FAILED_TRANSACTION; + if (!mAllocationsRegistered.insert(*iter).second) { + ALOGE("%s() duplicated uniqueId=%u", __func__, uniqueId); + return false; } - return status; + mAllocationsToBeMigrated.erase(iter); + + return true; } -private: - const sp<HGraphicBufferProducer> mBase; -}; + // Mapping from IGBP slots to the corresponding graphic buffers. + std::map<slot_t, std::pair<unique_id_t, sp<GraphicBuffer>>> mSlotId2GraphicBuffer; -// This class is used to notify the listener when a certain event happens. -class EventNotifier : public virtual android::RefBase { -public: - class Listener { - public: - virtual ~Listener() = default; + // Mapping from IGBP slots to the corresponding pool data. + std::map<slot_t, std::weak_ptr<C2BufferQueueBlockPoolData>> mSlotId2PoolData; - // Called by EventNotifier when a certain event happens. - virtual void onEventNotified() = 0; - }; + // Track the buffers registered at the current producer. + std::map<unique_id_t, std::shared_ptr<C2GraphicAllocation>> mAllocationsRegistered; - explicit EventNotifier(const std::shared_ptr<Listener>& listener) : mListener(listener) {} - virtual ~EventNotifier() = default; + // Track the buffers that should be migrated to the current producer. + std::map<unique_id_t, std::shared_ptr<C2GraphicAllocation>> mAllocationsToBeMigrated; -protected: - void notify() { - ALOGV("%s()", __func__); - std::shared_ptr<Listener> listener = mListener.lock(); - if (listener) { - listener->onEventNotified(); - } - } + // The counter for migrating lost buffers. Count down when a buffer is + // dequeued from IGBP. When it goes to 0, then we treat the remaining + // buffers at |mAllocationsToBeMigrated| lost, and migrate them to + // current IGBP. + size_t mMigrateLostBufferCounter = 0; - std::weak_ptr<Listener> mListener; + // The generation and usage of the current IGBP, used to migrate buffers. + uint32_t mGenerationToBeMigrated = 0; + uint64_t mUsageToBeMigrated = 0; }; -// Notifies the listener when the connected IGBP releases buffers. -class BufferReleasedNotifier : public EventNotifier, public HProducerListener { +class DrmHandleManager { public: - using EventNotifier::EventNotifier; - ~BufferReleasedNotifier() override = default; + DrmHandleManager() { mRenderFd = openRenderFd(); } - // HProducerListener implementation - Return<void> onBuffersReleased(uint32_t count) override { - ALOGV("%s(%u)", __func__, count); - if (count > 0) { - notify(); + ~DrmHandleManager() { + closeAllHandles(); + if (mRenderFd) { + close(*mRenderFd); } - return {}; } -}; -/** - * BlockPoolData implementation for C2VdaBqBlockPool. The life cycle of this object should be as - * long as its accompanied C2GraphicBlock. - * - * When C2VdaBqBlockPoolData is created, |mShared| is false, and the owner of the accompanied - * C2GraphicBlock is the component that called fetchGraphicBlock(). If this is released before - * sharing, the destructor will call detachBuffer() to BufferQueue to free the slot. - * - * When the accompanied C2GraphicBlock is going to share to client from component, component should - * call MarkBlockPoolDataAsShared() to set |mShared| to true, and then this will be released after - * the transition of C2GraphicBlock across HIDL interface. At this time, the destructor will not - * call detachBuffer(). - */ -struct C2VdaBqBlockPoolData : public _C2BlockPoolData { - // This type should be a different value than what _C2BlockPoolData::type_t has defined. - static constexpr int kTypeVdaBufferQueue = TYPE_BUFFERQUEUE + 256; - - C2VdaBqBlockPoolData(uint64_t producerId, int32_t slotId, - const std::shared_ptr<C2VdaBqBlockPool::Impl>& pool); - C2VdaBqBlockPoolData() = delete; - - // If |mShared| is false, call detach buffer to BufferQueue via |mPool| - virtual ~C2VdaBqBlockPoolData() override; - - type_t getType() const override { return static_cast<type_t>(kTypeVdaBufferQueue); } - - bool mShared = false; // whether is shared from component to client. - const uint64_t mProducerId; - const int32_t mSlotId; - const std::shared_ptr<C2VdaBqBlockPool::Impl> mPool; -}; + std::optional<unique_id_t> getHandle(int primeFd) { + if (!mRenderFd) { + return std::nullopt; + } -c2_status_t MarkBlockPoolDataAsShared(const C2ConstGraphicBlock& sharedBlock) { - std::shared_ptr<_C2BlockPoolData> data = _C2BlockFactory::GetGraphicBlockPoolData(sharedBlock); - if (!data || data->getType() != C2VdaBqBlockPoolData::kTypeVdaBufferQueue) { - // Skip this functtion if |sharedBlock| is not fetched from C2VdaBqBlockPool. - return C2_OMITTED; + std::optional<unique_id_t> handle = getDrmHandle(*mRenderFd, primeFd); + // Defer closing the handle until we don't need the buffer to keep the returned DRM handle + // the same. + if (handle) { + mHandles.insert(*handle); + } + return handle; } - const std::shared_ptr<C2VdaBqBlockPoolData> poolData = - std::static_pointer_cast<C2VdaBqBlockPoolData>(data); - if (poolData->mShared) { - ALOGE("C2VdaBqBlockPoolData(id=%" PRIu64 ", slot=%d) is already marked as shared...", - poolData->mProducerId, poolData->mSlotId); - return C2_BAD_STATE; + + void closeAllHandles() { + if (!mRenderFd) { + return; + } + + for (const unique_id_t& handle : mHandles) { + closeDrmHandle(*mRenderFd, handle); + } + mHandles.clear(); } - poolData->mShared = true; - return C2_OK; -} -// static -std::optional<uint32_t> C2VdaBqBlockPool::getBufferIdFromGraphicBlock(const C2Block2D& block) { - uint32_t width, height, format, stride, igbp_slot, generation; - uint64_t usage, igbp_id; - android::_UnwrapNativeCodec2GrallocMetadata(block.handle(), &width, &height, &format, &usage, - &stride, &generation, &igbp_id, &igbp_slot); - ALOGV("Unwrap Metadata: igbp[%" PRIu64 ", %u] (%u*%u, fmt %#x, usage %" PRIx64 ", stride %u)", - igbp_id, igbp_slot, width, height, format, usage, stride); - return igbp_slot; -} +private: + std::optional<int> mRenderFd; + std::set<unique_id_t> mHandles; +}; class C2VdaBqBlockPool::Impl : public std::enable_shared_from_this<C2VdaBqBlockPool::Impl>, public EventNotifier::Listener { @@ -452,15 +466,12 @@ public: std::shared_ptr<C2GraphicBlock>* block /* nonnull */); void setRenderCallback(const C2BufferQueueBlockPool::OnRenderCallback& renderCallback); void configureProducer(const sp<HGraphicBufferProducer>& producer); - c2_status_t requestNewBufferSet(int32_t bufferCount); - c2_status_t updateGraphicBlock(bool willCancel, uint32_t oldSlot, uint32_t* newSlot, - std::shared_ptr<C2GraphicBlock>* block /* nonnull */); - c2_status_t getMinBuffersForDisplay(size_t* bufferCount); + c2_status_t requestNewBufferSet(int32_t bufferCount, uint32_t width, uint32_t height, + uint32_t format, C2MemoryUsage usage); bool setNotifyBlockAvailableCb(::base::OnceClosure cb); + std::optional<unique_id_t> getBufferIdFromGraphicBlock(const C2Block2D& block); private: - friend struct C2VdaBqBlockPoolData; - // Requested buffer formats. struct BufferFormat { BufferFormat(uint32_t width, uint32_t height, uint32_t pixelFormat, @@ -474,133 +485,145 @@ private: C2AndroidMemoryUsage mUsage = C2MemoryUsage(0); }; - // For C2VdaBqBlockPoolData to detach corresponding slot buffer from BufferQueue. - void detachBuffer(uint64_t producerId, int32_t slotId); + status_t getFreeSlotLocked(uint32_t width, uint32_t height, uint32_t format, + C2MemoryUsage usage, slot_t* slot, sp<Fence>* fence); // Queries the generation and usage flags from the given producer by dequeuing and requesting a // buffer (the buffer is then detached and freed). - c2_status_t queryGenerationAndUsage(H2BGraphicBufferProducer* const producer, uint32_t width, - uint32_t height, uint32_t pixelFormat, - C2AndroidMemoryUsage androidUsage, uint32_t* generation, - uint64_t* usage); + status_t queryGenerationAndUsageLocked(uint32_t width, uint32_t height, uint32_t pixelFormat, + C2AndroidMemoryUsage androidUsage, uint32_t* generation, + uint64_t* usage); + + // Wait the fence. If any error occurs, cancel the buffer back to the producer. + status_t waitFence(slot_t slot, sp<Fence> fence); - // Switches producer and transfers allocated buffers from old producer to the new one. - bool switchProducer(H2BGraphicBufferProducer* const newProducer, uint64_t newProducerId); + // Call mProducer's allowAllocation if needed. + status_t allowAllocation(bool allow); const std::shared_ptr<C2Allocator> mAllocator; std::unique_ptr<H2BGraphicBufferProducer> mProducer; - uint64_t mProducerId; + uint64_t mProducerId = 0; + bool mAllowAllocation = false; + C2BufferQueueBlockPool::OnRenderCallback mRenderCallback; // Function mutex to lock at the start of each API function call for protecting the // synchronization of all member variables. std::mutex mMutex; - // The mutex of excluding the procedures of configuring producer and allocating buffers. They - // should be blocked mutually. Set the timeout for acquiring lock in case of any deadlock. - // Configuring producer: configureProducer() called by CCodec. - // Allocating buffers: requestNewBufferSet(), then a loop of fetchGraphicBlock() called by - // compoenent until |mSlotAllocations|.size() equals |mBuffersRequested|. - std::timed_mutex mConfigureProducerAndAllocateBuffersMutex; - // The unique lock of the procedure of allocating buffers. It should be locked in the beginning - // of requestNewBufferSet() and unlock in the end of the loop of fetchGraphicBlock(). Note that - // all calls should be in the same thread. - std::unique_lock<std::timed_mutex> mAllocateBuffersLock; - - // The map restored C2GraphicAllocation from corresponding slot index. - std::map<int32_t, std::shared_ptr<C2GraphicAllocation>> mSlotAllocations; + + TrackedGraphicBuffers mTrackedGraphicBuffers; + + // We treat DRM handle as uniqueId of GraphicBuffer. + DrmHandleManager mDrmHandleManager; + // Number of buffers requested on requestNewBufferSet() call. - size_t mBuffersRequested; + size_t mBuffersRequested = 0u; // Currently requested buffer formats. BufferFormat mBufferFormat; - // The map recorded the slot indices from old producer to new producer. - std::map<int32_t, int32_t> mProducerChangeSlotMap; - // The counter for representing the buffer count in client. Only used in producer switching - // case. It will be reset in switchProducer(), and accumulated in updateGraphicBlock() routine. - uint32_t mBuffersInClient = 0u; - // The indicator to record if producer has been switched. Set to true when producer is switched. - // Toggle off when requestNewBufferSet() is called. We forcedly detach all slots to make sure - // all slots are available, except the ones owned by client. - bool mProducerSwitched = false; // Listener for buffer release events. sp<EventNotifier> mFetchBufferNotifier; std::mutex mBufferReleaseMutex; - // Set to true when the buffer release event is triggered after dequeueing - // buffer from IGBP times out. + // Set to true when the buffer release event is triggered after dequeueing buffer from IGBP + // times out. Reset when fetching new slot times out, or |mNotifyBlockAvailableCb| is executed. bool mBufferReleasedAfterTimedOut GUARDED_BY(mBufferReleaseMutex) = false; // The callback to notify the caller the buffer is available. ::base::OnceClosure mNotifyBlockAvailableCb GUARDED_BY(mBufferReleaseMutex); + + // Set to true if any error occurs at previous configureProducer(). + bool mConfigureProducerError = false; }; C2VdaBqBlockPool::Impl::Impl(const std::shared_ptr<C2Allocator>& allocator) - : mAllocator(allocator), - mAllocateBuffersLock(mConfigureProducerAndAllocateBuffersMutex, std::defer_lock), - mBuffersRequested(0u) {} + : mAllocator(allocator) {} c2_status_t C2VdaBqBlockPool::Impl::fetchGraphicBlock( uint32_t width, uint32_t height, uint32_t format, C2MemoryUsage usage, std::shared_ptr<C2GraphicBlock>* block /* nonnull */) { - ALOGV("%s()", __func__); + ALOGV("%s(%ux%u)", __func__, width, height); std::lock_guard<std::mutex> lock(mMutex); - if (!mProducer) { - // Producer will not be configured in byte-buffer mode. Allocate buffers from allocator - // directly as a basic graphic block pool. - std::shared_ptr<C2GraphicAllocation> alloc; - c2_status_t err = mAllocator->newGraphicAllocation(width, height, format, usage, &alloc); - if (err != C2_OK) { - return err; - } - *block = _C2BlockFactory::CreateGraphicBlock(alloc); - return C2_OK; + if (width != mBufferFormat.mWidth || height != mBufferFormat.mHeight || + format != mBufferFormat.mPixelFormat || usage.expected != mBufferFormat.mUsage.expected) { + ALOGE("%s(): buffer format (%ux%u, format=%u, usage=%" PRIx64 + ") is different from requested format (%ux%u, format=%u, usage=%" PRIx64 ")", + __func__, width, height, format, usage.expected, mBufferFormat.mWidth, + mBufferFormat.mHeight, mBufferFormat.mPixelFormat, mBufferFormat.mUsage.expected); + return C2_BAD_VALUE; } - - // The existence of |mProducerChangeSlotMap| indicates producer is just switched. Use return - // code C2_BAD_STATE to inform the component to handle the procedure of producer change. - // TODO(johnylin): consider to inform producer change to component in an active way. - if (!mProducerChangeSlotMap.empty()) { - return C2_BAD_STATE; + if (mConfigureProducerError || !mProducer) { + ALOGE("%s(): error occurred at previous configureProducer()", __func__); + return C2_CORRUPTED; } - C2AndroidMemoryUsage androidUsage = usage; - uint32_t pixelFormat = format; - int32_t slot; + slot_t slot; sp<Fence> fence = new Fence(); - status_t status = - mProducer->dequeueBuffer(width, height, pixelFormat, androidUsage, &slot, &fence); - // The C2VdaBqBlockPool does not fully own the bufferqueue. After buffers are dequeued here, - // they are passed into the codec2 framework, processed, and eventually queued into the - // bufferqueue. The C2VdaBqBlockPool cannot determine exactly when a buffer gets queued. - // However, if every buffer is being processed by the codec2 framework, then dequeueBuffer() - // will return INVALID_OPERATION because of an attempt to dequeue too many buffers. - // The C2VdaBqBlockPool cannot prevent this from happening, so just map it to TIMED_OUT - // and let the C2VdaBqBlockPool's caller's timeout retry logic handle the failure. - if (status == android::INVALID_OPERATION) { - status = android::TIMED_OUT; - } - if (status == android::TIMED_OUT) { - std::lock_guard<std::mutex> lock(mBufferReleaseMutex); - mBufferReleasedAfterTimedOut = false; - } - if (status != android::NO_ERROR && status != BUFFER_NEEDS_REALLOCATION) { + const auto status = getFreeSlotLocked(width, height, format, usage, &slot, &fence); + if (status != OK) { return asC2Error(status); } - // Wait for acquire fence if we get one. - if (fence) { - status_t fenceStatus = fence->wait(kFenceWaitTimeMs); - if (fenceStatus != android::NO_ERROR) { - if (mProducer->cancelBuffer(slot, fence) != android::NO_ERROR) { + unique_id_t uniqueId; + sp<GraphicBuffer> slotBuffer; + std::tie(uniqueId, slotBuffer) = mTrackedGraphicBuffers.getSlotBuffer(slot); + ALOGV("%s(): dequeued slot=%d uniqueId=%u", __func__, slot, uniqueId); + + if (!mTrackedGraphicBuffers.hasUniqueId(uniqueId)) { + if (mTrackedGraphicBuffers.size() >= mBuffersRequested) { + // The dequeued slot has a pre-allocated buffer whose size and format is as same as + // currently requested (but was not dequeued during allocation cycle). Just detach it to + // free this slot. And try dequeueBuffer again. + ALOGD("dequeued a new slot %d but already allocated enough buffers. Detach it.", slot); + + if (mProducer->detachBuffer(slot) != OK) { return C2_CORRUPTED; } - if (fenceStatus == -ETIME) { // fence wait timed out - ALOGV("%s(): buffer (slot=%d) fence wait timed out", __func__, slot); - return C2_TIMED_OUT; + const auto allocationStatus = allowAllocation(false); + if (allocationStatus != OK) { + return asC2Error(allocationStatus); } - ALOGE("buffer fence wait error: %d", fenceStatus); + return C2_TIMED_OUT; + } + + std::shared_ptr<C2GraphicAllocation> allocation = + ConvertGraphicBuffer2C2Allocation(slotBuffer, mProducerId, slot, mAllocator.get()); + if (!allocation) { + return C2_CORRUPTED; + } + mTrackedGraphicBuffers.registerUniqueId(uniqueId, std::move(allocation)); + + ALOGV("%s(): mTrackedGraphicBuffers.size=%zu", __func__, mTrackedGraphicBuffers.size()); + if (mTrackedGraphicBuffers.size() == mBuffersRequested) { + ALOGV("Tracked IGBP slots: %s", mTrackedGraphicBuffers.debugString().c_str()); + // Already allocated enough buffers, set allowAllocation to false to restrict the + // eligible slots to allocated ones for future dequeue. + const auto allocationStatus = allowAllocation(false); + if (allocationStatus != OK) { + return asC2Error(allocationStatus); + } + } + } + + std::shared_ptr<C2SurfaceSyncMemory> syncMem; + std::shared_ptr<C2GraphicAllocation> allocation = + mTrackedGraphicBuffers.getRegisteredAllocation(uniqueId); + auto poolData = std::make_shared<C2BufferQueueBlockPoolData>( + slotBuffer->getGenerationNumber(), mProducerId, slot, + mProducer->getBase(), syncMem, 0); + mTrackedGraphicBuffers.updatePoolData(slot, poolData); + *block = _C2BlockFactory::CreateGraphicBlock(std::move(allocation), std::move(poolData)); + if (*block == nullptr) { + ALOGE("failed to create GraphicBlock: no memory"); + return C2_NO_MEMORY; + } + + // Wait for acquire fence at the last point of returning buffer. + if (fence) { + const auto fenceStatus = waitFence(slot, fence); + if (fenceStatus != OK) { return asC2Error(fenceStatus); } @@ -614,73 +637,54 @@ c2_status_t C2VdaBqBlockPool::Impl::fetchGraphicBlock( } } - auto iter = mSlotAllocations.find(slot); - if (iter == mSlotAllocations.end()) { - if (mSlotAllocations.size() >= mBuffersRequested) { - // The dequeued slot has a pre-allocated buffer whose size and format is as same as - // currently requested (but was not dequeued during allocation cycle). Just detach it to - // free this slot. And try dequeueBuffer again. - ALOGD("dequeued a new slot index but already allocated enough buffers. Detach it."); - - if (mProducer->detachBuffer(slot) != android::NO_ERROR) { - return C2_CORRUPTED; - } - return C2_TIMED_OUT; - } - if (status != BUFFER_NEEDS_REALLOCATION) { - // The dequeued slot has a pre-allocated buffer whose size and format is as same as - // currently requested, so there is no BUFFER_NEEDS_REALLOCATION flag. However since the - // buffer reference is already dropped, still call requestBuffer to re-allocate then. - // Add a debug note here for tracking. - ALOGD("dequeued a new slot index without BUFFER_NEEDS_REALLOCATION flag."); - } + return C2_OK; +} - // Call requestBuffer to allocate buffer for the slot and obtain the reference. - sp<GraphicBuffer> slotBuffer = new GraphicBuffer(); - status = mProducer->requestBuffer(slot, &slotBuffer); - if (status != android::NO_ERROR) { - if (mProducer->cancelBuffer(slot, fence) != android::NO_ERROR) { - return C2_CORRUPTED; - } - return asC2Error(status); +status_t C2VdaBqBlockPool::Impl::getFreeSlotLocked(uint32_t width, uint32_t height, uint32_t format, + C2MemoryUsage usage, slot_t* slot, + sp<Fence>* fence) { + if (mTrackedGraphicBuffers.needMigrateLostBuffers()) { + slot_t newSlot; + if (mTrackedGraphicBuffers.migrateLostBuffer(mAllocator.get(), mProducer.get(), mProducerId, + &newSlot) == OK) { + ALOGV("%s(): migrated buffer: slot=%d", __func__, newSlot); + *slot = newSlot; + return OK; } + } - // Convert GraphicBuffer to C2GraphicAllocation and wrap producer id and slot index - ALOGV("buffer wraps { producer id: %" PRIu64 ", slot: %d }", mProducerId, slot); - C2Handle* c2Handle = android::WrapNativeCodec2GrallocHandle( - slotBuffer->handle, slotBuffer->width, slotBuffer->height, slotBuffer->format, - slotBuffer->usage, slotBuffer->stride, slotBuffer->getGenerationNumber(), - mProducerId, slot); - if (!c2Handle) { - ALOGE("WrapNativeCodec2GrallocHandle failed"); - return C2_NO_MEMORY; - } + // Dequeue a free slot from IGBP. + ALOGV("%s(): try to dequeue free slot from IGBP.", __func__); + const auto dequeueStatus = mProducer->dequeueBuffer(width, height, format, usage, slot, fence); + if (dequeueStatus == TIMED_OUT) { + std::lock_guard<std::mutex> lock(mBufferReleaseMutex); + mBufferReleasedAfterTimedOut = false; + } + if (dequeueStatus != OK && dequeueStatus != BUFFER_NEEDS_REALLOCATION) { + return dequeueStatus; + } - std::shared_ptr<C2GraphicAllocation> alloc; - c2_status_t err = mAllocator->priorGraphicAllocation(c2Handle, &alloc); - if (err != C2_OK) { - ALOGE("priorGraphicAllocation failed: %d", err); - return err; + // Call requestBuffer to update GraphicBuffer for the slot and obtain the reference. + if (!mTrackedGraphicBuffers.hasSlotId(*slot) || dequeueStatus == BUFFER_NEEDS_REALLOCATION) { + sp<GraphicBuffer> slotBuffer = new GraphicBuffer(); + const auto requestStatus = mProducer->requestBuffer(*slot, &slotBuffer); + if (requestStatus != OK) { + mProducer->cancelBuffer(*slot, *fence); + return requestStatus; } - mSlotAllocations[slot] = std::move(alloc); - if (mSlotAllocations.size() == mBuffersRequested) { - // Already allocated enough buffers, set allowAllocation to false to restrict the - // eligible slots to allocated ones for future dequeue. - status = mProducer->allowAllocation(false); - if (status != android::NO_ERROR) { - return asC2Error(status); - } - // Store buffer formats for future usage. - mBufferFormat = BufferFormat(width, height, pixelFormat, androidUsage); - ALOG_ASSERT(mAllocateBuffersLock.owns_lock()); - mAllocateBuffersLock.unlock(); + const auto uniqueId = mDrmHandleManager.getHandle(slotBuffer->handle->data[0]); + if (!uniqueId) { + ALOGE("%s(): failed to get uniqueId of GraphicBuffer from slot=%d", __func__, *slot); + return UNKNOWN_ERROR; } + mTrackedGraphicBuffers.updateSlotBuffer(*slot, *uniqueId, std::move(slotBuffer)); } - auto poolData = std::make_shared<C2VdaBqBlockPoolData>(mProducerId, slot, shared_from_this()); - *block = _C2BlockFactory::CreateGraphicBlock(mSlotAllocations[slot], std::move(poolData)); - return C2_OK; + ALOGV("%s(%ux%u): dequeued slot=%d", __func__, mBufferFormat.mWidth, mBufferFormat.mHeight, + *slot); + mTrackedGraphicBuffers.onBufferDequeued(*slot); + return OK; } void C2VdaBqBlockPool::Impl::onEventNotified() { @@ -691,6 +695,7 @@ void C2VdaBqBlockPool::Impl::onEventNotified() { mBufferReleasedAfterTimedOut = true; if (mNotifyBlockAvailableCb) { + mBufferReleasedAfterTimedOut = false; outputCb = std::move(mNotifyBlockAvailableCb); } } @@ -701,56 +706,62 @@ void C2VdaBqBlockPool::Impl::onEventNotified() { } } -c2_status_t C2VdaBqBlockPool::Impl::queryGenerationAndUsage( - H2BGraphicBufferProducer* const producer, uint32_t width, uint32_t height, - uint32_t pixelFormat, C2AndroidMemoryUsage androidUsage, uint32_t* generation, - uint64_t* usage) { - ALOGV("queryGenerationAndUsage"); - sp<Fence> fence = new Fence(); - int32_t status; - int32_t slot; - - status = producer->dequeueBuffer(width, height, pixelFormat, androidUsage, &slot, &fence); - if (status != android::NO_ERROR && status != BUFFER_NEEDS_REALLOCATION) { - return asC2Error(status); - } +status_t C2VdaBqBlockPool::Impl::queryGenerationAndUsageLocked(uint32_t width, uint32_t height, + uint32_t pixelFormat, + C2AndroidMemoryUsage androidUsage, + uint32_t* generation, + uint64_t* usage) { + ALOGV("%s()", __func__); - // Wait for acquire fence if we get one. - if (fence) { - status_t fenceStatus = fence->wait(kFenceWaitTimeMs); - if (fenceStatus != android::NO_ERROR) { - if (producer->cancelBuffer(slot, fence) != android::NO_ERROR) { - return C2_CORRUPTED; - } - if (fenceStatus == -ETIME) { // fence wait timed out - ALOGV("%s(): buffer (slot=%d) fence wait timed out", __func__, slot); - return C2_TIMED_OUT; - } - ALOGE("buffer fence wait error: %d", fenceStatus); - return asC2Error(fenceStatus); - } + sp<Fence> fence = new Fence(); + slot_t slot; + const auto dequeueStatus = + mProducer->dequeueBuffer(width, height, pixelFormat, androidUsage, &slot, &fence); + if (dequeueStatus != OK && dequeueStatus != BUFFER_NEEDS_REALLOCATION) { + return dequeueStatus; } // Call requestBuffer to allocate buffer for the slot and obtain the reference. // Get generation number here. sp<GraphicBuffer> slotBuffer = new GraphicBuffer(); - status = producer->requestBuffer(slot, &slotBuffer); + const auto requestStatus = mProducer->requestBuffer(slot, &slotBuffer); // Detach and delete the temporary buffer. - if (producer->detachBuffer(slot) != android::NO_ERROR) { - return C2_CORRUPTED; + const auto detachStatus = mProducer->detachBuffer(slot); + if (detachStatus != OK) { + return detachStatus; } // Check requestBuffer return flag. - if (status != android::NO_ERROR) { - return asC2Error(status); + if (requestStatus != OK) { + return requestStatus; } // Get generation number and usage from the slot buffer. *usage = slotBuffer->getUsage(); *generation = slotBuffer->getGenerationNumber(); ALOGV("Obtained from temp buffer: generation = %u, usage = %" PRIu64 "", *generation, *usage); - return C2_OK; + return OK; +} + +status_t C2VdaBqBlockPool::Impl::waitFence(slot_t slot, sp<Fence> fence) { + const auto fenceStatus = fence->wait(kFenceWaitTimeMs); + if (fenceStatus == OK) { + return OK; + } + + const auto cancelStatus = mProducer->cancelBuffer(slot, fence); + if (cancelStatus != OK) { + ALOGE("%s(): failed to cancelBuffer(slot=%d)", __func__, slot); + return cancelStatus; + } + + if (fenceStatus == -ETIME) { // fence wait timed out + ALOGV("%s(): buffer (slot=%d) fence wait timed out", __func__, slot); + return TIMED_OUT; + } + ALOGE("buffer fence wait error: %d", fenceStatus); + return fenceStatus; } void C2VdaBqBlockPool::Impl::setRenderCallback( @@ -760,323 +771,135 @@ void C2VdaBqBlockPool::Impl::setRenderCallback( mRenderCallback = renderCallback; } -c2_status_t C2VdaBqBlockPool::Impl::requestNewBufferSet(int32_t bufferCount) { +c2_status_t C2VdaBqBlockPool::Impl::requestNewBufferSet(int32_t bufferCount, uint32_t width, + uint32_t height, uint32_t format, + C2MemoryUsage usage) { + ALOGV("%s(bufferCount=%d, size=%ux%u, format=0x%x, usage=%" PRIu64 ")", __func__, bufferCount, + width, height, format, usage.expected); + if (bufferCount <= 0) { ALOGE("Invalid requested buffer count = %d", bufferCount); return C2_BAD_VALUE; } - if (!mAllocateBuffersLock.try_lock_for(kTimedMutexTimeoutMs)) { - ALOGE("Cannot acquire allocate buffers / configure producer lock over %" PRId64 " ms...", - static_cast<int64_t>(kTimedMutexTimeoutMs.count())); - return C2_BLOCKING; - } - std::lock_guard<std::mutex> lock(mMutex); if (!mProducer) { ALOGD("No HGraphicBufferProducer is configured..."); return C2_NO_INIT; } - - if (mProducerSwitched) { - // Some slots can be occupied by buffers transferred from the old producer. They will not - // used in the current producer. Free the slots of the buffers here. But we cannot find a - // slot is associated with the staled buffer. We free all slots whose associated buffers - // are not owned by client. - ALOGI("requestNewBufferSet: detachBuffer all slots forcedly"); - for (int32_t slot = 0; slot < static_cast<int32_t>(NUM_BUFFER_SLOTS); ++slot) { - if (mSlotAllocations.find(slot) != mSlotAllocations.end()) { - // Skip detaching the buffer which is owned by client now. - continue; - } - status_t status = mProducer->detachBuffer(slot); - if (status == android::NO_INIT) { - // No more active buffer slot. Break the loop now. - break; - } else if (status != android::NO_ERROR) { - return C2_CORRUPTED; - } - } - mProducerSwitched = false; + if (mBuffersRequested == static_cast<size_t>(bufferCount) && mBufferFormat.mWidth == width && + mBufferFormat.mHeight == height && mBufferFormat.mPixelFormat == format && + mBufferFormat.mUsage.expected == usage.expected) { + ALOGD("%s() Request the same format and amount of buffers, skip", __func__); + return C2_OK; } - ALOGV("Requested new buffer count: %d, still dequeued buffer count: %zu", bufferCount, - mSlotAllocations.size()); - - // The remained slot indices in |mSlotAllocations| now are still dequeued (un-available). - // maxDequeuedBufferCount should be set to "new requested buffer count" + "still dequeued buffer - // count" to make sure it has enough available slots to request buffer from. - status_t status = mProducer->setMaxDequeuedBufferCount(bufferCount + mSlotAllocations.size()); - if (status != android::NO_ERROR) { + const auto status = allowAllocation(true); + if (status != OK) { return asC2Error(status); } // Release all remained slot buffer references here. CCodec should either cancel or queue its // owned buffers from this set before the next resolution change. - mSlotAllocations.clear(); - mProducerChangeSlotMap.clear(); + mTrackedGraphicBuffers.reset(); + mDrmHandleManager.closeAllHandles(); + mBuffersRequested = static_cast<size_t>(bufferCount); - status = mProducer->allowAllocation(true); - if (status != android::NO_ERROR) { - return asC2Error(status); - } + // Store buffer formats for future usage. + mBufferFormat = BufferFormat(width, height, format, C2AndroidMemoryUsage(usage)); + return C2_OK; } void C2VdaBqBlockPool::Impl::configureProducer(const sp<HGraphicBufferProducer>& producer) { - ALOGV("configureProducer"); + ALOGV("%s(producer=%p)", __func__, producer.get()); + + std::lock_guard<std::mutex> lock(mMutex); if (producer == nullptr) { - ALOGE("input producer is nullptr..."); - return; - } + ALOGI("input producer is nullptr..."); - std::unique_lock<std::timed_mutex> configureProducerLock( - mConfigureProducerAndAllocateBuffersMutex, std::defer_lock); - if (!configureProducerLock.try_lock_for(kTimedMutexTimeoutMs)) { - ALOGE("Cannot acquire configure producer / allocate buffers lock over %" PRId64 " ms...", - static_cast<int64_t>(kTimedMutexTimeoutMs.count())); + mProducer = nullptr; + mProducerId = 0; + mTrackedGraphicBuffers.reset(); + mDrmHandleManager.closeAllHandles(); return; } - std::lock_guard<std::mutex> lock(mMutex); auto newProducer = std::make_unique<H2BGraphicBufferProducer>(producer); - uint64_t producerId; - if (newProducer->getUniqueId(&producerId) != android::NO_ERROR) { + uint64_t newProducerId; + if (newProducer->getUniqueId(&newProducerId) != OK) { + ALOGE("%s(): failed to get IGBP ID", __func__); + mConfigureProducerError = true; return; } - - if (mProducer && mProducerId != producerId) { - ALOGI("Producer (Surface) is going to switch... ( %" PRIu64 " -> %" PRIu64 " )", - mProducerId, producerId); - if (!switchProducer(newProducer.get(), producerId)) { - mProducerChangeSlotMap.clear(); - return; - } - } else { - mSlotAllocations.clear(); - } - - if (newProducer->setDequeueTimeout(0) != android::NO_ERROR) { - ALOGE("%s(): failed to setDequeueTimeout(0)", __func__); + if (newProducerId == mProducerId) { + ALOGI("%s(): configure the same producer, ignore", __func__); return; } - // hack(b/146409777): Try to connect ARC-specific listener first. - sp<BufferReleasedNotifier> listener = new BufferReleasedNotifier(shared_from_this()); - if (newProducer->connect(listener, 'ARC\0', false) == android::NO_ERROR) { - ALOGI("connected to ARC-specific IGBP listener."); - mFetchBufferNotifier = listener; - } - - // HGraphicBufferProducer could (and should) be replaced if the client has set a new generation - // number to producer. The old HGraphicBufferProducer will be disconnected and deprecated then. + ALOGI("Producer (Surface) is going to switch... ( 0x%" PRIx64 " -> 0x%" PRIx64 " )", + mProducerId, newProducerId); mProducer = std::move(newProducer); - mProducerId = producerId; -} - -bool C2VdaBqBlockPool::Impl::switchProducer(H2BGraphicBufferProducer* const newProducer, - uint64_t newProducerId) { - if (mAllocator->getId() == android::V4L2AllocatorId::SECURE_GRAPHIC) { - // TODO(johnylin): support this when we meet the use case in the future. - ALOGE("Switch producer for secure buffer is not supported..."); - return false; - } - - // Set maxDequeuedBufferCount to new producer. - // Just like requestNewBufferSet(), maxDequeuedBufferCount should be set to "requested buffer - // count" + "buffer count in client" to make sure it has enough available slots to request - // buffers from. - // "Requested buffer count" could be obtained by the size of |mSlotAllocations|. However, it is - // not able to know "buffer count in client" in blockpool's aspect. The alternative solution is - // to set the worse case first, which is equal to the size of |mSlotAllocations|. And in the end - // of updateGraphicBlock() routine, we could get the arbitrary "buffer count in client" by - // counting the calls of updateGraphicBlock(willCancel=true). Then we set maxDequeuedBufferCount - // again to the correct value. - if (newProducer->setMaxDequeuedBufferCount(mSlotAllocations.size() * 2) != android::NO_ERROR) { - return false; - } - - // Reset "buffer count in client". It will be accumulated in updateGraphicBlock() routine. - mBuffersInClient = 0; + mProducerId = newProducerId; + mConfigureProducerError = false; + mAllowAllocation = false; // Set allowAllocation to new producer. - if (newProducer->allowAllocation(true) != android::NO_ERROR) { - return false; - } - - // Get a buffer from the new producer to get the generation number and usage of new producer. - // While attaching buffers, generation number and usage must be aligned to the producer. - uint32_t newGeneration; - uint64_t newUsage; - c2_status_t err = queryGenerationAndUsage(newProducer, mBufferFormat.mWidth, - mBufferFormat.mHeight, mBufferFormat.mPixelFormat, - mBufferFormat.mUsage, &newGeneration, &newUsage); - if (err != C2_OK) { - ALOGE("queryGenerationAndUsage failed: %d", err); - return false; + if (allowAllocation(true) != OK) { + ALOGE("%s(): failed to allowAllocation(true)", __func__); + mConfigureProducerError = true; + return; } - - // Attach all buffers to new producer. - mProducerChangeSlotMap.clear(); - int32_t slot; - std::map<int32_t, std::shared_ptr<C2GraphicAllocation>> newSlotAllocations; - for (auto iter = mSlotAllocations.begin(); iter != mSlotAllocations.end(); ++iter) { - // Convert C2GraphicAllocation to GraphicBuffer. - uint32_t width, height, format, stride, igbp_slot, generation; - uint64_t usage, igbp_id; - android::_UnwrapNativeCodec2GrallocMetadata(iter->second->handle(), &width, &height, - &format, &usage, &stride, &generation, &igbp_id, - &igbp_slot); - native_handle_t* grallocHandle = - android::UnwrapNativeCodec2GrallocHandle(iter->second->handle()); - - // Update generation number and usage. - sp<GraphicBuffer> graphicBuffer = - new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width, height, format, - 1, newUsage, stride); - if (graphicBuffer->initCheck() != android::NO_ERROR) { - ALOGE("Failed to create GraphicBuffer: %d", graphicBuffer->initCheck()); - return false; - } - graphicBuffer->setGenerationNumber(newGeneration); - native_handle_delete(grallocHandle); - - if (newProducer->attachBuffer(graphicBuffer, &slot) != android::NO_ERROR) { - return false; - } - // Convert back to C2GraphicAllocation wrapping new producer id, generation number, usage - // and slot index. - ALOGV("buffer wraps { producer id: %" PRIu64 ", slot: %d }", newProducerId, slot); - C2Handle* c2Handle = android::WrapNativeCodec2GrallocHandle( - graphicBuffer->handle, width, height, format, newUsage, stride, newGeneration, - newProducerId, slot); - if (!c2Handle) { - ALOGE("WrapNativeCodec2GrallocHandle failed"); - return false; - } - std::shared_ptr<C2GraphicAllocation> alloc; - c2_status_t err = mAllocator->priorGraphicAllocation(c2Handle, &alloc); - if (err != C2_OK) { - ALOGE("priorGraphicAllocation failed: %d", err); - return false; - } - - // Store to |newSlotAllocations| and also store old-to-new producer slot map. - ALOGV("Transfered buffer from old producer to new, slot prev: %d -> new %d", iter->first, - slot); - newSlotAllocations[slot] = std::move(alloc); - mProducerChangeSlotMap[iter->first] = slot; + if (mProducer->setDequeueTimeout(0) != OK) { + ALOGE("%s(): failed to setDequeueTimeout(0)", __func__); + mConfigureProducerError = true; + return; } - - // Set allowAllocation to false so producer could not allocate new buffers. - if (newProducer->allowAllocation(false) != android::NO_ERROR) { - ALOGE("allowAllocation(false) failed"); - return false; + if (mProducer->setMaxDequeuedBufferCount(kMaxDequeuedBufferCount) != OK) { + ALOGE("%s(): failed to setMaxDequeuedBufferCount(%d)", __func__, kMaxDequeuedBufferCount); + mConfigureProducerError = true; + return; } - // Try to detach all buffers from old producer. - for (const auto& slotAllocation : mSlotAllocations) { - status_t status = mProducer->detachBuffer(slotAllocation.first); - if (status != android::NO_ERROR) { - ALOGW("detachBuffer slot=%d from old producer failed: %d", slotAllocation.first, - status); + // Migrate existing buffers to the new producer. + if (mTrackedGraphicBuffers.size() > 0) { + uint32_t newGeneration = 0; + uint64_t newUsage = 0; + const status_t err = queryGenerationAndUsageLocked( + mBufferFormat.mWidth, mBufferFormat.mHeight, mBufferFormat.mPixelFormat, + mBufferFormat.mUsage, &newGeneration, &newUsage); + if (err != OK) { + ALOGE("failed to query generation and usage: %d", err); + mConfigureProducerError = true; + return; } - } - - mSlotAllocations = std::move(newSlotAllocations); - return true; -} - -c2_status_t C2VdaBqBlockPool::Impl::updateGraphicBlock( - bool willCancel, uint32_t oldSlot, uint32_t* newSlot, - std::shared_ptr<C2GraphicBlock>* block /* nonnull */) { - std::lock_guard<std::mutex> lock(mMutex); - - if (mProducerChangeSlotMap.empty()) { - ALOGD("A new buffer set is requested right after producer change, no more update needed."); - return C2_CANCELED; - } - - auto it = mProducerChangeSlotMap.find(static_cast<int32_t>(oldSlot)); - if (it == mProducerChangeSlotMap.end()) { - ALOGE("Cannot find old slot = %u in map...", oldSlot); - return C2_NOT_FOUND; - } - int32_t slot = it->second; - *newSlot = static_cast<uint32_t>(slot); - mProducerChangeSlotMap.erase(it); - - if (willCancel) { - sp<Fence> fence = new Fence(); - // The old C2GraphicBlock might be owned by client. Cancel this slot. - if (mProducer->cancelBuffer(slot, fence) != android::NO_ERROR) { - return C2_CORRUPTED; - } - // Client might try to attach the old buffer to the current producer on client's end, - // although it is useless for us anymore. However it will still occupy an available slot. - mBuffersInClient++; - } else { - // The old C2GraphicBlock is still owned by component, replace by the new one and keep this - // slot dequeued. - auto poolData = - std::make_shared<C2VdaBqBlockPoolData>(mProducerId, slot, shared_from_this()); - *block = _C2BlockFactory::CreateGraphicBlock(mSlotAllocations[slot], std::move(poolData)); - } - - if (mProducerChangeSlotMap.empty()) { - // The updateGraphicBlock() routine is about to finish. - // Set the correct maxDequeuedBufferCount to producer, which is "requested buffer count" + - // "buffer count in client". - ALOGV("Requested buffer count: %zu, buffer count in client: %u", mSlotAllocations.size(), - mBuffersInClient); - if (mProducer->setMaxDequeuedBufferCount(mSlotAllocations.size() + mBuffersInClient) != - android::NO_ERROR) { - return C2_CORRUPTED; + if (!mTrackedGraphicBuffers.migrateLocalBuffers(mProducer.get(), mProducerId, newGeneration, + newUsage)) { + ALOGE("%s(): failed to migrateLocalBuffers()", __func__); + mConfigureProducerError = true; + return; } - mProducerSwitched = true; - } - return C2_OK; -} - -c2_status_t C2VdaBqBlockPool::Impl::getMinBuffersForDisplay(size_t* bufferCount) { - std::lock_guard<std::mutex> lock(mMutex); - if (!mProducer) { - ALOGD("No HGraphicBufferProducer is configured..."); - return C2_NO_INIT; + if (mTrackedGraphicBuffers.size() == mBuffersRequested) { + if (allowAllocation(false) != OK) { + ALOGE("%s(): failed to allowAllocation(false)", __func__); + mConfigureProducerError = true; + return; + } + } } - int32_t status, value; - status = mProducer->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &value); - if (status != android::NO_ERROR) { - ALOGE("query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS) failed: %d", status); - return asC2Error(status); - } - if (value <= 0) { - ALOGE("Illegal value of NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS = %d", value); - return C2_BAD_VALUE; + // hack(b/146409777): Try to connect ARC-specific listener first. + sp<BufferReleasedNotifier> listener = new BufferReleasedNotifier(weak_from_this()); + if (mProducer->connect(listener, 'ARC\0', false) == OK) { + ALOGI("connected to ARC-specific IGBP listener."); + mFetchBufferNotifier = listener; } - *bufferCount = static_cast<size_t>(value); - return C2_OK; -} - -void C2VdaBqBlockPool::Impl::detachBuffer(uint64_t producerId, int32_t slotId) { - ALOGV("detachBuffer: producer id = %" PRIu64 ", slot = %d", producerId, slotId); - std::lock_guard<std::mutex> lock(mMutex); - if (producerId == mProducerId && mProducer) { - if (mProducer->detachBuffer(slotId) != android::NO_ERROR) { - return; - } - auto it = mSlotAllocations.find(slotId); - // It may happen that the slot is not included in |mSlotAllocations|, which means it is - // released after resolution change. - if (it != mSlotAllocations.end()) { - mSlotAllocations.erase(it); - } - } + // There might be free buffers at the new producer, notify the client if needed. + onEventNotified(); } bool C2VdaBqBlockPool::Impl::setNotifyBlockAvailableCb(::base::OnceClosure cb) { @@ -1092,6 +915,7 @@ bool C2VdaBqBlockPool::Impl::setNotifyBlockAvailableCb(::base::OnceClosure cb) { // If there is any buffer released after dequeueBuffer() timed out, then we could notify the // caller directly. if (mBufferReleasedAfterTimedOut) { + mBufferReleasedAfterTimedOut = false; outputCb = std::move(cb); } else { mNotifyBlockAvailableCb = std::move(cb); @@ -1105,6 +929,29 @@ bool C2VdaBqBlockPool::Impl::setNotifyBlockAvailableCb(::base::OnceClosure cb) { return true; } +std::optional<unique_id_t> C2VdaBqBlockPool::Impl::getBufferIdFromGraphicBlock( + const C2Block2D& block) { + return mDrmHandleManager.getHandle(block.handle()->data[0]); +} + +status_t C2VdaBqBlockPool::Impl::allowAllocation(bool allow) { + ALOGV("%s(%d)", __func__, allow); + + if (!mProducer) { + ALOGW("%s() mProducer is not initiailzed", __func__); + return NO_INIT; + } + if (mAllowAllocation == allow) { + return OK; + } + + const auto status = mProducer->allowAllocation(allow); + if (status == OK) { + mAllowAllocation = allow; + } + return status; +} + C2VdaBqBlockPool::C2VdaBqBlockPool(const std::shared_ptr<C2Allocator>& allocator, const local_id_t localId) : C2BufferQueueBlockPool(allocator, localId), mLocalId(localId), mImpl(new Impl(allocator)) {} @@ -1125,9 +972,11 @@ void C2VdaBqBlockPool::setRenderCallback( } } -c2_status_t C2VdaBqBlockPool::requestNewBufferSet(int32_t bufferCount) { +c2_status_t C2VdaBqBlockPool::requestNewBufferSet(int32_t bufferCount, uint32_t width, + uint32_t height, uint32_t format, + C2MemoryUsage usage) { if (mImpl) { - return mImpl->requestNewBufferSet(bufferCount); + return mImpl->requestNewBufferSet(bufferCount, width, height, format, usage); } return C2_NO_INIT; } @@ -1138,22 +987,6 @@ void C2VdaBqBlockPool::configureProducer(const sp<HGraphicBufferProducer>& produ } } -c2_status_t C2VdaBqBlockPool::updateGraphicBlock( - bool willCancel, uint32_t oldSlot, uint32_t* newSlot, - std::shared_ptr<C2GraphicBlock>* block /* nonnull */) { - if (mImpl) { - return mImpl->updateGraphicBlock(willCancel, oldSlot, newSlot, block); - } - return C2_NO_INIT; -} - -c2_status_t C2VdaBqBlockPool::getMinBuffersForDisplay(size_t* bufferCount) { - if (mImpl) { - return mImpl->getMinBuffersForDisplay(bufferCount); - } - return C2_NO_INIT; -} - bool C2VdaBqBlockPool::setNotifyBlockAvailableCb(::base::OnceClosure cb) { if (mImpl) { return mImpl->setNotifyBlockAvailableCb(std::move(cb)); @@ -1161,15 +994,11 @@ bool C2VdaBqBlockPool::setNotifyBlockAvailableCb(::base::OnceClosure cb) { return false; } -C2VdaBqBlockPoolData::C2VdaBqBlockPoolData(uint64_t producerId, int32_t slotId, - const std::shared_ptr<C2VdaBqBlockPool::Impl>& pool) - : mProducerId(producerId), mSlotId(slotId), mPool(pool) {} - -C2VdaBqBlockPoolData::~C2VdaBqBlockPoolData() { - if (mShared || !mPool) { - return; +std::optional<unique_id_t> C2VdaBqBlockPool::getBufferIdFromGraphicBlock(const C2Block2D& block) { + if (mImpl) { + return mImpl->getBufferIdFromGraphicBlock(block); } - mPool->detachBuffer(mProducerId, mSlotId); + return std::nullopt; } } // namespace android diff --git a/plugin_store/C2VdaPooledBlockPool.cpp b/plugin_store/C2VdaPooledBlockPool.cpp index 08fdfa0..48cc2e5 100644 --- a/plugin_store/C2VdaPooledBlockPool.cpp +++ b/plugin_store/C2VdaPooledBlockPool.cpp @@ -9,7 +9,6 @@ #include <time.h> -#include <C2AllocatorGralloc.h> #include <C2BlockInternal.h> #include <bufferpool/BufferPoolTypes.h> #include <log/log.h> diff --git a/plugin_store/DrmGrallocHelpers.cpp b/plugin_store/DrmGrallocHelpers.cpp new file mode 100644 index 0000000..0565a69 --- /dev/null +++ b/plugin_store/DrmGrallocHelpers.cpp @@ -0,0 +1,73 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#define LOG_NDEBUG 0 +#define LOG_TAG "DrmGrallocHelper" + +#include <v4l2_codec2/plugin_store/DrmGrallocHelpers.h> + +#include <fcntl.h> +#include <string.h> + +#include <drm/drm.h> +#include <log/log.h> + +namespace android { + +std::optional<int> openRenderFd() { + const char kVirglName[] = "virtio_gpu"; + + for (uint32_t i = 128; i < 192; i++) { + char devName[32]; + snprintf(devName, sizeof(devName), "/dev/dri/renderD%d", i); + + int fd = open(devName, O_RDWR | O_CLOEXEC); + if (fd < 0) { + continue; + } + + char name[32]; + struct drm_version v; + memset(&v, 0, sizeof(v)); + v.name = name; + v.name_len = sizeof(name); + + if (ioctl(fd, static_cast<int>(DRM_IOCTL_VERSION), &v)) { + close(fd); + continue; + } + if (v.name_len != sizeof(kVirglName) - 1 || memcmp(name, kVirglName, v.name_len)) { + close(fd); + continue; + } + return fd; + } + return std::nullopt; +} + +std::optional<uint32_t> getDrmHandle(int renderFd, int primeFd) { + ALOGV("%s(renderFd=%d, primeFd=%u)", __func__, renderFd, primeFd); + + struct drm_prime_handle prime; + memset(&prime, 0, sizeof(prime)); + prime.fd = primeFd; + + if (ioctl(renderFd, static_cast<int>(DRM_IOCTL_PRIME_FD_TO_HANDLE), &prime)) { + ALOGE("Can't translate prime fd %d to handle", prime.fd); + return std::nullopt; + } + return prime.handle; +} + +void closeDrmHandle(int renderFd, uint32_t handle) { + ALOGV("%s(renderFd=%d, handle=%u)", __func__, renderFd, handle); + + struct drm_gem_close gem; + memset(&gem, 0, sizeof(gem)); + gem.handle = handle; + + ioctl(renderFd, DRM_IOCTL_GEM_CLOSE, &gem); +} + +} // namespace android diff --git a/plugin_store/H2BGraphicBufferProducer.cpp b/plugin_store/H2BGraphicBufferProducer.cpp new file mode 100644 index 0000000..95251de --- /dev/null +++ b/plugin_store/H2BGraphicBufferProducer.cpp @@ -0,0 +1,287 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#define LOG_NDEBUG 0 +#define LOG_TAG "H2BGraphicBuferProducer" + +#include <v4l2_codec2/plugin_store/H2BGraphicBufferProducer.h> + +#include <log/log.h> +#include <types.h> +#include <ui/BufferQueueDefs.h> + +namespace android { + +using ::android::BufferQueueDefs::BUFFER_NEEDS_REALLOCATION; +using ::android::BufferQueueDefs::RELEASE_ALL_BUFFERS; +using ::android::hardware::Return; + +using HBuffer = ::android::hardware::graphics::common::V1_2::HardwareBuffer; +using HStatus = ::android::hardware::graphics::bufferqueue::V2_0::Status; +using HConnectionType = hardware::graphics::bufferqueue::V2_0::ConnectionType; +using HQueueBufferOutput = + ::android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer::QueueBufferOutput; + +using ::android::hardware::graphics::bufferqueue::V2_0::utils::b2h; +using ::android::hardware::graphics::bufferqueue::V2_0::utils::h2b; +using ::android::hardware::graphics::bufferqueue::V2_0::utils::HFenceWrapper; + +H2BGraphicBufferProducer::H2BGraphicBufferProducer(sp<HGraphicBufferProducer> base) : mBase(base) {} + +status_t H2BGraphicBufferProducer::requestBuffer(int slot, sp<GraphicBuffer>* buf) { + bool converted = false; + status_t status = UNKNOWN_ERROR; + Return<void> transResult = mBase->requestBuffer( + slot, [&converted, &status, buf](HStatus hStatus, HBuffer const& hBuffer, + uint32_t generationNumber) { + converted = h2b(hStatus, &status) && h2b(hBuffer, buf); + if (*buf) { + (*buf)->setGenerationNumber(generationNumber); + } + }); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!converted) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::setMaxDequeuedBufferCount(int maxDequeuedBuffers) { + status_t status = UNKNOWN_ERROR; + Return<HStatus> transResult = + mBase->setMaxDequeuedBufferCount(static_cast<int32_t>(maxDequeuedBuffers)); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!h2b(static_cast<HStatus>(transResult), &status)) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::dequeueBuffer(uint32_t width, uint32_t height, + uint32_t pixelFormat, + C2AndroidMemoryUsage androidUsage, int* slot, + sp<Fence>* fence) { + using Input = HGraphicBufferProducer::DequeueBufferInput; + using Output = HGraphicBufferProducer::DequeueBufferOutput; + Input input{width, height, pixelFormat, androidUsage.asGrallocUsage()}; + + bool converted = false; + status_t status = UNKNOWN_ERROR; + Return<void> transResult = mBase->dequeueBuffer( + input, [&converted, &status, &slot, &fence](HStatus hStatus, int32_t hSlot, + Output const& hOutput) { + converted = h2b(hStatus, &status); + if (!converted || status != OK) { + return; + } + + *slot = hSlot; + if (hOutput.bufferNeedsReallocation) { + status = BUFFER_NEEDS_REALLOCATION; + } + converted = h2b(hOutput.fence, fence); + }); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!converted) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + // The C2VdaBqBlockPool does not fully own the bufferqueue. After buffers are dequeued here, + // they are passed into the codec2 framework, processed, and eventually queued into the + // bufferqueue. The C2VdaBqBlockPool cannot determine exactly when a buffer gets queued. + // However, if every buffer is being processed by the codec2 framework, then dequeueBuffer() + // will return INVALID_OPERATION because of an attempt to dequeue too many buffers. + // The C2VdaBqBlockPool cannot prevent this from happening, so just map it to TIMED_OUT + // and let the C2VdaBqBlockPool's caller's timeout retry logic handle the failure. + if (status == INVALID_OPERATION) { + status = TIMED_OUT; + } + if (status != OK && status != BUFFER_NEEDS_REALLOCATION && status != TIMED_OUT) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::detachBuffer(int slot) { + status_t status = UNKNOWN_ERROR; + Return<HStatus> transResult = mBase->detachBuffer(static_cast<int32_t>(slot)); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!h2b(static_cast<HStatus>(transResult), &status)) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::attachBuffer(const sp<GraphicBuffer>& buffer, int* outSlot) { + HBuffer hBuffer; + uint32_t hGenerationNumber; + if (!b2h(buffer, &hBuffer, &hGenerationNumber)) { + ALOGE("%s: invalid input buffer.", __func__); + return BAD_VALUE; + } + + bool converted = false; + status_t status = UNKNOWN_ERROR; + Return<void> transResult = mBase->attachBuffer( + hBuffer, hGenerationNumber, + [&converted, &status, outSlot](HStatus hStatus, int32_t hSlot, bool releaseAllBuffers) { + converted = h2b(hStatus, &status); + *outSlot = static_cast<int>(hSlot); + if (converted && releaseAllBuffers && status == OK) { + status = RELEASE_ALL_BUFFERS; + } + }); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!converted) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::cancelBuffer(int slot, const sp<Fence>& fence) { + HFenceWrapper hFenceWrapper; + if (!b2h(fence, &hFenceWrapper)) { + ALOGE("%s(): corrupted input fence.", __func__); + return UNKNOWN_ERROR; + } + + status_t status = UNKNOWN_ERROR; + Return<HStatus> transResult = + mBase->cancelBuffer(static_cast<int32_t>(slot), hFenceWrapper.getHandle()); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!h2b(static_cast<HStatus>(transResult), &status)) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +int H2BGraphicBufferProducer::query(int what, int* value) { + int result = 0; + Return<void> transResult = + mBase->query(static_cast<int32_t>(what), [&result, value](int32_t r, int32_t v) { + result = static_cast<int>(r); + *value = static_cast<int>(v); + }); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + return result; +} + +status_t H2BGraphicBufferProducer::allowAllocation(bool allow) { + status_t status = UNKNOWN_ERROR; + Return<HStatus> transResult = mBase->allowAllocation(allow); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!h2b(static_cast<HStatus>(transResult), &status)) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + if (status != OK) { + ALOGD("%s() failed: %d", __func__, status); + } + return status; +} + +status_t H2BGraphicBufferProducer::getUniqueId(uint64_t* outId) const { + Return<uint64_t> transResult = mBase->getUniqueId(); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + + *outId = static_cast<uint64_t>(transResult); + return OK; +} + +// android::IProducerListener cannot be depended by vendor library, so we use HProducerListener +// directly. +status_t H2BGraphicBufferProducer::connect(sp<HProducerListener> const& hListener, int32_t api, + bool producerControlledByApp) { + bool converted = false; + status_t status = UNKNOWN_ERROR; + // hack(b/146409777): We pass self-defined api, so we don't use b2h() here. + Return<void> transResult = mBase->connect( + hListener, static_cast<HConnectionType>(api), producerControlledByApp, + [&converted, &status](HStatus hStatus, HQueueBufferOutput const& /* hOutput */) { + converted = h2b(hStatus, &status); + }); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!converted) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + return status; +} + +status_t H2BGraphicBufferProducer::setDequeueTimeout(nsecs_t timeout) { + status_t status = UNKNOWN_ERROR; + Return<HStatus> transResult = mBase->setDequeueTimeout(static_cast<int64_t>(timeout)); + + if (!transResult.isOk()) { + ALOGE("%s(): transaction failed: %s", __func__, transResult.description().c_str()); + return FAILED_TRANSACTION; + } + if (!h2b(static_cast<HStatus>(transResult), &status)) { + ALOGE("%s(): corrupted transaction.", __func__); + return FAILED_TRANSACTION; + } + return status; +} + +} // namespace android diff --git a/plugin_store/V4L2PluginStore.cpp b/plugin_store/V4L2PluginStore.cpp index 4475e2f..2d53c5f 100644 --- a/plugin_store/V4L2PluginStore.cpp +++ b/plugin_store/V4L2PluginStore.cpp @@ -32,7 +32,7 @@ C2Allocator* createAllocator(C2Allocator::id_t allocatorId) { return sAllocatorLoader->createAllocator(allocatorId); } - ALOGI("%s(): Fallback to create C2AllocatorGralloc (id=%u)", __func__, allocatorId); + ALOGI("%s(): Fallback to create C2AllocatorGralloc(id=%u)", __func__, allocatorId); return new C2AllocatorGralloc(allocatorId, true); } diff --git a/plugin_store/include/v4l2_codec2/plugin_store/C2VdaBqBlockPool.h b/plugin_store/include/v4l2_codec2/plugin_store/C2VdaBqBlockPool.h index fd524d2..fde6299 100644 --- a/plugin_store/include/v4l2_codec2/plugin_store/C2VdaBqBlockPool.h +++ b/plugin_store/include/v4l2_codec2/plugin_store/C2VdaBqBlockPool.h @@ -17,14 +17,6 @@ namespace android { /** - * Marks the BlockPoolData in |sharedBlock| as shared. The destructor of BlockPoolData would not - * call detachBuffer to BufferQueue if it is shared. - * - * \param sharedBlock the C2ConstGraphicBlock which is about to pass to client. - */ -c2_status_t MarkBlockPoolDataAsShared(const C2ConstGraphicBlock& sharedBlock); - -/** * The BufferQueue-backed block pool design which supports to request arbitrary count of graphic * buffers from IGBP, and use this buffer set among codec component and client. * @@ -39,18 +31,6 @@ public: ~C2VdaBqBlockPool() override = default; /** - * Extracts slot index as pool ID from the graphic block. - * - * \note C2VdaBqBlockPool-specific function - * - * \param block the graphic block allocated by bufferqueue block pool. - * - * Return the buffer's slot index in bufferqueue if extraction is successful. - * Otherwise return std::nullopt. - */ - static std::optional<uint32_t> getBufferIdFromGraphicBlock(const C2Block2D& block); - - /** * It's a trick here. Return C2PlatformAllocatorStore::BUFFERQUEUE instead of the ID of backing * allocator for client's query. It's because in platform side this ID is recognized as * BufferQueue-backed block pool which is only allowed to set surface. @@ -95,42 +75,8 @@ public: * \retval C2_BAD_VALUE |bufferCount| is not greater than zero. * \retval C2_CORRUPTED some unknown, unrecoverable error occured during operation (unexpected). */ - c2_status_t requestNewBufferSet(int32_t bufferCount); - - /** - * Updates the buffer from producer switch. - * - * \note C2VdaBqBlockPool-specific function - * - * \param willCancel if true, the corresponding slot will be canceled to new producer. Otherwise - * the new graphic block will be returned as |block|. - * \param oldSlot the slot index from old producer the caller provided. - * \param newSlot the corresponding slot index of new producer is filled. - * \param block if |willCancel| is false, the new graphic block is stored. - * - * \retval C2_OK the operation was successful. - * \retval C2_NO_INIT this class is not initialized. - * \retval C2_NOT_FOUND cannot find |oldSlot| in the slot changing map. - * \retval C2_CANCELED indicates buffer format is changed and a new buffer set is allocated, no - * more update needed. - * \retval C2_CORRUPTED some unknown, unrecoverable error occured during operation (unexpected). - */ - c2_status_t updateGraphicBlock(bool willCancel, uint32_t oldSlot, uint32_t* newSlot, - std::shared_ptr<C2GraphicBlock>* block /* nonnull */); - - /** - * Gets minimum undequeued buffer count for display from producer. - * - * \note C2VdaBqBlockPool-specific function - * - * \param bufferCount the minimum undequeued buffer count for display is filled. - * - * \retval C2_OK the operation was successful. - * \retval C2_NO_INIT this class is not initialized, or producer is not assigned. - * \retval C2_BAD_VALUE the queried value is illegal (less than 0). - * \retval C2_CORRUPTED some unknown, unrecoverable error occured during operation (unexpected). - */ - c2_status_t getMinBuffersForDisplay(size_t* bufferCount); + c2_status_t requestNewBufferSet(int32_t bufferCount, uint32_t width, uint32_t height, + uint32_t format, C2MemoryUsage usage); /** * Set the callback that will be triggered when there is block available. @@ -144,6 +90,8 @@ public: */ bool setNotifyBlockAvailableCb(base::OnceClosure cb); + std::optional<uint32_t> getBufferIdFromGraphicBlock(const C2Block2D& block); + private: friend struct C2VdaBqBlockPoolData; class Impl; diff --git a/plugin_store/include/v4l2_codec2/plugin_store/DrmGrallocHelpers.h b/plugin_store/include/v4l2_codec2/plugin_store/DrmGrallocHelpers.h new file mode 100644 index 0000000..46d2967 --- /dev/null +++ b/plugin_store/include/v4l2_codec2/plugin_store/DrmGrallocHelpers.h @@ -0,0 +1,19 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_PLUGIN_STORE_STORE_DRM_GRALLOC_HELPERS_H +#define ANDROID_V4L2_CODEC2_PLUGIN_STORE_STORE_DRM_GRALLOC_HELPERS_H + +#include <stdint.h> + +#include <optional> + +namespace android { + +std::optional<int> openRenderFd(); +std::optional<uint32_t> getDrmHandle(int renderFd, int primeFd); +void closeDrmHandle(int renderFd, uint32_t handle); + +} // namespace android +#endif // ANDROID_V4L2_CODEC2_PLUGIN_STORE_STORE_DRM_GRALLOC_HELPERS_H diff --git a/plugin_store/include/v4l2_codec2/plugin_store/H2BGraphicBufferProducer.h b/plugin_store/include/v4l2_codec2/plugin_store/H2BGraphicBufferProducer.h new file mode 100644 index 0000000..11185bb --- /dev/null +++ b/plugin_store/include/v4l2_codec2/plugin_store/H2BGraphicBufferProducer.h @@ -0,0 +1,48 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef ANDROID_V4L2_CODEC2_PLUGIN_STORE_H2B_GRAPHIC_BUFFER_PRODUCER_H +#define ANDROID_V4L2_CODEC2_PLUGIN_STORE_H2B_GRAPHIC_BUFFER_PRODUCER_H + +#include <C2Buffer.h> +#include <android/hardware/graphics/bufferqueue/2.0/IGraphicBufferProducer.h> +#include <android/hardware/graphics/bufferqueue/2.0/IProducerListener.h> +#include <ui/Fence.h> +#include <ui/GraphicBuffer.h> +#include <utils/StrongPointer.h> + +namespace android { + +class H2BGraphicBufferProducer { +public: + using HGraphicBufferProducer = + ::android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer; + using HProducerListener = ::android::hardware::graphics::bufferqueue::V2_0::IProducerListener; + + explicit H2BGraphicBufferProducer(sp<HGraphicBufferProducer> base); + ~H2BGraphicBufferProducer() = default; + + // Convert HIDL interface of IGraphicBufferProducer. + status_t requestBuffer(int slot, sp<GraphicBuffer>* buf); + status_t setMaxDequeuedBufferCount(int maxDequeuedBuffers); + status_t dequeueBuffer(uint32_t width, uint32_t height, uint32_t pixelFormat, + C2AndroidMemoryUsage androidUsage, int* slot, sp<Fence>* fence); + status_t detachBuffer(int slot); + status_t attachBuffer(const sp<GraphicBuffer>& buffer, int* outSlot); + status_t cancelBuffer(int slot, const sp<Fence>& fence); + int query(int what, int* value); + status_t allowAllocation(bool allow); + status_t getUniqueId(uint64_t* outId) const; + status_t connect(sp<HProducerListener> const& hListener, int32_t api, + bool producerControlledByApp); + status_t setDequeueTimeout(nsecs_t timeout); + + sp<HGraphicBufferProducer> getBase() { return mBase; } + +private: + const sp<HGraphicBufferProducer> mBase; +}; + +} // namespace android +#endif // ANDROID_V4L2_CODEC2_PLUGIN_STORE_H2B_GRAPHIC_BUFFER_PRODUCER_H diff --git a/service/Android.bp b/service/Android.bp index 3fcc66c..29ee3ff 100644 --- a/service/Android.bp +++ b/service/Android.bp @@ -1,3 +1,12 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_v4l2_codec2_license" + // to get the below license kinds: + // SPDX-license-identifier-BSD + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + cc_binary { name: "android.hardware.media.c2@1.0-service-v4l2", @@ -12,10 +21,8 @@ cc_binary { "service.cpp", ], - init_rc: ["android.hardware.media.c2@1.0-service-v4l2.rc"], - shared_libs: [ - "libv4l2_codec2_store", + "libv4l2_codec2_components", "libavservices_minijail_vendor", "libchrome", "libcutils", @@ -24,6 +31,17 @@ cc_binary { "libutils", ], - required: ["android.hardware.media.c2@1.1-default-seccomp_policy"], - compile_multilib: "32", + required: ["android.hardware.media.c2@1.2-default-seccomp_policy"], + + compile_multilib: "both", + multilib: { + lib32: { + suffix: "-32", + init_rc: ["android.hardware.media.c2@1.0-service-v4l2-32.rc"], + }, + lib64: { + suffix: "-64", + init_rc: ["android.hardware.media.c2@1.0-service-v4l2-64.rc"], + }, + }, } diff --git a/service/android.hardware.media.c2@1.0-service-v4l2.rc b/service/android.hardware.media.c2@1.0-service-v4l2-32.rc index bec4604..266a73e 100644 --- a/service/android.hardware.media.c2@1.0-service-v4l2.rc +++ b/service/android.hardware.media.c2@1.0-service-v4l2-32.rc @@ -1,4 +1,4 @@ -service android-hardware-media-c2-v4l2-hal-1-0 /vendor/bin/hw/android.hardware.media.c2@1.0-service-v4l2 +service android-hardware-media-c2-v4l2-hal-1-0 /vendor/bin/hw/android.hardware.media.c2@1.0-service-v4l2-32 class hal user media group mediadrm drmrpc diff --git a/service/android.hardware.media.c2@1.0-service-v4l2-64.rc b/service/android.hardware.media.c2@1.0-service-v4l2-64.rc new file mode 100644 index 0000000..a9e1c6f --- /dev/null +++ b/service/android.hardware.media.c2@1.0-service-v4l2-64.rc @@ -0,0 +1,7 @@ +service android-hardware-media-c2-v4l2-hal-1-0 /vendor/bin/hw/android.hardware.media.c2@1.0-service-v4l2-64 + class hal + user media + group mediadrm drmrpc + ioprio rt 4 + writepid /dev/cpuset/foreground/tasks + setenv MESA_GLSL_CACHE_DISABLE 1 diff --git a/service/service.cpp b/service/service.cpp index ba78759..616e641 100644 --- a/service/service.cpp +++ b/service/service.cpp @@ -12,11 +12,11 @@ #include <log/log.h> #include <minijail.h> -#include <v4l2_codec2/store/V4L2ComponentStore.h> +#include <v4l2_codec2/components/V4L2ComponentStore.h> // Default policy for codec2.0 service. static constexpr char kBaseSeccompPolicyPath[] = - "/vendor/etc/seccomp_policy/android.hardware.media.c2@1.1-default-seccomp_policy"; + "/vendor/etc/seccomp_policy/android.hardware.media.c2@1.2-default-seccomp_policy"; // Additional device-specific seccomp permissions can be added in this file. static constexpr char kExtSeccompPolicyPath[] = diff --git a/store/Android.bp b/store/Android.bp deleted file mode 100644 index c2a6169..0000000 --- a/store/Android.bp +++ /dev/null @@ -1,30 +0,0 @@ -cc_library_shared { - name: "libv4l2_codec2_store", - vendor: true, - - defaults: [ - "libcodec2-impl-defaults", - ], - - srcs: [ - "V4L2ComponentStore.cpp", - ], - export_include_dirs: [ - "include", - ], - - shared_libs: [ - "libcutils", - "liblog", - ], - static_libs: [ - "libv4l2_codec2_common", - ], - - cflags: [ - "-Werror", - "-Wall", - "-Wthread-safety", // Check thread annotation at build time. - ], -} - diff --git a/tests/c2_comp_intf/Android.bp b/tests/c2_comp_intf/Android.bp index bc7ae83..1d08f6b 100644 --- a/tests/c2_comp_intf/Android.bp +++ b/tests/c2_comp_intf/Android.bp @@ -1,3 +1,12 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_v4l2_codec2_license" + // to get the below license kinds: + // SPDX-license-identifier-BSD + default_applicable_licenses: ["external_v4l2_codec2_license"], +} + cc_test { name: "C2VEACompIntf_test", vendor: true, @@ -15,11 +24,9 @@ cc_test { "liblog", "libui", "libutils", - "libv4l2_codec2_accel", "libv4l2_codec2_components", ], include_dirs: [ - "external/v4l2_codec2/accel", "external/v4l2_codec2/common/include", "external/v4l2_codec2/components/include", "frameworks/av/media/codec2/components/base/include", diff --git a/tests/c2_e2e_test/Android.mk b/tests/c2_e2e_test/Android.mk index b12af4b..aa97741 100644 --- a/tests/c2_e2e_test/Android.mk +++ b/tests/c2_e2e_test/Android.mk @@ -16,6 +16,9 @@ LOCAL_RESOURCE_DIR := \ LOCAL_MULTILIB := both LOCAL_PACKAGE_NAME := C2E2ETest +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD legacy_unencumbered +LOCAL_LICENSE_CONDITIONS := notice unencumbered +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../NOTICE LOCAL_JNI_SHARED_LIBRARIES := libcodectest LOCAL_MODULE_TAGS := tests diff --git a/tests/c2_e2e_test/jni/Android.mk b/tests/c2_e2e_test/jni/Android.mk index 1cbcdd4..890a74b 100644 --- a/tests/c2_e2e_test/jni/Android.mk +++ b/tests/c2_e2e_test/jni/Android.mk @@ -5,9 +5,6 @@ LOCAL_PATH := $(call my-dir) include $(CLEAR_VARS) -LOCAL_C_INCLUDES := \ - system/core/include \ - LOCAL_SRC_FILES := \ video_encoder_e2e_test.cpp \ video_decoder_e2e_test.cpp \ @@ -26,12 +23,17 @@ LOCAL_SHARED_LIBRARIES := \ libmediandk \ libandroid \ +LOCAL_HEADER_LIBRARIES := liblog_headers + LOCAL_SDK_VERSION := 28 LOCAL_NDK_STL_VARIANT := c++_static LOCAL_STATIC_LIBRARIES := libgtest_ndk_c++ LOCAL_MODULE := libcodectest +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD legacy_unencumbered +LOCAL_LICENSE_CONDITIONS := notice unencumbered +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../../NOTICE # TODO(stevensd): Fix and reenable warnings LOCAL_CFLAGS += -Wno-everything diff --git a/tests/c2_e2e_test/jni/common.cpp b/tests/c2_e2e_test/jni/common.cpp index 9f1a1b0..50f4cdf 100644 --- a/tests/c2_e2e_test/jni/common.cpp +++ b/tests/c2_e2e_test/jni/common.cpp @@ -15,7 +15,7 @@ #include <numeric> #include <sstream> -#include <utils/Log.h> +#include <log/log.h> namespace android { @@ -46,14 +46,22 @@ void InputFile::Rewind() { file_.seekg(0); } -InputFileStream::InputFileStream(std::string file_path) - : InputFile(file_path, std::ifstream::binary) {} +CachedInputFileStream::CachedInputFileStream(std::string file_path) + : InputFile(file_path, std::ifstream::binary) { + if (IsValid()) { + data_.resize(GetLength()); + file_.read(data_.data(), GetLength()); + } +} -size_t InputFileStream::Read(char* buffer, size_t size) { - file_.read(buffer, size); - if (file_.fail()) return -1; +size_t CachedInputFileStream::Read(char* buffer, size_t size) { + memcpy(buffer, data_.data() + position_, size); + position_ += size; + return size; +} - return file_.gcount(); +void CachedInputFileStream::Rewind() { + position_ = 0; } InputFileASCII::InputFileASCII(std::string file_path) : InputFile(file_path) {} @@ -69,6 +77,100 @@ bool InputFileASCII::ReadLine(std::string* line) { return false; // no more lines } +IVFWriter::IVFWriter(std::ofstream* output_file, VideoCodecType codec) + : output_file_(output_file), codec_(codec) {} + +bool IVFWriter::WriteHeader(const Size& resolution, uint32_t frame_rate, uint32_t num_frames) { + constexpr uint16_t kIVFHeaderSize = 32; + char header[kIVFHeaderSize]; + + // Helper lambdas to write 16bit and 32bit data, expects the device to use little endian. + auto write16 = [&header](int i, uint16_t data) { memcpy(&header[i], &data, sizeof(data)); }; + auto write32 = [&header](int i, uint32_t data) { memcpy(&header[i], &data, sizeof(data)); }; + + const char* codec_str; + switch (codec_) { + case VideoCodecType::VP8: + codec_str = "VP80"; + break; + case VideoCodecType::VP9: + codec_str = "VP90"; + break; + default: + printf("[ERR] Unknown codec: \n"); + return false; + } + + strcpy(&header[0], "DKIF"); // Bytes 0-3 of an IVF file header always contain 'DKIF' signature. + constexpr uint16_t kVersion = 0; + write16(4, kVersion); + write16(6, kIVFHeaderSize); + strcpy(&header[8], codec_str); + write16(12, resolution.width); + write16(14, resolution.height); + write32(16, frame_rate); + write32(20, 1); + write32(24, num_frames); + write32(28, 0); // Reserved. + + output_file_->write(header, kIVFHeaderSize); + return !output_file_->bad(); +} + +bool IVFWriter::WriteFrame(const uint8_t* data, uint32_t data_size, uint64_t timestamp) { + constexpr size_t kIVFFrameHeaderSize = 12; + char frame_header[kIVFFrameHeaderSize]; + memcpy(&frame_header[0], &data_size, sizeof(data_size)); + memcpy(&frame_header[4], ×tamp, sizeof(timestamp)); + output_file_->write(frame_header, kIVFFrameHeaderSize); + output_file_->write(reinterpret_cast<const char*>(data), data_size); + return !output_file_->bad(); +} + +bool IVFWriter::SetNumFrames(uint32_t num_frames) { + output_file_->seekp(24); + output_file_->write(reinterpret_cast<const char*>(&num_frames), sizeof(num_frames)); + return !output_file_->bad(); +} + +bool OutputFile::Open(const std::string& file_path, VideoCodecType codec) { + output_file_.open(file_path, std::ofstream::binary); + if (!output_file_.is_open()) { + return false; + } + + if ((codec == VideoCodecType::VP8) || (codec == VideoCodecType::VP9)) { + ivf_writer_ = std::make_unique<IVFWriter>(&output_file_, codec); + } + return true; +} + +void OutputFile::Close() { + if (ivf_writer_) { + ivf_writer_->SetNumFrames(frame_index_); + ivf_writer_.reset(); + } + output_file_.close(); +} + +bool OutputFile::IsOpen() { + return output_file_.is_open(); +} + +// Write the file header. +bool OutputFile::WriteHeader(const Size& resolution, uint32_t frame_rate, uint32_t num_frames) { + return !ivf_writer_ || ivf_writer_->WriteHeader(resolution, frame_rate, num_frames); +} + +bool OutputFile::WriteFrame(uint32_t data_size, const uint8_t* data) { + if (ivf_writer_) { + return (ivf_writer_->WriteFrame(data, data_size, frame_index_++)); + } else { + output_file_.write(reinterpret_cast<const char*>(data), data_size); + return (output_file_.fail()); + } +} + bool FPSCalculator::RecordFrameTimeDiff() { int64_t now_us = GetNowUs(); if (last_frame_time_us_ != 0) { diff --git a/tests/c2_e2e_test/jni/common.h b/tests/c2_e2e_test/jni/common.h index b402a8b..ea8d212 100644 --- a/tests/c2_e2e_test/jni/common.h +++ b/tests/c2_e2e_test/jni/common.h @@ -7,6 +7,7 @@ #include <fstream> #include <ios> +#include <memory> #include <string> #include <vector> @@ -70,20 +71,26 @@ public: // Get the size of the file. size_t GetLength(); // Set position to the beginning of the file. - void Rewind(); + virtual void Rewind(); protected: std::ifstream file_; }; // Wrapper of std::ifstream for reading binary file. -class InputFileStream : public InputFile { +class CachedInputFileStream : public InputFile { public: - explicit InputFileStream(std::string file_path); + explicit CachedInputFileStream(std::string file_path); // Read the given number of bytes to the buffer. Return the number of bytes // read or -1 on error. size_t Read(char* buffer, size_t size); + + void Rewind() override; + +private: + std::vector<char> data_; + size_t position_ = 0; }; // Wrapper of std::ifstream for reading ASCII file. @@ -95,6 +102,40 @@ public: bool ReadLine(std::string* line); }; +// IVF file writer, can be used to write an encoded VP8/9 video to disk. +class IVFWriter { +public: + IVFWriter(std::ofstream* output_file, VideoCodecType codec); + + // Write the IVF file header. + bool WriteHeader(const Size& resolution, uint32_t frame_rate, uint32_t num_frames); + // Append the specified frame data to the IVF file. + bool WriteFrame(const uint8_t* data, uint32_t data_size, uint64_t timestamp); + // Set the number of video frames in the IVF file header. + bool SetNumFrames(uint32_t num_frames); + +private: + std::ofstream* output_file_; + VideoCodecType codec_ = VideoCodecType::UNKNOWN; +}; + +class OutputFile { +public: + bool Open(const std::string& file_path, VideoCodecType codec); + void Close(); + bool IsOpen(); + + // Write the video file header. + bool WriteHeader(const Size& resolution, uint32_t frame_rate, uint32_t num_frames); + // Append the specified frame data to the video file. + bool WriteFrame(uint32_t data_size, const uint8_t* data); + +private: + std::ofstream output_file_; + std::unique_ptr<IVFWriter> ivf_writer_; + uint64_t frame_index_ = 0; +}; + // The helper class to calculate FPS. class FPSCalculator { public: diff --git a/tests/c2_e2e_test/jni/e2e_test_jni.cpp b/tests/c2_e2e_test/jni/e2e_test_jni.cpp index 9535fe7..841cb83 100644 --- a/tests/c2_e2e_test/jni/e2e_test_jni.cpp +++ b/tests/c2_e2e_test/jni/e2e_test_jni.cpp @@ -10,7 +10,7 @@ #include <sstream> #include <android/native_window_jni.h> -#include <utils/Log.h> +#include <log/log.h> #include "e2e_test_jni.h" #include "mediacodec_decoder.h" @@ -25,10 +25,10 @@ public: static constexpr char* kClassName = "org/chromium/c2/test/E2eTestActivity"; - void OnDecoderReady(void* decoder) override { + void OnCodecReady(void* codec) override { jclass cls = env_->FindClass(kClassName); - jmethodID methodid = env_->GetMethodID(cls, "onDecoderReady", "(J)V"); - env_->CallVoidMethod(thiz_, methodid, (jlong)decoder); + jmethodID methodid = env_->GetMethodID(cls, "onCodecReady", "(J)V"); + env_->CallVoidMethod(thiz_, methodid, (jlong)codec); } void OnSizeChanged(int width, int height) override { @@ -78,10 +78,10 @@ JNIEXPORT jint JNICALL Java_org_chromium_c2_test_E2eTestActivity_c2VideoTest( ANativeWindow* native_window = ANativeWindow_fromSurface(env, surface); int res; + JniConfigureCallback cb(env, thiz); if (encode) { - res = RunEncoderTests(final_args, test_args_count + 1); + res = RunEncoderTests(final_args, test_args_count + 1, &cb); } else { - JniConfigureCallback cb(env, thiz); res = RunDecoderTests(final_args, test_args_count + 1, native_window, &cb); } delete[] final_args; diff --git a/tests/c2_e2e_test/jni/e2e_test_jni.h b/tests/c2_e2e_test/jni/e2e_test_jni.h index 14a63bb..90b7976 100644 --- a/tests/c2_e2e_test/jni/e2e_test_jni.h +++ b/tests/c2_e2e_test/jni/e2e_test_jni.h @@ -12,8 +12,8 @@ namespace android { // Callback to communicate from the test back to the Activity. class ConfigureCallback { public: - // Provides a reference to the current test's decoder, or clears the reference. - virtual void OnDecoderReady(void* decoder) = 0; + // Provides a reference to the current test's codec, or clears the reference. + virtual void OnCodecReady(void* codec) = 0; // Configures the surface with the size of the current video. virtual void OnSizeChanged(int width, int height) = 0; virtual ~ConfigureCallback() = default; @@ -24,6 +24,6 @@ public: int RunDecoderTests(char** test_args, int test_args_count, ANativeWindow* window, android::ConfigureCallback* cb); -int RunEncoderTests(char** test_args, int test_args_count); +int RunEncoderTests(char** test_args, int test_args_count, android::ConfigureCallback* cb); #endif // C2_E2E_TEST_E2E_TEST_JNI_H_ diff --git a/tests/c2_e2e_test/jni/encoded_data_helper.cpp b/tests/c2_e2e_test/jni/encoded_data_helper.cpp index d17e7cb..0ced022 100644 --- a/tests/c2_e2e_test/jni/encoded_data_helper.cpp +++ b/tests/c2_e2e_test/jni/encoded_data_helper.cpp @@ -12,7 +12,7 @@ #include <utility> -#include <utils/Log.h> +#include <log/log.h> namespace android { @@ -95,7 +95,7 @@ std::string GetBytesForNextFrame(const std::string& data, size_t* next_pos) { EncodedDataHelper::EncodedDataHelper(const std::string& file_path, VideoCodecType type) : type_(type) { - InputFileStream input(file_path); + CachedInputFileStream input(file_path); if (!input.IsValid()) { ALOGE("Failed to open file: %s", file_path.c_str()); return; diff --git a/tests/c2_e2e_test/jni/mediacodec_decoder.cpp b/tests/c2_e2e_test/jni/mediacodec_decoder.cpp index 1b835c0..b14a841 100644 --- a/tests/c2_e2e_test/jni/mediacodec_decoder.cpp +++ b/tests/c2_e2e_test/jni/mediacodec_decoder.cpp @@ -13,8 +13,8 @@ #include <utility> #include <vector> +#include <log/log.h> #include <media/NdkMediaFormat.h> -#include <utils/Log.h> namespace android { namespace { @@ -31,6 +31,8 @@ constexpr int kTimeoutWaitForInputUs = 1000; // 1 millisecond constexpr size_t kTimeoutMaxRetries = 500; // Helper function to get possible C2 hardware decoder names from |type|. +// Note: A single test APK is built for both ARC++ and ARCVM, so both the VDA decoder and the new +// V4L2 decoder names need to be specified here. std::vector<const char*> GetC2VideoDecoderNames(VideoCodecType type) { switch (type) { case VideoCodecType::H264: @@ -45,14 +47,16 @@ std::vector<const char*> GetC2VideoDecoderNames(VideoCodecType type) { } // Helper function to get possible software decoder names from |type|. +// Note: A single test APK is built for both ARC++ and ARCVM, so both the OMX decoder used on +// Android P and the c2.android decoder used on Android R need to be specified here. std::vector<const char*> GetSwVideoDecoderNames(VideoCodecType type) { switch (type) { case VideoCodecType::H264: - return {"OMX.google.h264.decoder"}; + return {"c2.android.avc.decoder", "OMX.google.h264.decoder"}; case VideoCodecType::VP8: - return {"OMX.google.vp8.decoder"}; + return {"c2.android.vp8.decoder", "OMX.google.vp8.decoder"}; case VideoCodecType::VP9: - return {"OMX.google.vp9.decoder"}; + return {"c2.android.vp9.decoder", "OMX.google.vp9.decoder"}; default: // unknown type return {}; } @@ -74,12 +78,10 @@ int64_t RoundUp(int64_t n, int64_t multiple) { } // namespace // static -std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create(const std::string& input_path, - VideoCodecProfile profile, - bool use_sw_decoder, - const Size& video_size, int frame_rate, - ANativeWindow* surface, - bool render_on_release, bool loop) { +std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create( + const std::string& input_path, VideoCodecProfile profile, bool use_sw_decoder, + const Size& video_size, int frame_rate, ANativeWindow* surface, bool render_on_release, + bool loop, bool use_fake_renderer) { if (video_size.IsEmpty()) { ALOGE("Size is not valid: %dx%d", video_size.width, video_size.height); return nullptr; @@ -110,7 +112,7 @@ std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create(const std::string& auto ret = std::unique_ptr<MediaCodecDecoder>( new MediaCodecDecoder(codec, std::move(encoded_data_helper), type, video_size, - frame_rate, surface, render_on_release, loop)); + frame_rate, surface, render_on_release, loop, use_fake_renderer)); AMediaCodecOnAsyncNotifyCallback cb{ .onAsyncInputAvailable = @@ -148,7 +150,8 @@ std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create(const std::string& MediaCodecDecoder::MediaCodecDecoder(AMediaCodec* codec, std::unique_ptr<EncodedDataHelper> encoded_data_helper, VideoCodecType type, const Size& size, int frame_rate, - ANativeWindow* surface, bool render_on_release, bool loop) + ANativeWindow* surface, bool render_on_release, bool loop, + bool use_fake_renderer) : codec_(codec), encoded_data_helper_(std::move(encoded_data_helper)), type_(type), @@ -156,12 +159,17 @@ MediaCodecDecoder::MediaCodecDecoder(AMediaCodec* codec, frame_rate_(frame_rate), surface_(surface), render_on_release_(render_on_release), - looping_(loop) {} + looping_(loop), + fake_renderer_running_(use_fake_renderer), + fake_render_thread_([](MediaCodecDecoder* dec) { dec->FakeRenderLoop(); }, this) {} MediaCodecDecoder::~MediaCodecDecoder() { if (codec_ != nullptr) { AMediaCodec_delete(codec_); } + fake_renderer_running_ = false; + fake_render_cv_.notify_one(); + fake_render_thread_.join(); } void MediaCodecDecoder::AddOutputBufferReadyCb(const OutputBufferReadyCb& cb) { @@ -244,6 +252,13 @@ bool MediaCodecDecoder::Decode() { case FORMAT_CHANGED: success = GetOutputFormat(); break; + case FAKE_FRAME_RENDERED: + media_status_t status = AMediaCodec_releaseOutputBuffer(codec_, evt.idx, false); + if (status != AMEDIA_OK) { + ALOGE("Failed to releaseOutputBuffer(index=%zu): %d", evt.idx, status); + success = false; + } + break; } assert(success); } @@ -285,8 +300,9 @@ bool MediaCodecDecoder::DequeueOutputBuffer(int32_t index, AMediaCodecBufferInfo base_timestamp_ns_ = now; } else if (now > GetReleaseTimestampNs(received_outputs_)) { drop_frame_count_++; - ALOGD("Drop frame #%d: deadline %" PRIu64 "us, actual %" PRIu64 "us", drop_frame_count_, - (received_outputs_ * 1000000 / frame_rate_), (now - base_timestamp_ns_) / 1000); + ALOGD("Drop frame #%d: frame %d deadline %" PRIu64 "us, actual %" PRIu64 "us", + drop_frame_count_, received_outputs_, (received_outputs_ * 1000000ull / frame_rate_), + (now - base_timestamp_ns_) / 1000); render_frame = false; // We don't render the dropped frame. } @@ -355,7 +371,7 @@ bool MediaCodecDecoder::FeedEOSInputBuffer(size_t index) { return true; } -bool MediaCodecDecoder::ReceiveOutputBuffer(size_t index, const AMediaCodecBufferInfo& info, +bool MediaCodecDecoder::ReceiveOutputBuffer(int32_t index, const AMediaCodecBufferInfo& info, bool render_buffer) { size_t out_size = 0; uint8_t* buf = nullptr; @@ -377,17 +393,50 @@ bool MediaCodecDecoder::ReceiveOutputBuffer(size_t index, const AMediaCodecBuffe callback(buf, info.size, received_outputs_); } - media_status_t status = - render_buffer ? AMediaCodec_releaseOutputBufferAtTime( - codec_, index, GetReleaseTimestampNs(received_outputs_)) - : AMediaCodec_releaseOutputBuffer(codec_, index, false /* render */); - if (status != AMEDIA_OK) { - ALOGE("Failed to releaseOutputBuffer(index=%zu): %d", index, status); - return false; + if (fake_renderer_running_) { + std::lock_guard<std::mutex> lock(fake_render_mut_); + fake_render_frames_.emplace(index, GetReleaseTimestampNs(received_outputs_)); + fake_render_cv_.notify_one(); + } else { + media_status_t status = + render_buffer ? AMediaCodec_releaseOutputBufferAtTime( + codec_, index, GetReleaseTimestampNs(received_outputs_)) + : AMediaCodec_releaseOutputBuffer(codec_, index, false /* render */); + if (status != AMEDIA_OK) { + ALOGE("Failed to releaseOutputBuffer(index=%zu): %d", index, status); + return false; + } } return true; } +void MediaCodecDecoder::FakeRenderLoop() { + while (fake_renderer_running_) { + std::pair<int32_t, int64_t> next_frame; + { + std::unique_lock<std::mutex> lock(fake_render_mut_); + fake_render_cv_.wait(lock, [&]() { + return !fake_renderer_running_ || !fake_render_frames_.empty(); + }); + if (!fake_renderer_running_) { + break; + } + + next_frame = fake_render_frames_.front(); + fake_render_frames_.pop(); + } + + const uint64_t now = GetCurrentTimeNs(); + if (now < next_frame.second) { + usleep((next_frame.second - now) / 1000); + } + + std::lock_guard<std::mutex> lock(event_queue_mut_); + event_queue_.push({.type = FAKE_FRAME_RENDERED, .idx = next_frame.first}); + event_queue_cv_.notify_one(); + } +} + bool MediaCodecDecoder::GetOutputFormat() { AMediaFormat* format = AMediaCodec_getOutputFormat(codec_); bool success = true; diff --git a/tests/c2_e2e_test/jni/mediacodec_decoder.h b/tests/c2_e2e_test/jni/mediacodec_decoder.h index 4f4fe60..d5e916f 100644 --- a/tests/c2_e2e_test/jni/mediacodec_decoder.h +++ b/tests/c2_e2e_test/jni/mediacodec_decoder.h @@ -10,6 +10,7 @@ #include <mutex> #include <queue> #include <string> +#include <thread> #include <vector> #include <media/NdkMediaCodec.h> @@ -27,7 +28,7 @@ public: VideoCodecProfile profile, bool use_sw_decoder, const Size& video_size, int frame_rate, ANativeWindow* surface, bool renderOnRelease, - bool loop); + bool loop, bool use_fake_renderer); MediaCodecDecoder() = delete; ~MediaCodecDecoder(); @@ -70,8 +71,10 @@ public: void OnAsyncOutputAvailable(int32_t idx, AMediaCodecBufferInfo* info); void OnAsyncFormatChanged(AMediaFormat* format); + void FakeRenderLoop(); + private: - enum CodecEventType { INPUT_AVAILABLE, OUTPUT_AVAILABLE, FORMAT_CHANGED }; + enum CodecEventType { INPUT_AVAILABLE, OUTPUT_AVAILABLE, FORMAT_CHANGED, FAKE_FRAME_RENDERED }; struct CodecEvent { CodecEventType type; int32_t idx; @@ -80,7 +83,7 @@ private: MediaCodecDecoder(AMediaCodec* codec, std::unique_ptr<EncodedDataHelper> encoded_data_helper, VideoCodecType type, const Size& size, int frame_rate, ANativeWindow* surface, - bool renderOnRelease, bool loop); + bool renderOnRelease, bool loop, bool use_fake_renderer); // Enum class of the status of dequeueing output buffer. enum class DequeueStatus { RETRY, SUCCESS, FAILURE }; @@ -103,7 +106,7 @@ private: // Receive the output buffer and make mOutputBufferReadyCb callback if given. // |index| is the index of the target output buffer. // |info| is the buffer info of the target output buffer. - bool ReceiveOutputBuffer(size_t index, const AMediaCodecBufferInfo& info, bool render_buffer); + bool ReceiveOutputBuffer(int32_t index, const AMediaCodecBufferInfo& info, bool render_buffer); // Get output format by AMediaCodec_getOutputFormat and make // |output_format_changed_cb_| callback if given. @@ -153,6 +156,12 @@ private: std::queue<CodecEvent> event_queue_; // guarded by event_queue_mut_ std::mutex event_queue_mut_; std::condition_variable event_queue_cv_; + + std::atomic<bool> fake_renderer_running_; + std::thread fake_render_thread_; + std::mutex fake_render_mut_; + std::condition_variable fake_render_cv_; + std::queue<std::pair<int32_t, int64_t>> fake_render_frames_; }; } // namespace android diff --git a/tests/c2_e2e_test/jni/mediacodec_encoder.cpp b/tests/c2_e2e_test/jni/mediacodec_encoder.cpp index e4c90a4..d15f285 100644 --- a/tests/c2_e2e_test/jni/mediacodec_encoder.cpp +++ b/tests/c2_e2e_test/jni/mediacodec_encoder.cpp @@ -12,15 +12,18 @@ #include <utility> #include <vector> +#include <log/log.h> #include <media/NdkMediaFormat.h> -#include <utils/Log.h> namespace android { namespace { -// The values are defined at +// These values are defined at // <android_root>/frameworks/base/media/java/android/media/MediaCodecInfo.java. constexpr int32_t COLOR_FormatYUV420Planar = 19; constexpr int32_t BITRATE_MODE_CBR = 2; +constexpr int32_t AVCProfileBaseline = 0x01; +constexpr int32_t VP8ProfileMain = 0x01; +constexpr int32_t VP9Profile0 = 0x01; // The time interval between two key frames. constexpr int32_t kIFrameIntervalSec = 10; @@ -33,23 +36,58 @@ constexpr int kTimeoutUs = 1000; // 1ms. // output buffer. constexpr int kBufferPeriodTimeoutUs = 1000000; // 1 sec -// Helper function to get possible encoder names from |type|. +// Helper function to get possible C2 hardware encoder names from |type|. // Note: A single test APK is built for both ARC++ and ARCVM, so both the C2 VEA encoder and the new // V4L2 encoder names need to be specified here. -std::vector<const char*> GetArcVideoEncoderNames(VideoCodecType type) { +std::vector<const char*> GetHWVideoEncoderNames(VideoCodecType type) { switch (type) { case VideoCodecType::H264: return {"c2.v4l2.avc.encoder", "c2.vea.avc.encoder"}; - default: // unsupported type: VP8, VP9, or unknown + case VideoCodecType::VP8: + return {"c2.v4l2.vp8.encoder"}; // Only supported on ARCVM + case VideoCodecType::VP9: + return {"c2.v4l2.vp9.encoder"}; // Only supported on ARCVM + default: return {}; } } +// Helper function to get possible software encoder names from |type|. +// Note: A single test APK is built for both ARC++ and ARCVM, so both the OMX encoder used on +// Android P and the c2.android encoder used on Android R need to be specified here. +std::vector<const char*> GetSWVideoEncoderNames(VideoCodecType type) { + switch (type) { + case VideoCodecType::H264: + return {"c2.android.avc.encoder", "OMX.google.h264.encoder"}; + case VideoCodecType::VP8: + return {"c2.android.vp8.encoder", "OMX.google.vp8.encoder"}; + case VideoCodecType::VP9: + return {"c2.android.vp9.encoder", "OMX.google.vp9.encoder"}; + default: + return {}; + } +} + +// Helper function to get the profile associated with the specified codec. +int32_t GetProfile(VideoCodecType type) { + switch (type) { + case VideoCodecType::H264: + return AVCProfileBaseline; + case VideoCodecType::VP8: + return VP8ProfileMain; + case VideoCodecType::VP9: + return VP9Profile0; + default: + return AVCProfileBaseline; + } +} + } // namespace // static std::unique_ptr<MediaCodecEncoder> MediaCodecEncoder::Create(std::string input_path, - Size visible_size) { + VideoCodecType type, Size visible_size, + bool use_sw_encoder) { if (visible_size.width <= 0 || visible_size.height <= 0 || visible_size.width % 2 == 1 || visible_size.height % 2 == 1) { ALOGE("Size is not valid: %dx%d", visible_size.width, visible_size.height); @@ -57,7 +95,7 @@ std::unique_ptr<MediaCodecEncoder> MediaCodecEncoder::Create(std::string input_p } size_t buffer_size = visible_size.width * visible_size.height * 3 / 2; - std::unique_ptr<InputFileStream> input_file(new InputFileStream(input_path)); + std::unique_ptr<CachedInputFileStream> input_file(new CachedInputFileStream(input_path)); if (!input_file->IsValid()) { ALOGE("Failed to open file: %s", input_path.c_str()); return nullptr; @@ -70,8 +108,8 @@ std::unique_ptr<MediaCodecEncoder> MediaCodecEncoder::Create(std::string input_p } AMediaCodec* codec = nullptr; - // Only H264 is supported now. - auto encoder_names = GetArcVideoEncoderNames(VideoCodecType::H264); + auto encoder_names = + use_sw_encoder ? GetSWVideoEncoderNames(type) : GetHWVideoEncoderNames(type); for (const auto& encoder_name : encoder_names) { codec = AMediaCodec_createCodecByName(encoder_name); if (codec) { @@ -84,17 +122,19 @@ std::unique_ptr<MediaCodecEncoder> MediaCodecEncoder::Create(std::string input_p return nullptr; } - return std::unique_ptr<MediaCodecEncoder>(new MediaCodecEncoder( - codec, std::move(input_file), visible_size, buffer_size, file_size / buffer_size)); + return std::unique_ptr<MediaCodecEncoder>( + new MediaCodecEncoder(codec, type, std::move(input_file), visible_size, buffer_size, + file_size / buffer_size)); } -MediaCodecEncoder::MediaCodecEncoder(AMediaCodec* codec, - std::unique_ptr<InputFileStream> input_file, Size size, +MediaCodecEncoder::MediaCodecEncoder(AMediaCodec* codec, VideoCodecType type, + std::unique_ptr<CachedInputFileStream> input_file, Size size, size_t buffer_size, size_t num_total_frames) : kVisibleSize(size), kBufferSize(buffer_size), kNumTotalFrames(num_total_frames), codec_(codec), + type_(type), num_encoded_frames_(num_total_frames), input_file_(std::move(input_file)) {} @@ -120,7 +160,8 @@ void MediaCodecEncoder::Rewind() { bool MediaCodecEncoder::Configure(int32_t bitrate, int32_t framerate) { ALOGV("Configure encoder bitrate=%d, framerate=%d", bitrate, framerate); AMediaFormat* format = AMediaFormat_new(); - AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, "video/avc"); + AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, GetMimeType(type_)); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PROFILE, GetProfile(type_)); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE, BITRATE_MODE_CBR); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, kIFrameIntervalSec); diff --git a/tests/c2_e2e_test/jni/mediacodec_encoder.h b/tests/c2_e2e_test/jni/mediacodec_encoder.h index 8580061..53e6fdf 100644 --- a/tests/c2_e2e_test/jni/mediacodec_encoder.h +++ b/tests/c2_e2e_test/jni/mediacodec_encoder.h @@ -18,7 +18,8 @@ namespace android { class MediaCodecEncoder { public: // Checks the argument and create MediaCodecEncoder instance. - static std::unique_ptr<MediaCodecEncoder> Create(std::string input_path, Size visible_size); + static std::unique_ptr<MediaCodecEncoder> Create(std::string input_path, VideoCodecType type, + Size visible_size, bool use_sw_encoder); MediaCodecEncoder() = delete; ~MediaCodecEncoder(); @@ -53,7 +54,8 @@ public: void set_run_at_fps(bool run_at_fps); private: - MediaCodecEncoder(AMediaCodec* codec, std::unique_ptr<InputFileStream> inputFile, Size size, + MediaCodecEncoder(AMediaCodec* codec, VideoCodecType type, + std::unique_ptr<CachedInputFileStream> inputFile, Size size, size_t bufferSize, size_t numTotalFrames); // Read the content from the |input_file_| and feed into the input buffer. @@ -79,11 +81,13 @@ private: // The target mediacodec encoder. AMediaCodec* codec_; + // The output codec type. + VideoCodecType type_; // The number of frames to encode. size_t num_encoded_frames_; // The input video raw stream file. The file size must be the multiple of // |kBufferSize|. - std::unique_ptr<InputFileStream> input_file_; + std::unique_ptr<CachedInputFileStream> input_file_; // The target output bitrate. int bitrate_ = 192000; // The target output framerate. diff --git a/tests/c2_e2e_test/jni/video_decoder_e2e_test.cpp b/tests/c2_e2e_test/jni/video_decoder_e2e_test.cpp index 1251625..46b497f 100644 --- a/tests/c2_e2e_test/jni/video_decoder_e2e_test.cpp +++ b/tests/c2_e2e_test/jni/video_decoder_e2e_test.cpp @@ -14,7 +14,7 @@ #include <vector> #include <gtest/gtest.h> -#include <utils/Log.h> +#include <log/log.h> #include "common.h" #include "e2e_test_jni.h" @@ -33,11 +33,12 @@ C2VideoDecoderTestEnvironment* g_env; class C2VideoDecoderTestEnvironment : public testing::Environment { public: - C2VideoDecoderTestEnvironment(bool loop, bool use_sw_decoder, const std::string& data, - const std::string& output_frames_path, ANativeWindow* surface, - ConfigureCallback* cb) + C2VideoDecoderTestEnvironment(bool loop, bool use_sw_decoder, bool use_fake_renderer, + const std::string& data, const std::string& output_frames_path, + ANativeWindow* surface, ConfigureCallback* cb) : loop_(loop), use_sw_decoder_(use_sw_decoder), + use_fake_renderer_(use_fake_renderer), test_video_data_(data), output_frames_path_(output_frames_path), surface_(surface), @@ -103,12 +104,14 @@ public: ConfigureCallback* configure_cb() const { return configure_cb_; } bool loop() const { return loop_; } bool use_sw_decoder() const { return use_sw_decoder_; } + bool use_fake_renderer() const { return use_fake_renderer_; } ANativeWindow* surface() const { return surface_; } protected: bool loop_; bool use_sw_decoder_; + bool use_fake_renderer_; std::string test_video_data_; std::string output_frames_path_; @@ -253,10 +256,10 @@ protected: decoder_ = MediaCodecDecoder::Create(g_env->input_file_path(), g_env->video_codec_profile(), g_env->use_sw_decoder(), g_env->visible_size(), g_env->frame_rate(), surface, renderOnRelease(), - g_env->loop()); + g_env->loop(), g_env->use_fake_renderer()); ASSERT_TRUE(decoder_); - g_env->configure_cb()->OnDecoderReady(decoder_.get()); + g_env->configure_cb()->OnCodecReady(decoder_.get()); decoder_->Rewind(); @@ -286,7 +289,7 @@ protected: EXPECT_EQ(g_env->num_frames(), decoded_frames_); } - g_env->configure_cb()->OnDecoderReady(nullptr); + g_env->configure_cb()->OnCodecReady(nullptr); decoder_.reset(); } @@ -370,13 +373,14 @@ TEST_F(C2VideoDecoderSurfaceNoRenderE2ETest, TestFPS) { } // namespace android bool GetOption(int argc, char** argv, std::string* test_video_data, std::string* output_frames_path, - bool* loop, bool* use_sw_decoder) { + bool* loop, bool* use_sw_decoder, bool* use_fake_renderer) { const char* const optstring = "t:o:"; static const struct option opts[] = { {"test_video_data", required_argument, nullptr, 't'}, {"output_frames_path", required_argument, nullptr, 'o'}, {"loop", no_argument, nullptr, 'l'}, {"use_sw_decoder", no_argument, nullptr, 's'}, + {"fake_renderer", no_argument, nullptr, 'f'}, {nullptr, 0, nullptr, 0}, }; @@ -396,6 +400,9 @@ bool GetOption(int argc, char** argv, std::string* test_video_data, std::string* case 's': *use_sw_decoder = true; break; + case 'f': + *use_fake_renderer = true; + break; default: printf("[WARN] Unknown option: getopt_long() returned code 0x%x.\n", opt); break; @@ -415,8 +422,9 @@ int RunDecoderTests(char** test_args, int test_args_count, ANativeWindow* surfac std::string output_frames_path; bool loop = false; bool use_sw_decoder = false; + bool use_fake_renderer = false; if (!GetOption(test_args_count, test_args, &test_video_data, &output_frames_path, &loop, - &use_sw_decoder)) { + &use_sw_decoder, &use_fake_renderer)) { ALOGE("GetOption failed"); return EXIT_FAILURE; } @@ -424,7 +432,8 @@ int RunDecoderTests(char** test_args, int test_args_count, ANativeWindow* surfac if (android::g_env == nullptr) { android::g_env = reinterpret_cast<android::C2VideoDecoderTestEnvironment*>( testing::AddGlobalTestEnvironment(new android::C2VideoDecoderTestEnvironment( - loop, use_sw_decoder, test_video_data, output_frames_path, surface, cb))); + loop, use_sw_decoder, use_fake_renderer, test_video_data, + output_frames_path, surface, cb))); } else { ALOGE("Trying to reuse test process"); return EXIT_FAILURE; diff --git a/tests/c2_e2e_test/jni/video_encoder_e2e_test.cpp b/tests/c2_e2e_test/jni/video_encoder_e2e_test.cpp index be3a489..5215052 100644 --- a/tests/c2_e2e_test/jni/video_encoder_e2e_test.cpp +++ b/tests/c2_e2e_test/jni/video_encoder_e2e_test.cpp @@ -15,9 +15,10 @@ #include <vector> #include <gtest/gtest.h> -#include <utils/Log.h> +#include <log/log.h> #include "common.h" +#include "e2e_test_jni.h" #include "mediacodec_encoder.h" namespace android { @@ -52,11 +53,13 @@ struct CmdlineArgs { std::string test_stream_data; bool run_at_fps = false; size_t num_encoded_frames = 0; + bool use_sw_encoder = false; }; class C2VideoEncoderTestEnvironment : public testing::Environment { public: - explicit C2VideoEncoderTestEnvironment(const CmdlineArgs& args) : args_(args) {} + explicit C2VideoEncoderTestEnvironment(const CmdlineArgs& args, ConfigureCallback* cb) + : args_(args), configure_cb_(cb) {} void SetUp() override { ParseTestStreamData(); } @@ -68,7 +71,6 @@ public: // (see http://www.fourcc.org/yuv.php#IYUV). // - |width| and |height| are in pixels. // - |profile| to encode into (values of VideoCodecProfile). - // NOTE: Only H264PROFILE_MAIN(1) is supported. Now we ignore this value. // - |output_file_path| filename to save the encoded stream to (optional). // The format for H264 is Annex-B byte stream. // - |requested_bitrate| requested bitrate in bits per second. @@ -95,7 +97,21 @@ public: if (fields.size() >= 4 && !fields[3].empty()) { int profile = stoi(fields[3]); - if (profile != 1) printf("[WARN] Only H264PROFILE_MAIN(1) is supported.\n"); + switch (profile) { + case VideoCodecProfile::H264PROFILE_MAIN: + codec_ = VideoCodecType::H264; + break; + case VideoCodecProfile::VP8PROFILE_ANY: + codec_ = VideoCodecType::VP8; + break; + case VideoCodecProfile::VP9PROFILE_PROFILE0: + codec_ = VideoCodecType::VP9; + break; + default: + printf("[WARN] Only H264PROFILE_MAIN, VP8PROFILE_ANY and VP9PROFILE_PROFILE0 are" + "supported.\n"); + codec_ = VideoCodecType::H264; + } } if (fields.size() >= 5 && !fields[4].empty()) { @@ -138,6 +154,7 @@ public: } Size visible_size() const { return visible_size_; } + VideoCodecType codec() const { return codec_; } std::string input_file_path() const { return input_file_path_; } std::string output_file_path() const { return output_file_path_; } int requested_bitrate() const { return requested_bitrate_; } @@ -147,11 +164,16 @@ public: bool run_at_fps() const { return args_.run_at_fps; } size_t num_encoded_frames() const { return args_.num_encoded_frames; } + bool use_sw_encoder() const { return args_.use_sw_encoder; } + + ConfigureCallback* configure_cb() const { return configure_cb_; } private: const CmdlineArgs args_; + ConfigureCallback* configure_cb_; Size visible_size_; + VideoCodecType codec_; std::string input_file_path_; std::string output_file_path_; @@ -164,12 +186,10 @@ private: class C2VideoEncoderE2ETest : public testing::Test { public: // Callback functions of getting output buffers from encoder. - void WriteOutputBufferToFile(const uint8_t* data, const AMediaCodecBufferInfo& info) { - if (output_file_.is_open()) { - output_file_.write(reinterpret_cast<const char*>(data), info.size); - if (output_file_.fail()) { - printf("[ERR] Failed to write encoded buffer into file.\n"); - } + void WriteOutputBufferToFile(VideoCodecType type, const uint8_t* data, + const AMediaCodecBufferInfo& info) { + if (output_file_.IsOpen() && !output_file_.WriteFrame(info.size, data)) { + printf("[ERR] Failed to write encoded buffer into file.\n"); } } @@ -179,8 +199,11 @@ public: protected: void SetUp() override { - encoder_ = MediaCodecEncoder::Create(g_env->input_file_path(), g_env->visible_size()); + encoder_ = MediaCodecEncoder::Create(g_env->input_file_path(), g_env->codec(), + g_env->visible_size(), g_env->use_sw_encoder()); ASSERT_TRUE(encoder_); + g_env->configure_cb()->OnCodecReady(encoder_.get()); + encoder_->Rewind(); ASSERT_TRUE(encoder_->Configure(static_cast<int32_t>(g_env->requested_bitrate()), @@ -191,18 +214,22 @@ protected: void TearDown() override { EXPECT_TRUE(encoder_->Stop()); - output_file_.close(); + output_file_.Close(); encoder_.reset(); } bool CreateOutputFile() { if (g_env->output_file_path().empty()) return false; - output_file_.open(g_env->output_file_path(), std::ofstream::binary); - if (!output_file_.is_open()) { + if (!output_file_.Open(g_env->output_file_path(), g_env->codec())) { printf("[ERR] Failed to open file: %s\n", g_env->output_file_path().c_str()); return false; } + if (!output_file_.WriteHeader(g_env->visible_size(), g_env->requested_framerate(), 0)) { + printf("[ERR] Failed to write file header\n"); + return false; + } + return true; } @@ -214,7 +241,7 @@ protected: std::unique_ptr<MediaCodecEncoder> encoder_; // The output file to write the encoded video bitstream. - std::ofstream output_file_; + OutputFile output_file_; // Used to accumulate the output buffer size. size_t total_output_buffer_size_; }; @@ -260,7 +287,7 @@ TEST_F(C2VideoEncoderE2ETest, TestSimpleEncode) { // Write the output buffers to file. if (CreateOutputFile()) { encoder_->SetOutputBufferReadyCb(std::bind(&C2VideoEncoderE2ETest::WriteOutputBufferToFile, - this, std::placeholders::_1, + this, g_env->codec(), std::placeholders::_1, std::placeholders::_2)); } encoder_->set_run_at_fps(g_env->run_at_fps()); @@ -325,6 +352,7 @@ bool GetOption(int argc, char** argv, android::CmdlineArgs* args) { {"test_stream_data", required_argument, nullptr, 't'}, {"run_at_fps", no_argument, nullptr, 'r'}, {"num_encoded_frames", required_argument, nullptr, 'n'}, + {"use_sw_encoder", no_argument, nullptr, 's'}, {nullptr, 0, nullptr, 0}, }; @@ -340,6 +368,9 @@ bool GetOption(int argc, char** argv, android::CmdlineArgs* args) { case 'n': args->num_encoded_frames = static_cast<size_t>(atoi(optarg)); break; + case 's': + args->use_sw_encoder = true; + break; default: printf("[WARN] Unknown option: getopt_long() returned code 0x%x.\n", opt); break; @@ -353,12 +384,13 @@ bool GetOption(int argc, char** argv, android::CmdlineArgs* args) { return true; } -int RunEncoderTests(char** test_args, int test_args_count) { +int RunEncoderTests(char** test_args, int test_args_count, android::ConfigureCallback* cb) { android::CmdlineArgs args; if (!GetOption(test_args_count, test_args, &args)) return EXIT_FAILURE; android::g_env = reinterpret_cast<android::C2VideoEncoderTestEnvironment*>( - testing::AddGlobalTestEnvironment(new android::C2VideoEncoderTestEnvironment(args))); + testing::AddGlobalTestEnvironment( + new android::C2VideoEncoderTestEnvironment(args, cb))); testing::InitGoogleTest(&test_args_count, test_args); return RUN_ALL_TESTS(); } diff --git a/tests/c2_e2e_test/jni/video_frame.cpp b/tests/c2_e2e_test/jni/video_frame.cpp index a15fe27..6a9b361 100644 --- a/tests/c2_e2e_test/jni/video_frame.cpp +++ b/tests/c2_e2e_test/jni/video_frame.cpp @@ -9,7 +9,7 @@ #include "video_frame.h" -#include <utils/Log.h> +#include <log/log.h> namespace android { diff --git a/tests/c2_e2e_test/src/org/chromium/c2/test/E2eTestActivity.java b/tests/c2_e2e_test/src/org/chromium/c2/test/E2eTestActivity.java index 140ff82..6b24ea6 100644 --- a/tests/c2_e2e_test/src/org/chromium/c2/test/E2eTestActivity.java +++ b/tests/c2_e2e_test/src/org/chromium/c2/test/E2eTestActivity.java @@ -7,7 +7,10 @@ package org.chromium.c2.test; import android.app.Activity; +import android.content.BroadcastReceiver; +import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.os.AsyncTask; import android.os.Bundle; import android.os.Handler; @@ -34,7 +37,9 @@ public class E2eTestActivity extends Activity implements SurfaceHolder.Callback private Size mExpectedSize; private CountDownLatch mLatch; - private long mDecoderPtr; + private long mCodecPtr = 0; + + private BroadcastReceiver mCodecConfigReceiver; @Override protected void onCreate(Bundle savedInstanceState) { @@ -48,11 +53,17 @@ public class E2eTestActivity extends Activity implements SurfaceHolder.Callback mSurfaceView.getHolder().addCallback(this); mCanStartTest = !getIntent().getBooleanExtra("delay-start", false); + + mCodecConfigReceiver = new CodecReadyReceiver(); + registerReceiver( + mCodecConfigReceiver, + new IntentFilter("org.chromium.c2.test.CHECK_CODEC_CONFIGURED")); } @Override protected void onDestroy() { super.onDestroy(); + unregisterReceiver(mCodecConfigReceiver); // gtest can't reuse a process System.exit(0); } @@ -99,9 +110,9 @@ public class E2eTestActivity extends Activity implements SurfaceHolder.Callback }); } - void onDecoderReady(long decoderPtr) { + void onCodecReady(long codecPtr) { synchronized (this) { - mDecoderPtr = decoderPtr; + mCodecPtr = codecPtr; } } @@ -114,8 +125,8 @@ public class E2eTestActivity extends Activity implements SurfaceHolder.Callback } synchronized (this) { - if (mDecoderPtr != 0) { - stopDecoderLoop(mDecoderPtr); + if (mCodecPtr != 0) { + stopDecoderLoop(mCodecPtr); } } } @@ -166,6 +177,17 @@ public class E2eTestActivity extends Activity implements SurfaceHolder.Callback } } + class CodecReadyReceiver extends BroadcastReceiver { + @Override + public void onReceive(Context ctx, Intent intent) { + boolean ready; + synchronized (E2eTestActivity.this) { + ready = mCodecPtr != 0; + } + setResultCode(ready ? 1 : 0); + } + } + public native int c2VideoTest( boolean encode, String[] testArgs, int testArgsCount, Surface surface, String tmpFile); |