diff options
Diffstat (limited to 'accel/v4l2_video_decode_accelerator.h')
| -rw-r--r-- | accel/v4l2_video_decode_accelerator.h | 497 |
1 files changed, 497 insertions, 0 deletions
diff --git a/accel/v4l2_video_decode_accelerator.h b/accel/v4l2_video_decode_accelerator.h new file mode 100644 index 0000000..99076ed --- /dev/null +++ b/accel/v4l2_video_decode_accelerator.h @@ -0,0 +1,497 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// This file contains an implementation of VideoDecodeAccelerator +// that utilizes hardware video decoders, which expose Video4Linux 2 API +// (http://linuxtv.org/downloads/v4l-dvb-apis/). +// Note: ported from Chromium commit head: 85fdf90 +// Note: image processor is not ported. + +#ifndef MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ +#define MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <list> +#include <memory> +#include <queue> +#include <vector> + +#include "base/callback_forward.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/synchronization/waitable_event.h" +#include "base/threading/thread.h" +#include "picture.h" +#include "size.h" +#include "v4l2_device.h" +#include "video_decode_accelerator.h" + +namespace media { + +// This class handles video accelerators directly through a V4L2 device exported +// by the hardware blocks. +// +// The threading model of this class is driven by the fact that it needs to +// interface two fundamentally different event queues -- the one Chromium +// provides through MessageLoop, and the one driven by the V4L2 devices which +// is waited on with epoll(). There are three threads involved in this class: +// +// * The child thread, which is the main GPU process thread which calls the +// VideoDecodeAccelerator entry points. Calls from this thread +// generally do not block (with the exception of Initialize() and Destroy()). +// They post tasks to the decoder_thread_, which actually services the task +// and calls back when complete through the +// VideoDecodeAccelerator::Client interface. +// * The decoder_thread_, owned by this class. It services API tasks, through +// the *Task() routines, as well as V4L2 device events, through +// ServiceDeviceTask(). Almost all state modification is done on this thread +// (this doesn't include buffer (re)allocation sequence, see below). +// * The device_poll_thread_, owned by this class. All it does is epoll() on +// the V4L2 in DevicePollTask() and schedule a ServiceDeviceTask() on the +// decoder_thread_ when something interesting happens. +// TODO(sheu): replace this thread with an TYPE_IO decoder_thread_. +// +// Note that this class has (almost) no locks, apart from the pictures_assigned_ +// WaitableEvent. Everything (apart from buffer (re)allocation) is serviced on +// the decoder_thread_, so there are no synchronization issues. +// ... well, there are, but it's a matter of getting messages posted in the +// right order, not fiddling with locks. +// Buffer creation is a two-step process that is serviced partially on the +// Child thread, because we need to wait for the client to provide textures +// for the buffers we allocate. We cannot keep the decoder thread running while +// the client allocates Pictures for us, because we need to REQBUFS first to get +// the required number of output buffers from the device and that cannot be done +// unless we free the previous set of buffers, leaving the decoding in a +// inoperable state for the duration of the wait for Pictures. So to prevent +// subtle races (esp. if we get Reset() in the meantime), we block the decoder +// thread while we wait for AssignPictureBuffers from the client. +// +// V4L2VideoDecodeAccelerator may use image processor to convert the output. +// There are three cases: +// Flush: V4L2VDA should wait until image processor returns all processed +// frames. +// Reset: V4L2VDA doesn't need to wait for image processor. When image processor +// returns an old frame, drop it. +// Resolution change: V4L2VDA destroy image processor when destroying output +// buffrers. We cannot drop any frame during resolution change. So V4L2VDA +// should destroy output buffers after image processor returns all the frames. +class V4L2VideoDecodeAccelerator + : public VideoDecodeAccelerator { + public: + V4L2VideoDecodeAccelerator( + const scoped_refptr<V4L2Device>& device); + ~V4L2VideoDecodeAccelerator() override; + + // VideoDecodeAccelerator implementation. + // Note: Initialize() and Destroy() are synchronous. + bool Initialize(const Config& config, Client* client) override; + void Decode(const BitstreamBuffer& bitstream_buffer) override; + void AssignPictureBuffers(const std::vector<PictureBuffer>& buffers) override; + void ImportBufferForPicture( + int32_t picture_buffer_id, + VideoPixelFormat pixel_format, + const NativePixmapHandle& native_pixmap_handle) override; + void ReusePictureBuffer(int32_t picture_buffer_id) override; + void Flush() override; + void Reset() override; + void Destroy() override; + bool TryToSetupDecodeOnSeparateThread( + const base::WeakPtr<Client>& decode_client, + const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) + override; + + static VideoDecodeAccelerator::SupportedProfiles GetSupportedProfiles(); + + private: + // These are rather subjectively tuned. + enum { + kInputBufferCount = 8, + // TODO(posciak): determine input buffer size based on level limits. + // See http://crbug.com/255116. + // Input bitstream buffer size for up to 1080p streams. + kInputBufferMaxSizeFor1080p = 1024 * 1024, + // Input bitstream buffer size for up to 4k streams. + kInputBufferMaxSizeFor4k = 4 * kInputBufferMaxSizeFor1080p, + // This is originally from media/base/limits.h in Chromium. + kMaxVideoFrames = 4, + // Number of output buffers to use for each VDA stage above what's required + // by the decoder (e.g. DPB size, in H264). We need + // limits::kMaxVideoFrames to fill up the GpuVideoDecode pipeline, + // and +1 for a frame in transit. + kDpbOutputBufferExtraCount = kMaxVideoFrames + 1, + // Number of extra output buffers if image processor is used. + kDpbOutputBufferExtraCountForImageProcessor = 1, + }; + + // Internal state of the decoder. + enum State { + kUninitialized, // Initialize() not yet called. + kInitialized, // Initialize() returned true; ready to start decoding. + kDecoding, // DecodeBufferInitial() successful; decoding frames. + kResetting, // Presently resetting. + // Performing resolution change and waiting for image processor to return + // all frames. + kChangingResolution, + // Requested new PictureBuffers via ProvidePictureBuffers(), awaiting + // AssignPictureBuffers(). + kAwaitingPictureBuffers, + kError, // Error in kDecoding state. + }; + + enum OutputRecordState { + kFree, // Ready to be queued to the device. + kAtDevice, // Held by device. + kAtProcessor, // Held by image processor. + kAtClient, // Held by client of V4L2VideoDecodeAccelerator. + }; + + enum BufferId { + kFlushBufferId = -2 // Buffer id for flush buffer, queued by FlushTask(). + }; + + // Auto-destruction reference for BitstreamBuffer, for message-passing from + // Decode() to DecodeTask(). + struct BitstreamBufferRef; + + // Record for decoded pictures that can be sent to PictureReady. + struct PictureRecord { + PictureRecord(bool cleared, const Picture& picture); + ~PictureRecord(); + bool cleared; // Whether the texture is cleared and safe to render from. + Picture picture; // The decoded picture. + }; + + // Record for input buffers. + struct InputRecord { + bool at_device = false; // held by device. + std::unique_ptr<BitstreamBufferRef> bitstream_buffer; + }; + + // Record for output buffers. + struct OutputRecord { + OutputRecord(); + OutputRecord(OutputRecord&&) = default; + ~OutputRecord(); + OutputRecordState state; + int32_t picture_id; // picture buffer id as returned to PictureReady(). + bool cleared; // Whether the texture is cleared and safe to render + // from. See TextureManager for details. + // Output fds of the decoded frame. + std::vector<base::ScopedFD> output_fds; + // offsets of each decoded frame from each fd in |output_fds|. + std::vector<size_t> offsets; + }; + + // + // Decoding tasks, to be run on decode_thread_. + // + + // Task to finish initialization on decoder_thread_. + void InitializeTask(); + + // Enqueue a BitstreamBuffer to decode. This will enqueue a buffer to the + // decoder_input_queue_, then queue a DecodeBufferTask() to actually decode + // the buffer. + void DecodeTask(const BitstreamBuffer& bitstream_buffer); + + // Decode from the buffers queued in decoder_input_queue_. Calls + // DecodeBufferInitial() or DecodeBufferContinue() as appropriate. + void DecodeBufferTask(); + // Schedule another DecodeBufferTask() if we're behind. + void ScheduleDecodeBufferTaskIfNeeded(); + + // Return true if we should continue to schedule DecodeBufferTask()s after + // completion. + bool DecodeBufferInitial(); + bool DecodeBufferContinue(); + + // Flush data for one decoded frame. + bool TrySubmitInputFrame(); + + // Allocate V4L2 buffers and assign them to |buffers| provided by the client + // via AssignPictureBuffers() on decoder thread. + void AssignPictureBuffersTask(const std::vector<PictureBuffer>& buffers); + + // Use buffer backed by dmabuf file descriptors in |dmabuf_fds| for the + // OutputRecord associated with |picture_buffer_id|, taking ownership of the + // file descriptors. + void ImportBufferForPictureTask(int32_t picture_buffer_id, + std::vector<size_t> offsets, + std::vector<base::ScopedFD> dmabuf_fds); + + // Service I/O on the V4L2 devices. This task should only be scheduled from + // DevicePollTask(). If |event_pending| is true, one or more events + // on file descriptor are pending. + void ServiceDeviceTask(bool event_pending); + // Handle the various device queues. + void Enqueue(); + void Dequeue(); + // Dequeue one input buffer. Return true if success. + bool DequeueInputBuffer(); + // Dequeue one output buffer. Return true if success. + bool DequeueOutputBuffer(); + + // Return true if there is a resolution change event pending. + bool DequeueResolutionChangeEvent(); + + // Enqueue a buffer on the corresponding queue. + bool EnqueueInputRecord(); + bool EnqueueOutputRecord(); + + // Process a ReusePictureBuffer() API call. The API call create an EGLSync + // object on the main (GPU process) thread; we will record this object so we + // can wait on it before reusing the buffer. + void ReusePictureBufferTask(int32_t picture_buffer_id); + + // Flush() task. Child thread should not submit any more buffers until it + // receives the NotifyFlushDone callback. This task will schedule an empty + // BitstreamBufferRef (with input_id == kFlushBufferId) to perform the flush. + void FlushTask(); + // Notify the client of a flush completion, if required. This should be + // called any time a relevant queue could potentially be emptied: see + // function definition. + void NotifyFlushDoneIfNeeded(); + // Returns true if VIDIOC_DECODER_CMD is supported. + bool IsDecoderCmdSupported(); + // Send V4L2_DEC_CMD_START to the driver. Return true if success. + bool SendDecoderCmdStop(); + + // Reset() task. Drop all input buffers. If V4L2VDA is not doing resolution + // change or waiting picture buffers, call FinishReset. + void ResetTask(); + // This will schedule a ResetDoneTask() that will send the NotifyResetDone + // callback, then set the decoder state to kResetting so that all intervening + // tasks will drain. + void FinishReset(); + void ResetDoneTask(); + + // Device destruction task. + void DestroyTask(); + + // Start |device_poll_thread_|. + bool StartDevicePoll(); + + // Stop |device_poll_thread_|. + bool StopDevicePoll(); + + bool StopInputStream(); + bool StopOutputStream(); + + void StartResolutionChange(); + void FinishResolutionChange(); + + // Try to get output format and visible size, detected after parsing the + // beginning of the stream. Sets |again| to true if more parsing is needed. + // |visible_size| could be nullptr and ignored. + bool GetFormatInfo(struct v4l2_format* format, + Size* visible_size, + bool* again); + // Create output buffers for the given |format| and |visible_size|. + bool CreateBuffersForFormat(const struct v4l2_format& format, + const Size& visible_size); + + // Try to get |visible_size|. Return visible size, or, if querying it is not + // supported or produces invalid size, return |coded_size| instead. + Size GetVisibleSize(const Size& coded_size); + + // + // Device tasks, to be run on device_poll_thread_. + // + + // The device task. + void DevicePollTask(bool poll_device); + + // + // Safe from any thread. + // + + // Error notification (using PostTask() to child thread, if necessary). + void NotifyError(Error error); + + // Set the decoder_state_ to kError and notify the client (if necessary). + void SetErrorState(Error error); + + // + // Other utility functions. Called on decoder_thread_, unless + // decoder_thread_ is not yet started, in which case the child thread can call + // these (e.g. in Initialize() or Destroy()). + // + + // Create the buffers we need. + bool CreateInputBuffers(); + bool CreateOutputBuffers(); + + // Destroy buffers. + void DestroyInputBuffers(); + // In contrast to DestroyInputBuffers, which is called only on destruction, + // we call DestroyOutputBuffers also during playback, on resolution change. + // Even if anything fails along the way, we still want to go on and clean + // up as much as possible, so return false if this happens, so that the + // caller can error out on resolution change. + bool DestroyOutputBuffers(); + + // Set input and output formats before starting decode. + bool SetupFormats(); + + // + // Methods run on child thread. + // + + // Send decoded pictures to PictureReady. + void SendPictureReady(); + + // Callback that indicates a picture has been cleared. + void PictureCleared(); + + // Our original calling task runner for the child thread. + scoped_refptr<base::SingleThreadTaskRunner> child_task_runner_; + + // Task runner Decode() and PictureReady() run on. + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; + + // WeakPtr<> pointing to |this| for use in posting tasks from the decoder or + // device worker threads back to the child thread. Because the worker threads + // are members of this class, any task running on those threads is guaranteed + // that this object is still alive. As a result, tasks posted from the child + // thread to the decoder or device thread should use base::Unretained(this), + // and tasks posted the other way should use |weak_this_|. + base::WeakPtr<V4L2VideoDecodeAccelerator> weak_this_; + + // To expose client callbacks from VideoDecodeAccelerator. + // NOTE: all calls to these objects *MUST* be executed on + // child_task_runner_. + std::unique_ptr<base::WeakPtrFactory<Client>> client_ptr_factory_; + base::WeakPtr<Client> client_; + // Callbacks to |decode_client_| must be executed on |decode_task_runner_|. + base::WeakPtr<Client> decode_client_; + + // + // Decoder state, owned and operated by decoder_thread_. + // Before decoder_thread_ has started, the decoder state is managed by + // the child (main) thread. After decoder_thread_ has started, the decoder + // thread should be the only one managing these. + // + + // This thread services tasks posted from the VDA API entry points by the + // child thread and device service callbacks posted from the device thread. + base::Thread decoder_thread_; + // Decoder state machine state. + State decoder_state_; + + Config::OutputMode output_mode_; + + // BitstreamBuffer we're presently reading. + std::unique_ptr<BitstreamBufferRef> decoder_current_bitstream_buffer_; + // The V4L2Device this class is operating upon. + scoped_refptr<V4L2Device> device_; + // FlushTask() and ResetTask() should not affect buffers that have been + // queued afterwards. For flushing or resetting the pipeline then, we will + // delay these buffers until after the flush or reset completes. + int decoder_delay_bitstream_buffer_id_; + // We track the number of buffer decode tasks we have scheduled, since each + // task execution should complete one buffer. If we fall behind (due to + // resource backpressure, etc.), we'll have to schedule more to catch up. + int decoder_decode_buffer_tasks_scheduled_; + // Picture buffers held by the client. + int decoder_frames_at_client_; + + // Are we flushing? + bool decoder_flushing_; + // True if VIDIOC_DECODER_CMD is supported. + bool decoder_cmd_supported_; + // True if flushing is waiting for last output buffer. After + // VIDIOC_DECODER_CMD is sent to the driver, this flag will be set to true to + // wait for the last output buffer. When this flag is true, flush done will + // not be sent. After an output buffer that has the flag V4L2_BUF_FLAG_LAST is + // received, this is set to false. + bool flush_awaiting_last_output_buffer_; + + // Got a reset request while we were performing resolution change or waiting + // picture buffers. + bool reset_pending_; + // Input queue for decoder_thread_: BitstreamBuffers in. + std::queue<std::unique_ptr<BitstreamBufferRef>> decoder_input_queue_; + + // + // Hardware state and associated queues. Since decoder_thread_ services + // the hardware, decoder_thread_ owns these too. + // output_buffer_map_, free_output_buffers_ and output_planes_count_ are an + // exception during the buffer (re)allocation sequence, when the + // decoder_thread_ is blocked briefly while the Child thread manipulates + // them. + // + + // Completed decode buffers. + std::queue<int> input_ready_queue_; + + // Input buffer state. + bool input_streamon_; + // Input buffers enqueued to device. + int input_buffer_queued_count_; + // Input buffers ready to use, as a LIFO since we don't care about ordering. + std::vector<int> free_input_buffers_; + // Mapping of int index to input buffer record. + std::vector<InputRecord> input_buffer_map_; + // The size of input buffer that bitstream buffer can be copied. + size_t input_buffer_size_; + + // Output buffer state. + bool output_streamon_; + // Output buffers enqueued to device. + int output_buffer_queued_count_; + // Output buffers ready to use, as a FIFO since we want oldest-first to hide + // synchronization latency with GL. + std::list<int> free_output_buffers_; + // Mapping of int index to output buffer record. + std::vector<OutputRecord> output_buffer_map_; + // Required size of DPB for decoding. + int output_dpb_size_; + + // Number of planes (i.e. separate memory buffers) for output. + size_t output_planes_count_; + + // Pictures that are ready but not sent to PictureReady yet. + std::queue<PictureRecord> pending_picture_ready_; + + // The number of pictures that are sent to PictureReady and will be cleared. + int picture_clearing_count_; + + // Output picture coded size. + Size coded_size_; + + // Output picture visible size. + Size visible_size_; + + // + // The device polling thread handles notifications of V4L2 device changes. + // + + // The thread. + base::Thread device_poll_thread_; + + // + // Other state, held by the child (main) thread. + // + + // The codec we'll be decoding for. + VideoCodecProfile video_profile_; + // Chosen input format for video_profile_. + uint32_t input_format_fourcc_; + // Chosen output format. + uint32_t output_format_fourcc_; + + // Input format V4L2 fourccs this class supports. + static const uint32_t supported_input_fourccs_[]; + + // The WeakPtrFactory for |weak_this_|. + base::WeakPtrFactory<V4L2VideoDecodeAccelerator> weak_this_factory_; + + DISALLOW_COPY_AND_ASSIGN(V4L2VideoDecodeAccelerator); +}; + +} // namespace media + +#endif // MEDIA_GPU_V4L2_VIDEO_DECODE_ACCELERATOR_H_ |