drm_hwcomposer: Add non-blocking commit support

This change fixes FPS drop on multidisplay devices.

Also in some cases it gives graphics pipeline more free time to draw
in advance.

Signed-off-by: Roman Stratiienko <roman.o.stratiienko@globallogic.com>

2 files changed, 167 insertions, 22 deletions

diff --git a/drm/DrmAtomicStateManager.cpp b/drm/DrmAtomicStateManager.cpp
index 65fb19e..bca446c 100644
--- a/drm/DrmAtomicStateManager.cpp
+++ b/drm/DrmAtomicStateManager.cpp
@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+#undef NDEBUG /* Required for assert to work */
+
 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
 #define LOG_TAG "hwc-drm-atomic-state-manager"
 
@@ -26,6 +28,7 @@
 #include <utils/Trace.h>
 
 #include <array>
+#include <cassert>
 #include <cstdlib>
 #include <ctime>
 #include <sstream>
@@ -70,13 +73,15 @@ auto DrmAtomicStateManager::CommitFrame(AtomicCommitArgs &args) -> int {
     return -ENOMEM;
   }
 
-  int64_t out_fence = -1;
-  if (crtc->GetOutFencePtrProperty() &&
-      !crtc->GetOutFencePtrProperty().AtomicSet(*pset, uint64_t(&out_fence))) {
+  int out_fence = -1;
+  if (!crtc->GetOutFencePtrProperty().AtomicSet(*pset, uint64_t(&out_fence))) {
     return -EINVAL;
   }
 
+  bool nonblock = true;
+
   if (args.active) {
+    nonblock = false;
     new_frame_state.crtc_active_state = *args.active;
     if (!crtc->GetActiveProperty().AtomicSet(*pset, *args.active ? 1 : 0) ||
         !connector->GetCrtcIdProperty().AtomicSet(*pset, crtc->GetId())) {
@@ -100,7 +105,6 @@ auto DrmAtomicStateManager::CommitFrame(AtomicCommitArgs &args) -> int {
   auto unused_planes = new_frame_state.used_planes;
 
   if (args.composition) {
-    new_frame_state.used_framebuffers.clear();
     new_frame_state.used_planes.clear();
 
     for (auto &joining : args.composition->plan) {
@@ -130,31 +134,126 @@ auto DrmAtomicStateManager::CommitFrame(AtomicCommitArgs &args) -> int {
   }
 
   uint32_t flags = DRM_MODE_ATOMIC_ALLOW_MODESET;
-  if (args.test_only)
-    flags |= DRM_MODE_ATOMIC_TEST_ONLY;
+
+  if (args.test_only) {
+    return drmModeAtomicCommit(drm->GetFd(), pset.get(),
+                               flags | DRM_MODE_ATOMIC_TEST_ONLY, drm);
+  }
+
+  if (last_present_fence_) {
+    ATRACE_NAME("WaitPriorFramePresented");
+
+    constexpr int kTimeoutMs = 500;
+    int err = sync_wait(last_present_fence_.Get(), kTimeoutMs);
+    if (err != 0) {
+      ALOGE("sync_wait(fd=%i) returned: %i (errno: %i)",
+            last_present_fence_.Get(), err, errno);
+    }
+
+    CleanupPriorFrameResources();
+  }
+
+  if (nonblock) {
+    flags |= DRM_MODE_ATOMIC_NONBLOCK;
+  }
 
   int err = drmModeAtomicCommit(drm->GetFd(), pset.get(), flags, drm);
+
   if (err != 0) {
-    if (!args.test_only)
-      ALOGE("Failed to commit pset ret=%d\n", err);
+    ALOGE("Failed to commit pset ret=%d\n", err);
     return err;
   }
 
-  if (!args.test_only) {
-    if (args.display_mode) {
-      /* TODO(nobody): we still need this for synthetic vsync, remove after
-       * vsync reworked */
-      connector->SetActiveMode(*args.display_mode);
+  if (nonblock) {
+    last_present_fence_ = UniqueFd::Dup(out_fence);
+    staged_frame_state_ = std::move(new_frame_state);
+    frames_staged_++;
+    ptt_->Notify();
+  } else {
+    active_frame_state_ = std::move(new_frame_state);
+  }
+
+  if (args.display_mode) {
+    /* TODO(nobody): we still need this for synthetic vsync, remove after
+     * vsync reworked */
+    connector->SetActiveMode(*args.display_mode);
+  }
+
+  args.out_fence = UniqueFd(out_fence);
+
+  return 0;
+}
+
+PresentTrackerThread::PresentTrackerThread(DrmAtomicStateManager *st_man)
+    : st_man_(st_man),
+      mutex_(&st_man_->pipe_->device->GetResMan().GetMainLock()) {
+  pt_ = std::thread(&PresentTrackerThread::PresentTrackerThreadFn, this);
+}
+
+PresentTrackerThread::~PresentTrackerThread() {
+  ALOGI("PresentTrackerThread successfully destroyed");
+}
+
+void PresentTrackerThread::PresentTrackerThreadFn() {
+  /* object should be destroyed on thread exit */
+  auto self = std::unique_ptr<PresentTrackerThread>(this);
+
+  int tracking_at_the_moment = -1;
+
+  for (;;) {
+    UniqueFd present_fence;
+
+    {
+      std::unique_lock lk(*mutex_);
+      cv_.wait(lk, [&] {
+        return st_man_ == nullptr ||
+               st_man_->frames_staged_ > tracking_at_the_moment;
+      });
+
+      if (st_man_ == nullptr) {
+        break;
+      }
+
+      tracking_at_the_moment = st_man_->frames_staged_;
+
+      present_fence = UniqueFd::Dup(st_man_->last_present_fence_.Get());
+      if (!present_fence) {
+        continue;
+      }
     }
 
-    active_frame_state_ = std::move(new_frame_state);
+    {
+      ATRACE_NAME("AsyncWaitForBuffersSwap");
+      constexpr int kTimeoutMs = 500;
+      int err = sync_wait(present_fence.Get(), kTimeoutMs);
+      if (err != 0) {
+        ALOGE("sync_wait(fd=%i) returned: %i (errno: %i)", present_fence.Get(),
+              err, errno);
+      }
+    }
 
-    if (crtc->GetOutFencePtrProperty()) {
-      args.out_fence = UniqueFd((int)out_fence);
+    {
+      std::unique_lock lk(*mutex_);
+      if (st_man_ == nullptr) {
+        break;
+      }
+
+      /* If resources is already cleaned-up by main thread, skip */
+      if (tracking_at_the_moment > st_man_->frames_tracked_) {
+        st_man_->CleanupPriorFrameResources();
+      }
     }
   }
+}
 
-  return 0;
+void DrmAtomicStateManager::CleanupPriorFrameResources() {
+  assert(frames_staged_ - frames_tracked_ == 1);
+  assert(last_present_fence_);
+
+  ATRACE_NAME("CleanupPriorFrameResources");
+  frames_tracked_++;
+  active_frame_state_ = std::move(staged_frame_state_);
+  last_present_fence_ = {};
 }
 
 auto DrmAtomicStateManager::ExecuteAtomicCommit(AtomicCommitArgs &args) -> int {
diff --git a/drm/DrmAtomicStateManager.h b/drm/DrmAtomicStateManager.h
index 08a1c13..a50a214 100644
--- a/drm/DrmAtomicStateManager.h
+++ b/drm/DrmAtomicStateManager.h
@@ -49,11 +49,45 @@ struct AtomicCommitArgs {
   }
 };
 
+class PresentTrackerThread {
+ public:
+  explicit PresentTrackerThread(DrmAtomicStateManager *st_man);
+
+  ~PresentTrackerThread();
+
+  void Stop() {
+    /* Exit thread by signalling that object is no longer valid */
+    st_man_ = nullptr;
+    Notify();
+    pt_.detach();
+  }
+
+  void Notify() {
+    cv_.notify_all();
+  }
+
+ private:
+  DrmAtomicStateManager *st_man_{};
+
+  void PresentTrackerThreadFn();
+
+  std::condition_variable cv_;
+  std::thread pt_;
+  std::mutex *mutex_;
+};
+
 class DrmAtomicStateManager {
+  friend class PresentTrackerThread;
+
  public:
-  explicit DrmAtomicStateManager(DrmDisplayPipeline *pipe) : pipe_(pipe){};
+  explicit DrmAtomicStateManager(DrmDisplayPipeline *pipe)
+      : pipe_(pipe),
+        ptt_(std::make_unique<PresentTrackerThread>(this).release()){};
+
   DrmAtomicStateManager(const DrmAtomicStateManager &) = delete;
-  ~DrmAtomicStateManager() = default;
+  ~DrmAtomicStateManager() {
+    ptt_->Stop();
+  }
 
   auto ExecuteAtomicCommit(AtomicCommitArgs &args) -> int;
   auto ActivateDisplayUsingDPMS() -> int;
@@ -70,20 +104,32 @@ class DrmAtomicStateManager {
 
     DrmModeUserPropertyBlobUnique mode_blob;
 
+    int release_fence_pt_index{};
+
     /* To avoid setting the inactive state twice, which will fail the commit */
     bool crtc_active_state{};
   } active_frame_state_;
 
   auto NewFrameState() -> KmsState {
+    auto *prev_frame_state = &active_frame_state_;
     return (KmsState){
-        .used_planes = active_frame_state_.used_planes,
-        .used_framebuffers = active_frame_state_.used_framebuffers,
-        .crtc_active_state = active_frame_state_.crtc_active_state,
+        .used_planes = prev_frame_state->used_planes,
+        .crtc_active_state = prev_frame_state->crtc_active_state,
     };
   }
 
   DrmDisplayPipeline *const pipe_;
+
+  void CleanupPriorFrameResources();
+
+  /* Present (swap) tracking */
+  PresentTrackerThread *ptt_;
+  KmsState staged_frame_state_;
+  UniqueFd last_present_fence_;
+  int frames_staged_{};
+  int frames_tracked_{};
 };
+
 }  // namespace android
 
 #endif  // ANDROID_DRM_DISPLAY_COMPOSITOR_H_