mm-video-v4l2: venc: Support for frame accurate dynamic config

To ensure frame accurate parameter configuration,
support for 2 new vendor extensions is added:
1. “vendor.qti-ext-enc-app-input-control.enable”
2. “vendor.qti-ext-enc-input-trigger.timestamp”
With these extensions dynamic video encoder properties
can be applied at specific frame.

CRs-Fixed: 2066874
Change-Id: Ie52e945216c37199f2d83a6c1513c43eb218cc12

5 files changed, 110 insertions, 26 deletions

diff --git a/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp b/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp
index 04ee892a..2d8b09aa 100644
--- a/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp
+++ b/mm-video-v4l2/vidc/vdec/src/omx_vdec_extensions.hpp
@@ -73,7 +73,7 @@ OMX_ERRORTYPE omx_vdec::get_vendor_extension_config(
 OMX_ERRORTYPE omx_vdec::set_vendor_extension_config(
                 OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *ext) {
 
-    ALOGI("set_vendor_extension_config");
+    DEBUG_PRINT_LOW("set_vendor_extension_config");
     if (ext->nIndex >= mVendorExtensionStore.size()) {
         DEBUG_PRINT_ERROR("unrecognized vendor extension index (%u) max(%u)",
                 ext->nIndex, mVendorExtensionStore.size());
diff --git a/mm-video-v4l2/vidc/venc/inc/omx_video_base.h b/mm-video-v4l2/vidc/venc/inc/omx_video_base.h
index 74691ec5..3d6743f3 100644
--- a/mm-video-v4l2/vidc/venc/inc/omx_video_base.h
+++ b/mm-video-v4l2/vidc/venc/inc/omx_video_base.h
@@ -678,11 +678,10 @@ class omx_video: public qc_omx_component
         QOMX_INDEXDOWNSCALAR m_sParamDownScalar;
         PrependSPSPPSToIDRFramesParams m_sPrependSPSPPS;
         struct timestamp_info {
-            OMX_U64 m_TimeStamp;
-            bool is_buffer_pending;
-            OMX_BUFFERHEADERTYPE *pending_buffer;
+            OMX_S64 ts;
+            omx_cmd_queue deferred_inbufq;
             pthread_mutex_t m_lock;
-        } timestamp;
+        } m_TimeStampInfo;
         OMX_U32 m_sExtraData;
         OMX_U32 m_input_msg_id;
         OMX_VIDEO_CONFIG_ANDROID_INTRAREFRESHTYPE m_sConfigIntraRefresh;
@@ -691,6 +690,8 @@ class omx_video: public qc_omx_component
         OMX_VIDEO_PARAM_ANDROID_TEMPORALLAYERINGTYPE m_sParamTemporalLayers;
         OMX_VIDEO_CONFIG_ANDROID_TEMPORALLAYERINGTYPE m_sConfigTemporalLayers;
         QOMX_ENABLETYPE m_sParamAVTimerTimestampMode;   // use VT-timestamps in gralloc-handle
+        QOMX_ENABLETYPE m_sParamControlInputQueue;
+        OMX_TIME_CONFIG_TIMESTAMPTYPE m_sConfigInputTrigTS;
 
         // fill this buffer queue
         omx_cmd_queue m_ftb_q;
diff --git a/mm-video-v4l2/vidc/venc/src/omx_video_base.cpp b/mm-video-v4l2/vidc/venc/src/omx_video_base.cpp
index 5157b5e4..2eadb145 100644
--- a/mm-video-v4l2/vidc/venc/src/omx_video_base.cpp
+++ b/mm-video-v4l2/vidc/venc/src/omx_video_base.cpp
@@ -270,8 +270,9 @@ omx_video::omx_video():
 
     mUsesColorConversion = false;
     pthread_mutex_init(&m_lock, NULL);
-    pthread_mutex_init(&timestamp.m_lock, NULL);
-    timestamp.is_buffer_pending = false;
+    pthread_mutex_init(&m_TimeStampInfo.m_lock, NULL);
+    m_TimeStampInfo.deferred_inbufq.m_size=0;
+    m_TimeStampInfo.deferred_inbufq.m_read = m_TimeStampInfo.deferred_inbufq.m_write = 0;
     sem_init(&m_cmd_lock,0,0);
     DEBUG_PRINT_LOW("meta_buffer_hdr = %p", meta_buffer_hdr);
 
@@ -310,7 +311,7 @@ omx_video::~omx_video()
     /*For V4L2 based drivers, pthread_join is done in device_close
      * so no need to do it here*/
     pthread_mutex_destroy(&m_lock);
-    pthread_mutex_destroy(&timestamp.m_lock);
+    pthread_mutex_destroy(&m_TimeStampInfo.m_lock);
     sem_destroy(&m_cmd_lock);
     DEBUG_PRINT_HIGH("m_etb_count = %" PRIu64 ", m_fbd_count = %" PRIu64, m_etb_count,
             m_fbd_count);
@@ -1272,6 +1273,10 @@ bool omx_video::execute_input_flush(void)
             m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,(OMX_BUFFERHEADERTYPE *)p2);
         }
     }
+    while (m_TimeStampInfo.deferred_inbufq.m_size) {
+        m_TimeStampInfo.deferred_inbufq.pop_entry(&p1,&p2,&ident);
+        m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,(OMX_BUFFERHEADERTYPE *)p1);
+    }
     if (mUseProxyColorFormat) {
         if (psource_frame) {
             m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,psource_frame);
@@ -1334,6 +1339,12 @@ bool omx_video::execute_flush_all(void)
             m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,(OMX_BUFFERHEADERTYPE *)p2);
         }
     }
+
+    while (m_TimeStampInfo.deferred_inbufq.m_size) {
+        m_TimeStampInfo.deferred_inbufq.pop_entry(&p1,&p2,&ident);
+        m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,(OMX_BUFFERHEADERTYPE *)p1);
+    }
+
     if(mUseProxyColorFormat) {
         if(psource_frame) {
             m_pCallbacks.EmptyBufferDone(&m_cmp,m_app_data,psource_frame);
diff --git a/mm-video-v4l2/vidc/venc/src/omx_video_encoder.cpp b/mm-video-v4l2/vidc/venc/src/omx_video_encoder.cpp
index 71c36c66..54c1732e 100644
--- a/mm-video-v4l2/vidc/venc/src/omx_video_encoder.cpp
+++ b/mm-video-v4l2/vidc/venc/src/omx_video_encoder.cpp
@@ -424,6 +424,11 @@ OMX_ERRORTYPE omx_venc::component_init(OMX_STRING role)
     m_sConfigQP.nPortIndex = (OMX_U32) PORT_INDEX_OUT;
     m_sConfigQP.nQP = 30;
 
+    OMX_INIT_STRUCT(&m_sParamControlInputQueue , QOMX_ENABLETYPE);
+    m_sParamControlInputQueue.bEnable = OMX_FALSE;
+
+    OMX_INIT_STRUCT(&m_sConfigInputTrigTS, OMX_TIME_CONFIG_TIMESTAMPTYPE);
+
     m_state                   = OMX_StateLoaded;
     m_sExtraData = 0;
 
@@ -1455,6 +1460,12 @@ OMX_ERRORTYPE  omx_venc::set_parameter(OMX_IN OMX_HANDLETYPE     hComp,
                 memcpy(&m_sParamDownScalar, paramData, sizeof(QOMX_INDEXDOWNSCALAR));
                 break;
             }
+        case OMX_QcomIndexParamVencControlInputQueue:
+            {
+                VALIDATE_OMX_PARAM_DATA(paramData, QOMX_ENABLETYPE);
+                memcpy(&m_sParamControlInputQueue, paramData, sizeof(QOMX_ENABLETYPE));
+                break;
+            }
         case OMX_IndexParamVideoSliceFMO:
         default:
             {
@@ -1841,16 +1852,21 @@ OMX_ERRORTYPE  omx_venc::set_config(OMX_IN OMX_HANDLETYPE      hComp,
             {
                 OMX_TIME_CONFIG_TIMESTAMPTYPE* pParam =
                     (OMX_TIME_CONFIG_TIMESTAMPTYPE*) configData;
-                pthread_mutex_lock(&timestamp.m_lock);
-                timestamp.m_TimeStamp = (OMX_U64)pParam->nTimestamp;
-                DEBUG_PRINT_LOW("Buffer = %p, Timestamp = %llu", timestamp.pending_buffer, (OMX_U64)pParam->nTimestamp);
-                if (timestamp.is_buffer_pending && (OMX_U64)timestamp.pending_buffer->nTimeStamp == timestamp.m_TimeStamp) {
-                    DEBUG_PRINT_INFO("Queueing back pending buffer %p", timestamp.pending_buffer);
-                    this->post_event((unsigned long)hComp,(unsigned long)timestamp.pending_buffer,m_input_msg_id);
-                    timestamp.pending_buffer = NULL;
-                    timestamp.is_buffer_pending = false;
-                }
-                pthread_mutex_unlock(&timestamp.m_lock);
+                unsigned long buf;
+                unsigned long p2;
+                unsigned long bufTime;
+
+                pthread_mutex_lock(&m_TimeStampInfo.m_lock);
+                m_TimeStampInfo.ts = (OMX_S64)pParam->nTimestamp;
+                while (m_TimeStampInfo.deferred_inbufq.m_size &&
+                       !(m_TimeStampInfo.deferred_inbufq.get_q_msg_type() > m_TimeStampInfo.ts)) {
+                    m_TimeStampInfo.deferred_inbufq.pop_entry(&buf,&p2,&bufTime);
+                    DEBUG_PRINT_INFO("Queueing back pending buffer %lu timestamp %lu", buf, bufTime);
+                    this->post_event((unsigned long)hComp, (unsigned long)buf, m_input_msg_id);
+                }
+                pthread_mutex_unlock(&m_TimeStampInfo.m_lock);
+
+                memcpy(&m_sConfigInputTrigTS, pParam, sizeof(m_sConfigInputTrigTS));
                 break;
             }
        case OMX_IndexConfigAndroidIntraRefresh:
@@ -2074,20 +2090,22 @@ bool omx_venc::dev_buffer_ready_to_queue(OMX_BUFFERHEADERTYPE *buffer)
     bool bRet = true;
 
     /* do not defer the buffer if m_TimeStamp is not initialized */
-    if (!timestamp.m_TimeStamp)
+    if (!m_sParamControlInputQueue.bEnable)
         return true;
 
-    pthread_mutex_lock(&timestamp.m_lock);
+    pthread_mutex_lock(&m_TimeStampInfo.m_lock);
 
-    if ((OMX_U64)buffer->nTimeStamp == (OMX_U64)timestamp.m_TimeStamp) {
+    if ((!m_sParamControlInputQueue.bEnable) ||
+        (OMX_S64)buffer->nTimeStamp <= (OMX_S64)m_TimeStampInfo.ts) {
         DEBUG_PRINT_LOW("ETB is ready to be queued");
     } else {
-        DEBUG_PRINT_INFO("ETB is defeffed due to timeStamp mismatch");
-        timestamp.is_buffer_pending = true;
-        timestamp.pending_buffer = buffer;
+        DEBUG_PRINT_INFO(
+            "ETB is deferred due to timeStamp mismatch buf_ts %lld m_TimeStampInfo.ts %lld",
+             (OMX_S64)buffer->nTimeStamp, (OMX_S64)m_TimeStampInfo.ts);
+        m_TimeStampInfo.deferred_inbufq.insert_entry((unsigned long)buffer, 0, buffer->nTimeStamp);
         bRet = false;
     }
-    pthread_mutex_unlock(&timestamp.m_lock);
+    pthread_mutex_unlock(&m_TimeStampInfo.m_lock);
     return bRet;
 }
 
diff --git a/mm-video-v4l2/vidc/venc/src/omx_video_extensions.hpp b/mm-video-v4l2/vidc/venc/src/omx_video_extensions.hpp
index 18df993e..7bea2a2b 100644
--- a/mm-video-v4l2/vidc/venc/src/omx_video_extensions.hpp
+++ b/mm-video-v4l2/vidc/venc/src/omx_video_extensions.hpp
@@ -54,6 +54,12 @@ void omx_video::init_vendor_extensions(VendorExtensionStore &store) {
     ADD_EXTENSION("qti-ext-enc-down-scalar", OMX_QcomIndexParamVideoDownScalar, OMX_DirOutput)
     ADD_PARAM    ("output-width", OMX_AndroidVendorValueInt32)
     ADD_PARAM_END("output-height", OMX_AndroidVendorValueInt32)
+
+    ADD_EXTENSION("qti-ext-enc-app-input-control", OMX_QcomIndexParamVencControlInputQueue, OMX_DirInput)
+    ADD_PARAM_END("enable", OMX_AndroidVendorValueInt32)
+
+    ADD_EXTENSION("qti-ext-enc-input-trigger", OMX_IndexConfigTimePosition, OMX_DirInput)
+    ADD_PARAM_END("timestamp", OMX_AndroidVendorValueInt64)
 }
 
 OMX_ERRORTYPE omx_video::get_vendor_extension_config(
@@ -118,6 +124,16 @@ OMX_ERRORTYPE omx_video::get_vendor_extension_config(
             setStatus &= vExt.setParamInt32(ext, "output-height", m_sParamDownScalar.nOutputHeight);
             break;
         }
+        case OMX_QcomIndexParamVencControlInputQueue:
+        {
+            setStatus &= vExt.setParamInt32(ext, "enable", m_sParamControlInputQueue.bEnable);
+            break;
+        }
+        case OMX_IndexConfigTimePosition:
+        {
+            setStatus &= vExt.setParamInt64(ext, "timestamp", m_sConfigInputTrigTS.nTimestamp);
+            break;
+        }
         default:
         {
             return OMX_ErrorNotImplemented;
@@ -129,7 +145,7 @@ OMX_ERRORTYPE omx_video::get_vendor_extension_config(
 OMX_ERRORTYPE omx_video::set_vendor_extension_config(
                 OMX_CONFIG_ANDROID_VENDOR_EXTENSIONTYPE *ext) {
 
-    ALOGI("set_vendor_extension_config");
+    DEBUG_PRINT_LOW("set_vendor_extension_config");
     if (ext->nIndex >= mVendorExtensionStore.size()) {
         DEBUG_PRINT_ERROR("unrecognized vendor extension index (%u) max(%u)",
                 ext->nIndex, mVendorExtensionStore.size());
@@ -289,6 +305,44 @@ OMX_ERRORTYPE omx_video::set_vendor_extension_config(
             }
             break;
         }
+        case OMX_QcomIndexParamVencControlInputQueue:
+        {
+            QOMX_ENABLETYPE controlInputQueueParam;
+            memcpy(&controlInputQueueParam, &m_sParamControlInputQueue, sizeof(QOMX_ENABLETYPE));
+            valueSet |= vExt.readParamInt32(ext, "enable", (OMX_S32 *)&(controlInputQueueParam.bEnable));
+            if (!valueSet) {
+                break;
+            }
+
+            DEBUG_PRINT_HIGH("VENDOR-EXT: set_param: control input queue enable=%u", controlInputQueueParam.bEnable);
+
+            err = set_parameter(
+                    NULL, (OMX_INDEXTYPE)OMX_QcomIndexParamVencControlInputQueue, &controlInputQueueParam);
+            if (err != OMX_ErrorNone) {
+                DEBUG_PRINT_ERROR("set_param: OMX_QcomIndexParamVencControlInputQueue failed !");
+            }
+
+            break;
+        }
+        case OMX_IndexConfigTimePosition:
+        {
+            OMX_TIME_CONFIG_TIMESTAMPTYPE triggerTimeStamp;
+            memcpy(&triggerTimeStamp, &m_sConfigInputTrigTS, sizeof(OMX_TIME_CONFIG_TIMESTAMPTYPE));
+            valueSet |= vExt.readParamInt64(ext, "timestamp", (OMX_S64 *)&(triggerTimeStamp.nTimestamp));
+            if (!valueSet) {
+                break;
+            }
+
+            DEBUG_PRINT_HIGH("VENDOR-EXT: set_config: trigger timestamp =%lld", triggerTimeStamp.nTimestamp);
+
+            err = set_config(
+                    NULL, (OMX_INDEXTYPE)OMX_IndexConfigTimePosition, &triggerTimeStamp);
+            if (err != OMX_ErrorNone) {
+                DEBUG_PRINT_ERROR("set_config: OMX_IndexConfigTimePosition failed !");
+            }
+
+            break;
+        }
         default:
         {
             return OMX_ErrorNotImplemented;