RIO-6283: introduce authoring clock for a better AV sync.

4 files changed, 227 insertions, 14 deletions

diff --git a/android/author/android_audio_input.cpp b/android/author/android_audio_input.cpp
index 9b83c6172..a36cd0202 100644
--- a/android/author/android_audio_input.cpp
+++ b/android/author/android_audio_input.cpp
@@ -59,7 +59,11 @@ AndroidAudioInput::AndroidAudioInput(uint32 audioSource)
     iState(STATE_IDLE),
     iMaxAmplitude(0),
     iTrackMaxAmplitude(false),
-    iAudioThreadStarted(false)
+    iAudioThreadStarted(false),
+    iAuthorClock(NULL),
+    iClockNotificationsInf(NULL),
+    iFirstFrameReceived(false),
+    iFirstFrameTs(0)
 {
     LOGV("AndroidAudioInput constructor %p", this);
     // semaphore used to communicate between this  mio and the audio output thread
@@ -748,13 +752,32 @@ void AndroidAudioInput::Run()
         }
     }
 
+    if ((iState == STATE_STARTED) && (iStartCmd.iType == AI_CMD_START)) {
+        // This means Audio MIO is waiting for
+        // first audio frame to be received
+        if (iFirstFrameReceived) {
+            // Set the clock with iFirstFrameTs
+            if (iAuthorClock) {
+                 bool tmpbool = false;
+                 iAuthorClock->SetStartTime32((uint32&)iFirstFrameTs, PVMF_MEDIA_CLOCK_MSEC, tmpbool);
+            }
+
+            LOGV("First frame received, Complete the start");
+            // First frame is received now complete Start
+            DoRequestCompleted(iStartCmd, PVMFSuccess);
+
+            // Set the iStartCmd type to be invalid now
+            iStartCmd.iType = AI_INVALID_CMD;
+         }
+    }
+
     if(!iCmdQueue.empty())
     {
         // Run again if there are more things to process
         RunIfNotReady();
     }
 
-    if(iState == STATE_STARTED)
+    if((iState == STATE_STARTED) && (iFirstFrameReceived))
     {
         SendMicData();
     }
@@ -791,6 +814,12 @@ void AndroidAudioInput::AddDataEventToQueue(uint32 aMicroSecondsToEvent)
 void AndroidAudioInput::DoRequestCompleted(const AndroidAudioInputCmd& aCmd, PVMFStatus aStatus, OsclAny* aEventData)
 {
     LOGV("DoRequestCompleted(%d, %d) this %p", aCmd.iId, aStatus, this);
+    if ((aStatus == PVMFPending) && (aCmd.iType == AI_CMD_START)) {
+        LOGV("Start is pending, Return here, wait for success or failure");
+        // Copy the command
+        iStartCmd = aCmd;
+        return;
+    }
     PVMFCmdResp response(aCmd.iId, aCmd.iContext, aStatus, aEventData);
 
     for(uint32 i = 0; i < iObservers.size(); i++)
@@ -833,6 +862,17 @@ PVMFStatus AndroidAudioInput::DoStart()
 {
     LOGV("DoStart");
 
+    // Set the clock state observer
+    if (iAuthorClock) {
+        iAuthorClock->ConstructMediaClockNotificationsInterface(iClockNotificationsInf, *this);
+
+        if (iClockNotificationsInf == NULL) {
+             return PVMFErrNoMemory;
+        }
+
+        iClockNotificationsInf->SetClockStateObserver(*this);
+    }
+
     iAudioThreadStartLock->lock();
     iAudioThreadStarted = false;
 
@@ -861,6 +901,18 @@ PVMFStatus AndroidAudioInput::DoStart()
     iState = STATE_STARTED;
 
     AddDataEventToQueue(0);
+
+    // Hold back onto Start until the first audio frame is received by
+    // the audio thread.
+    // We want to hold back the start until first audio frame since
+    // recording time origin will start with the first audio sample.
+    // First audio sample will serve as the reference for audio and video
+    // frames timestamps in msecs.
+    if (!iFirstFrameReceived) {
+        LOGV("First Frame not received, hold back the start");
+        return PVMFPending;
+    }
+
     return PVMFSuccess;
 }
 
@@ -885,8 +937,12 @@ PVMFStatus AndroidAudioInput::DoPause()
 PVMFStatus AndroidAudioInput::DoReset()
 {
     LOGV("DoReset");
+    // Remove and destroy the clock state observer
+    RemoveDestroyClockStateObs();
     iExitAudioThread = true;
     iDataEventCounter = 0;
+    iFirstFrameReceived = false;
+    iFirstFrameTs = 0;
     if(iAudioThreadStarted ){
     iAudioThreadSem->Signal();
     iAudioThreadTermSem->Wait();
@@ -929,8 +985,14 @@ PVMFStatus AndroidAudioInput::DoFlush()
 PVMFStatus AndroidAudioInput::DoStop()
 {
     LOGV("DoStop");
+
+    // Remove and destroy the clock state observer
+    RemoveDestroyClockStateObs();
+
     iExitAudioThread = true;
     iDataEventCounter = 0;
+    iFirstFrameReceived = false;
+    iFirstFrameTs = 0;
     iState = STATE_STOPPED;
     if(iAudioThreadStarted ){
     iAudioThreadSem->Signal();
@@ -1105,6 +1167,20 @@ int AndroidAudioInput::audin_thread_func() {
             if (numOfBytes <= 0)
                 break;
 
+            if (iFirstFrameReceived == false) {
+                iFirstFrameReceived = true;
+
+                // Get the AudioRecord latency and
+                // get the system clock at this point
+                // The difference in 2 will give the actual time
+                // of first audio capture.
+                uint32 systime = (uint32) (systemTime() / 1000000L);
+                // TODO: add audio hardware input latency here
+                uint32 recordLatency = record->latency();
+                iFirstFrameTs = systime - recordLatency;
+                LOGV("First Audio Frame received systime %d, recordLatency %d, iFirstFrameTs %d", systime, recordLatency, iFirstFrameTs);
+            }
+
             if (numFramesRecorded < kAutoRampStartFrames) {
                 // Start with silence...
                 memset(data, 0, numOfBytes);
@@ -1178,6 +1254,7 @@ void AndroidAudioInput::SendMicData(void)
         return;
     }
     MicData &data = iWriteResponseQueue[0];
+
     // send data to Peer & store the id
     PvmiMediaXferHeader data_hdr;
     data_hdr.seq_num=iDataEventCounter-1;
@@ -1266,6 +1343,12 @@ PVMFStatus AndroidAudioInput::VerifyAndSetParameter(PvmiKvp* aKvp, bool aSetPara
             return PVMFFailure;
         }
     }
+    else if (pv_mime_strcmp(aKvp->key, PVMF_AUTHORING_CLOCK_KEY) == 0)
+    {
+        LOGV("AndroidAudioInput::VerifyAndSetParameter() PVMF_AUTHORING_CLOCK_KEY value %p", aKvp->value.key_specific_value);
+        iAuthorClock = (PVMFMediaClock*) aKvp->value.key_specific_value;
+        return PVMFSuccess;
+    }
 
     LOGE("unsupported parameter: %s", aKvp->key);
     return PVMFFailure;
@@ -1282,3 +1365,30 @@ int AndroidAudioInput::maxAmplitude()
     iMaxAmplitude = 0;
     return result;
 }
+
+void AndroidAudioInput::NotificationsInterfaceDestroyed()
+{
+    iClockNotificationsInf = NULL;
+}
+
+void AndroidAudioInput::ClockStateUpdated()
+{
+    PVMFMediaClock::PVMFMediaClockState iClockState = iAuthorClock->GetState();
+    uint32 currentTime = 0;
+    bool tmpbool = false;
+    iAuthorClock->GetCurrentTime32(currentTime, tmpbool, PVMF_MEDIA_CLOCK_MSEC);
+    LOGV("ClockStateUpdated State %d Current clock value %d", iClockState, currentTime);
+}
+
+void AndroidAudioInput::RemoveDestroyClockStateObs()
+{
+    if (iAuthorClock != NULL) {
+        if (iClockNotificationsInf != NULL) {
+            iClockNotificationsInf->RemoveClockStateObserver(*this);
+            iAuthorClock->DestroyMediaClockNotificationsInterface(iClockNotificationsInf);
+            iClockNotificationsInf = NULL;
+        }
+    }
+}
+
+
diff --git a/android/author/android_audio_input.h b/android/author/android_audio_input.h
index ad1af24e0..a410f2ab7 100644
--- a/android/author/android_audio_input.h
+++ b/android/author/android_audio_input.h
@@ -53,6 +53,9 @@
 #ifndef ANDROID_AUDIO_INPUT_THREADSAFE_CALLBACK_AO_H_INCLUDED
 #include "android_audio_input_threadsafe_callbacks.h"
 #endif
+#ifndef PVMF_MEDIA_CLOCK_H_INCLUDED
+#include "pvmf_media_clock.h"
+#endif
 
 #include <utils/RefBase.h>
 
@@ -206,7 +209,8 @@ class AndroidAudioInput : public OsclTimerObject,
     public PvmiMIOControl,
     public PvmiMediaTransfer,
     public PvmiCapabilityAndConfig,
-    public RefBase
+    public RefBase,
+    public PVMFMediaClockStateObserver
 {
 public:
     AndroidAudioInput(uint32 audioSource);
@@ -294,6 +298,10 @@ public:
     /* Set the input number of channels */
     bool setAudioNumChannels(int32 iNumChannels);
 
+    /* From PVMFMediaClockStateObserver and its base*/
+    void ClockStateUpdated();
+    void NotificationsInterfaceDestroyed();
+
 private:
     AndroidAudioInput();
     void Run();
@@ -340,6 +348,8 @@ private:
     void RampVolume(int32 timeInFrames, int32 kAutoRampDurationFrames,
                     void *_data, size_t numBytes) const;
 
+    void RemoveDestroyClockStateObs();
+
     // Command queue
     uint32 iCmdIdCounter;
     Oscl_Vector<AndroidAudioInputCmd, OsclMemAllocator> iCmdQueue;
@@ -444,6 +454,19 @@ private:
     Condition *iAudioThreadStartCV;
     volatile status_t iAudioThreadStartResult;
     volatile bool iAudioThreadStarted;
+
+    PVMFMediaClock *iAuthorClock;
+    PVMFMediaClockNotificationsInterface *iClockNotificationsInf;
+
+    // These variables tracks whether or not first audio frame was received.
+    // This is needed to start the clock since the time origin is synced to the
+    // first audio sample.
+    volatile bool iFirstFrameReceived;
+    volatile PVMFTimestamp iFirstFrameTs;
+
+    // This stores the Start cmd when Audio MIO is waiting for
+    // first audio frame to be received from the device.
+    AndroidAudioInputCmd iStartCmd;
 };
 
 }; // namespace android
diff --git a/android/author/android_camera_input.cpp b/android/author/android_camera_input.cpp
index 3273ca6ad..5a54dd0b0 100644
--- a/android/author/android_camera_input.cpp
+++ b/android/author/android_camera_input.cpp
@@ -40,14 +40,16 @@ OSCL_DLL_ENTRY_POINT_DEFAULT()
 // camera MIO
 AndroidCameraInput::AndroidCameraInput()
     : OsclTimerObject(OsclActiveObject::EPriorityNominal, "AndroidCameraInput"),
-    iWriteState(EWriteOK)
+    iWriteState(EWriteOK),
+    iAuthorClock(NULL),
+    iClockNotificationsInf(NULL),
+    iAudioFirstFrameTs(0)
 {
     LOGV("constructor(%p)", this);
     iCmdIdCounter = 0;
     iPeer = NULL;
     iThreadLoggedOn = false;
     iDataEventCounter = 0;
-    iStartTickCount = 0;
     iTimeStamp = 0;
     iMilliSecondsPerDataEvent = 0;
     iMicroSecondsPerDataEvent = 0;
@@ -903,6 +905,18 @@ PVMFStatus AndroidCameraInput::DoInit()
 PVMFStatus AndroidCameraInput::DoStart()
 {
     LOGV("DoStart");
+
+    // Set the clock state observer
+    if (iAuthorClock) {
+        iAuthorClock->ConstructMediaClockNotificationsInterface(iClockNotificationsInf, *this);
+
+        if (iClockNotificationsInf == NULL) {
+             return PVMFErrNoMemory;
+        }
+
+        iClockNotificationsInf->SetClockStateObserver(*this);
+    }
+
     PVMFStatus status = PVMFFailure;
     iWriteState = EWriteOK;
     if (mCamera == NULL) {
@@ -918,7 +932,6 @@ PVMFStatus AndroidCameraInput::DoStart()
             status = PVMFSuccess;
         }
     }
-    iStartTickCount = (uint32) (systemTime() / 1000000L);
     AddDataEventToQueue(iMilliSecondsPerDataEvent);
     return status;
 }
@@ -934,7 +947,10 @@ PVMFStatus AndroidCameraInput::DoPause()
 PVMFStatus AndroidCameraInput::DoReset()
 {
     LOGV("DoReset");
+    // Remove and destroy the clock state observer
+    RemoveDestroyClockObs();
     iDataEventCounter = 0;
+    iAudioFirstFrameTs = 0;
     iWriteState = EWriteOK;
     if ( (iState == STATE_STARTED) || (iState == STATE_PAUSED) ) {
     if (mCamera != NULL) {
@@ -967,7 +983,12 @@ PVMFStatus AndroidCameraInput::DoFlush(const AndroidCameraInputCmd& aCmd)
 PVMFStatus AndroidCameraInput::DoStop(const AndroidCameraInputCmd& aCmd)
 {
     LOGV("DoStop");
+
+    // Remove and destroy the clock state observer
+    RemoveDestroyClockObs();
+
     iDataEventCounter = 0;
+    iAudioFirstFrameTs = 0;
     iWriteState = EWriteOK;
     if (mCamera != NULL) {
     mCamera->setListener(NULL);
@@ -1040,6 +1061,12 @@ PVMFStatus AndroidCameraInput::VerifyAndSetParameter(PvmiKvp* aKvp,
             return PVMFFailure;
         }
     }
+    else if (pv_mime_strcmp(aKvp->key, PVMF_AUTHORING_CLOCK_KEY) == 0)
+    {
+        LOGV("AndroidCameraInput::VerifyAndSetParameter() PVMF_AUTHORING_CLOCK_KEY value %p", aKvp->value.key_specific_value);
+	iAuthorClock = (PVMFMediaClock*)aKvp->value.key_specific_value;
+        return PVMFSuccess;
+    }
 
     LOGE("Unsupported parameter(%s)", aKvp->key);
     return PVMFFailure;
@@ -1089,19 +1116,29 @@ PVMFStatus AndroidCameraInput::postWriteAsync(nsecs_t timestamp, const sp<IMemor
         return PVMFFailure;
     }
 
-    if((!iPeer) || (!isRecorderStarting()) || (iWriteState == EWriteBusy)) {
-        LOGV("Recording is not ready (iPeer %p iState %d iWriteState %d), frame dropped", iPeer, iState, iWriteState);
+    if((!iPeer) || (!isRecorderStarting()) || (iWriteState == EWriteBusy) || (iAuthorClock->GetState() != PVMFMediaClock::RUNNING)) {
+        LOGE("Recording is not ready (iPeer %p iState %d iWriteState %d iClockState %d), frame dropped", iPeer, iState, iWriteState, iAuthorClock->GetState());
         mCamera->releaseRecordingFrame(frame);
         return PVMFSuccess;
     }
 
-    // calculate timestamp as offset from start time
-    uint32 t = (uint32)(timestamp / 1000000L) - iStartTickCount;
+    // Now compare the video timestamp with the AudioFirstTimestamp
+    // if video timestamp is earlier to audio drop it
+    // or else send it downstream with correct timestamp
+    uint32 ts = (uint32)(timestamp / 1000000L);
+    if (ts < iAudioFirstFrameTs) {
+        // Drop the frame
+        mCamera->releaseRecordingFrame(frame);
+        return PVMFSuccess;
+    } else {
+         // calculate timestamp as offset from start time
+         ts -= iAudioFirstFrameTs;
+    }
 
     // Make sure that no two samples have the same timestamp
     if (iDataEventCounter != 0) {
-        if (iTimeStamp != t) {
-            iTimeStamp = t;
+        if (iTimeStamp != ts) {
+            iTimeStamp = ts;
         } else {
             ++iTimeStamp;
         }
@@ -1144,3 +1181,33 @@ void AndroidCameraInputListener::postDataTimestamp(nsecs_t timestamp, int32_t ms
     }
 }
 
+void AndroidCameraInput::NotificationsInterfaceDestroyed()
+{
+    iClockNotificationsInf = NULL;
+}
+
+void AndroidCameraInput::ClockStateUpdated()
+{
+    PVMFMediaClock::PVMFMediaClockState iClockState = iAuthorClock->GetState();
+    if ((iClockState == PVMFMediaClock::RUNNING) && (iAudioFirstFrameTs == 0)) {
+        // Get the clock time here
+        // this will be the time of first audio frame capture
+        bool tmpbool = false;
+        iAuthorClock->GetCurrentTime32(iAudioFirstFrameTs, tmpbool, PVMF_MEDIA_CLOCK_MSEC);
+        LOGV("Audio first ts %d", iAudioFirstFrameTs);
+    }
+}
+
+void AndroidCameraInput::RemoveDestroyClockObs()
+{
+    if (iAuthorClock != NULL) {
+        if (iClockNotificationsInf != NULL) {
+            iClockNotificationsInf->RemoveClockStateObserver(*this);
+            iAuthorClock->DestroyMediaClockNotificationsInterface(iClockNotificationsInf);
+            iClockNotificationsInf = NULL;
+        }
+    }
+}
+
+
+
diff --git a/android/author/android_camera_input.h b/android/author/android_camera_input.h
index c05f8ee2d..2a0719bcf 100644
--- a/android/author/android_camera_input.h
+++ b/android/author/android_camera_input.h
@@ -51,6 +51,9 @@
 #ifndef PVMF_SIMPLE_MEDIA_BUFFER_H_INCLUDED
 #include "pvmf_simple_media_buffer.h"
 #endif
+#ifndef PVMF_MEDIA_CLOCK_H_INCLUDED
+#include "pvmf_media_clock.h"
+#endif
 
 #ifdef HIDE_MIO_SYMBOLS
 #pragma GCC visibility push(hidden)
@@ -191,7 +194,8 @@ class AndroidCameraInput
     : public OsclTimerObject,
       public PvmiMIOControl,
       public PvmiMediaTransfer,
-      public PvmiCapabilityAndConfig
+      public PvmiCapabilityAndConfig,
+      public PVMFMediaClockStateObserver
 {
 public:
     AndroidCameraInput();
@@ -316,6 +320,10 @@ public:
 
     bool isRecorderStarting() { return iState==STATE_STARTED?true:false; }
 
+    /* From PVMFMediaClockStateObserver and its base */
+    void ClockStateUpdated();
+    void NotificationsInterfaceDestroyed();
+
 private:
     // release all queued recording frames that have not been
     // given the chance to be sent out.
@@ -360,6 +368,8 @@ private:
      */
     PVMFStatus VerifyAndSetParameter(PvmiKvp* aKvp, bool aSetParam=false);
 
+    void RemoveDestroyClockObs();
+
     // Command queue
     uint32 iCmdIdCounter;
     Oscl_Vector<AndroidCameraInputCmd, OsclMemAllocator> iCmdQueue;
@@ -373,7 +383,6 @@ private:
     bool iThreadLoggedOn;
 
     int32 iDataEventCounter;
-    int32 iStartTickCount;
 
     // Timing
     int32 iMilliSecondsPerDataEvent;
@@ -425,6 +434,10 @@ private:
     enum WriteState {EWriteBusy, EWriteOK};
     WriteState iWriteState;
 
+    PVMFMediaClock *iAuthorClock;
+    PVMFMediaClockNotificationsInterface *iClockNotificationsInf;
+
+    uint32 iAudioFirstFrameTs;
 };
 
 #ifdef HIDE_MIO_SYMBOLS