[mesh-forwarder] remove message if no pending tx in `SendMessage()` (#9495)

This commit updates `MeshForwarder::SendMessage()` on FTD to check if
the message is marked for direct transmission and/or indirect
transmission to a sleepy child. If there is no pending transmission,
the message is removed.

This situation can occur if the message destination is a multicast
address larger than realm-local scope. In such a case, `SendMessage()`
skips `SetDirectTransmission()` on the message(since such message
will be forwarded using IP-in-IP encapsulation by `Ip6` module) and
assumes the message is for a sleepy child. However, if none of the
children are subscribed to this address, the message will not be
marked for indirect transmission either. Without the fix in this
commit, such messages would have remained in the `mSendQueue` and not
been removed or freed, as messages are only checked for removal after
a direct or indirect transmission attempt.

3 files changed, 15 insertions, 3 deletions

diff --git a/src/core/thread/mesh_forwarder.cpp b/src/core/thread/mesh_forwarder.cpp
index d4e61cf08..413321c28 100644
--- a/src/core/thread/mesh_forwarder.cpp
+++ b/src/core/thread/mesh_forwarder.cpp
@@ -1320,8 +1320,10 @@ exit:
     mScheduleTransmissionTask.Post();
 }
 
-void MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
+bool MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
 {
+    bool didRemove = false;
+
 #if OPENTHREAD_FTD
     VerifyOrExit(!aMessage.IsDirectTransmission() && !aMessage.IsChildPending());
 #else
@@ -1335,9 +1337,10 @@ void MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
     }
 
     mSendQueue.DequeueAndFree(aMessage);
+    didRemove = true;
 
 exit:
-    return;
+    return didRemove;
 }
 
 void MeshForwarder::HandleReceivedFrame(Mac::RxFrame &aFrame)
diff --git a/src/core/thread/mesh_forwarder.hpp b/src/core/thread/mesh_forwarder.hpp
index ffa8af2c4..dfdc749fd 100644
--- a/src/core/thread/mesh_forwarder.hpp
+++ b/src/core/thread/mesh_forwarder.hpp
@@ -567,7 +567,7 @@ private:
     void UpdateNeighborLinkFailures(Neighbor &aNeighbor, Error aError, bool aAllowNeighborRemove, uint8_t aFailLimit);
     void HandleSentFrame(Mac::TxFrame &aFrame, Error aError);
     void UpdateSendMessage(Error aFrameTxError, Mac::Address &aMacDest, Neighbor *aNeighbor);
-    void RemoveMessageIfNoPendingTx(Message &aMessage);
+    bool RemoveMessageIfNoPendingTx(Message &aMessage);
 
     void HandleTimeTick(void);
     void ScheduleTransmissionTask(void);
diff --git a/src/core/thread/mesh_forwarder_ftd.cpp b/src/core/thread/mesh_forwarder_ftd.cpp
index 5e84f4c0e..16eea50bc 100644
--- a/src/core/thread/mesh_forwarder_ftd.cpp
+++ b/src/core/thread/mesh_forwarder_ftd.cpp
@@ -135,12 +135,21 @@ Error MeshForwarder::SendMessage(Message &aMessage)
         break;
     }
 
+    // Ensure that the message is marked for direct tx and/or for indirect tx
+    // to a sleepy child. Otherwise, remove the message.
+
+    if (RemoveMessageIfNoPendingTx(aMessage))
+    {
+        ExitNow();
+    }
+
 #if (OPENTHREAD_CONFIG_MAX_FRAMES_IN_DIRECT_TX_QUEUE > 0)
     ApplyDirectTxQueueLimit(aMessage);
 #endif
 
     mScheduleTransmissionTask.Post();
 
+exit:
     return error;
 }