Postpone unmap of msg buffer until necessary am: 6031ca5b25

Original change: https://android-review.googlesource.com/c/trusty/lk/trusty/+/2553743

Change-Id: Iaaf5ddfff5a93ced0769114f1f60ae277acdc80c
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

3 files changed, 267 insertions, 12 deletions

diff --git a/lib/trusty/tipc_virtio_dev.c b/lib/trusty/tipc_virtio_dev.c
index d883f6f..fe29117 100644
--- a/lib/trusty/tipc_virtio_dev.c
+++ b/lib/trusty/tipc_virtio_dev.c
@@ -90,6 +90,8 @@ struct tipc_dev {
     struct vqueue vqs[TIPC_VQ_NUM];
     struct tipc_ept epts[TIPC_ADDR_MAX_NUM];
     unsigned long inuse[BITMAP_NUM_WORDS(TIPC_ADDR_MAX_NUM)];
+    struct vqueue_mapped_list send_mapped;
+    struct vqueue_mapped_list receive_mapped;
 
     event_t have_handles;
     struct handle_list handle_list;
@@ -101,6 +103,7 @@ struct tipc_dev {
 
     bool tx_stop;
     bool rx_stop;
+    bool reuse_mapping;
 };
 
 struct tipc_shm {
@@ -126,6 +129,10 @@ enum tipc_ctrl_msg_types {
     TIPC_CTRL_MSGTYPE_CONN_RSP,
     TIPC_CTRL_MSGTYPE_DISC_REQ,
     TIPC_CTRL_MSGTYPE_RELEASE,
+    TIPC_CTRL_MSGTYPE_REUSE_MSGBUF_REQ,
+    TIPC_CTRL_MSGTYPE_REUSE_MSGBUF_RSP,
+    TIPC_CTRL_MSGTYPE_UNMAP_REQ,
+    TIPC_CTRL_MSGTYPE_UNMAP_RSP,
 };
 
 /*
@@ -164,6 +171,15 @@ struct tipc_release_body {
     uint64_t id;
 } __PACKED;
 
+struct tipc_unmap_req_body {
+    uint64_t id;
+} __PACKED;
+
+struct tipc_unmap_rsp_body {
+    int32_t result;
+    uint64_t id;
+} __PACKED;
+
 typedef int (*tipc_data_cb_t)(uint8_t* dst, size_t sz, void* ctx);
 
 struct tipc_ext_mem {
@@ -474,6 +490,48 @@ static int handle_disc_req(struct tipc_dev* dev,
     return NO_ERROR;
 }
 
+static int handle_reuse_msgbuf_req(struct tipc_dev* dev, uint32_t remote) {
+    struct {
+        struct tipc_ctrl_msg_hdr hdr;
+    } msg;
+
+    /* on or off based on request */
+    dev->reuse_mapping = true;
+
+    /* send response */
+    msg.hdr.type = TIPC_CTRL_MSGTYPE_REUSE_MSGBUF_RSP;
+    msg.hdr.body_len = 0;
+
+    return tipc_send_buf(dev, TIPC_CTRL_ADDR, TIPC_CTRL_ADDR, &msg, sizeof(msg),
+                         true);
+}
+
+static int handle_unmap_req(struct tipc_dev* dev,
+                            uint32_t remote,
+                            const volatile struct tipc_unmap_req_body* ns_req) {
+    struct vqueue_mapped_list* mapped[2];
+    struct {
+        struct tipc_ctrl_msg_hdr hdr;
+        struct tipc_unmap_rsp_body body;
+    } msg;
+
+    mapped[0] = &dev->send_mapped;
+    mapped[1] = &dev->receive_mapped;
+
+    /* try to unmap */
+    msg.body.result = vqueue_unmap_memid(ns_req->id, mapped, 2);
+
+    /* copy id from request to response so that host can reclaim */
+    msg.body.id = ns_req->id;
+
+    /* send response */
+    msg.hdr.type = TIPC_CTRL_MSGTYPE_UNMAP_RSP;
+    msg.hdr.body_len = sizeof(msg.body);
+
+    return tipc_send_buf(dev, TIPC_CTRL_ADDR, TIPC_CTRL_ADDR, &msg, sizeof(msg),
+                         true);
+}
+
 static int handle_ctrl_msg(struct tipc_dev* dev,
                            uint32_t remote,
                            const volatile void* ns_data,
@@ -510,6 +568,16 @@ static int handle_ctrl_msg(struct tipc_dev* dev,
             break;
         return handle_disc_req(dev, remote, ns_msg_body);
 
+    case TIPC_CTRL_MSGTYPE_REUSE_MSGBUF_REQ:
+        if (msg_body_len != 0)
+            break;
+        return handle_reuse_msgbuf_req(dev, remote);
+
+    case TIPC_CTRL_MSGTYPE_UNMAP_REQ:
+        if (msg_body_len != sizeof(struct tipc_unmap_req_body))
+            break;
+        return handle_unmap_req(dev, remote, ns_msg_body);
+
     default:
         break;
     }
@@ -684,7 +752,8 @@ static int handle_rx_msg(struct tipc_dev* dev, struct vqueue_buf* buf) {
 
     /* map in_iovs, Non-secure, no-execute, cached, read-only */
     uint map_flags = ARCH_MMU_FLAG_PERM_NO_EXECUTE | ARCH_MMU_FLAG_PERM_RO;
-    int ret = vqueue_map_iovs(dev->vd.client_id, &buf->in_iovs, map_flags);
+    int ret = vqueue_map_iovs(dev->vd.client_id, &buf->in_iovs, map_flags,
+                              &dev->receive_mapped);
     if (ret) {
         TRACEF("failed to map iovs %d\n", ret);
         return ret;
@@ -726,7 +795,9 @@ static int handle_rx_msg(struct tipc_dev* dev, struct vqueue_buf* buf) {
     }
 
 done:
-    vqueue_unmap_iovs(&buf->in_iovs);
+    if (!dev->reuse_mapping) {
+        vqueue_unmap_iovs(&buf->in_iovs, &dev->receive_mapped);
+    }
 
     return ret;
 }
@@ -1262,7 +1333,8 @@ static int tipc_send_data(struct tipc_dev* dev,
 
     /* map in provided buffers (no-execute, read-write) */
     uint map_flags = ARCH_MMU_FLAG_PERM_NO_EXECUTE;
-    ret = vqueue_map_iovs(dev->vd.client_id, &buf.out_iovs, map_flags);
+    ret = vqueue_map_iovs(dev->vd.client_id, &buf.out_iovs, map_flags,
+                          &dev->send_mapped);
     if (ret == NO_ERROR) {
         struct tipc_hdr* hdr = buf.out_iovs.iovs[0].iov_base;
 
@@ -1287,7 +1359,8 @@ static int tipc_send_data(struct tipc_dev* dev,
             ret += sizeof(struct tipc_hdr);
         }
 
-        vqueue_unmap_iovs(&buf.out_iovs);
+        if (!dev->reuse_mapping)
+            vqueue_unmap_iovs(&buf.out_iovs, &dev->send_mapped);
     }
 
 done:
@@ -1357,7 +1430,16 @@ status_t create_tipc_device(struct trusty_virtio_bus* vb,
     handle_list_init(&dev->handle_list);
     event_init(&dev->have_handles, false, EVENT_FLAG_AUTOUNSIGNAL);
 
+    bst_root_initialize(&dev->send_mapped.list);
+    mutex_init(&dev->send_mapped.lock);
+    dev->send_mapped.in_direction = false;
+    bst_root_initialize(&dev->receive_mapped.list);
+    mutex_init(&dev->receive_mapped.lock);
+    dev->receive_mapped.in_direction = true;
+    dev->reuse_mapping = false;
+
     ret = virtio_register_device(vb, &dev->vd);
+
     if (ret != NO_ERROR)
         goto err_register;
 
diff --git a/lib/trusty/vqueue.c b/lib/trusty/vqueue.c
index 8f4cd59..437fd14 100644
--- a/lib/trusty/vqueue.c
+++ b/lib/trusty/vqueue.c
@@ -211,11 +211,64 @@ int vqueue_get_avail_buf(struct vqueue* vq, struct vqueue_buf* iovbuf) {
     return ret;
 }
 
+struct vqueue_mem_obj {
+    ext_mem_client_id_t client_id;
+    ext_mem_obj_id_t id;
+    void* iov_base;
+    size_t size;
+    struct bst_node node;
+};
+
+static struct vqueue_mem_obj* vqueue_mem_obj_from_bst_node(
+        struct bst_node* node) {
+    return containerof(node, struct vqueue_mem_obj, node);
+}
+
+static int vqueue_mem_obj_cmp(struct bst_node* a_bst, struct bst_node* b_bst) {
+    struct vqueue_mem_obj* a = vqueue_mem_obj_from_bst_node(a_bst);
+    struct vqueue_mem_obj* b = vqueue_mem_obj_from_bst_node(b_bst);
+
+    return a->id < b->id ? 1 : a->id > b->id ? -1 : 0;
+}
+
+static void vqueue_mem_obj_initialize(struct vqueue_mem_obj* obj,
+                                      ext_mem_client_id_t client_id,
+                                      ext_mem_obj_id_t id,
+                                      void* iov_base,
+                                      size_t size) {
+    obj->client_id = client_id;
+    obj->id = id;
+    obj->iov_base = iov_base;
+    obj->size = size;
+    bst_node_initialize(&obj->node);
+}
+
+static bool vqueue_mem_insert(struct bst_root* objs,
+                              struct vqueue_mem_obj* obj) {
+    return bst_insert(objs, &obj->node, vqueue_mem_obj_cmp);
+}
+
+static struct vqueue_mem_obj* vqueue_mem_lookup(struct bst_root* objs,
+                                                ext_mem_obj_id_t id) {
+    struct vqueue_mem_obj ref_obj;
+    ref_obj.id = id;
+    return bst_search_type(objs, &ref_obj, vqueue_mem_obj_cmp,
+                           struct vqueue_mem_obj, node);
+}
+
+static inline void vqueue_mem_delete(struct bst_root* objs,
+                                     struct vqueue_mem_obj* obj) {
+    bst_delete(objs, &obj->node);
+}
+
 int vqueue_map_iovs(ext_mem_client_id_t client_id,
                     struct vqueue_iovs* vqiovs,
-                    u_int flags) {
+                    u_int flags,
+                    struct vqueue_mapped_list* mapped_list) {
     uint i;
     int ret;
+    size_t size;
+    struct vqueue_mem_obj* obj;
 
     DEBUG_ASSERT(vqiovs);
     DEBUG_ASSERT(vqiovs->shared_mem_id);
@@ -223,14 +276,61 @@ int vqueue_map_iovs(ext_mem_client_id_t client_id,
     DEBUG_ASSERT(vqiovs->used <= vqiovs->cnt);
 
     for (i = 0; i < vqiovs->used; i++) {
+        /* see if it's already been mapped */
+        mutex_acquire(&mapped_list->lock);
+        obj = vqueue_mem_lookup(&mapped_list->list, vqiovs->shared_mem_id[i]);
+        mutex_release(&mapped_list->lock);
+
+        if (obj && obj->client_id == client_id &&
+            vqiovs->iovs[i].iov_len <= obj->size) {
+            LTRACEF("iov restored %s id= %lu (base= %p, size= %lu)\n",
+                    mapped_list->in_direction ? "IN" : "OUT",
+                    (unsigned long)vqiovs->shared_mem_id[i], obj->iov_base,
+                    (unsigned long)obj->size);
+            vqiovs->iovs[i].iov_base = obj->iov_base;
+            continue; /* use the previously mapped */
+        } else if (obj) {
+            /* otherwise, we need to drop old mapping and remap  */
+            TRACEF("iov needs remapped for id= %lu\n",
+                   (unsigned long)vqiovs->shared_mem_id[i]);
+            mutex_acquire(&mapped_list->lock);
+            vqueue_mem_delete(&mapped_list->list, obj);
+            mutex_release(&mapped_list->lock);
+            free(obj);
+        }
+
+        /* allocate since it may be reused instead of unmapped after use */
+        obj = calloc(1, sizeof(struct vqueue_mem_obj));
+        if (unlikely(!obj)) {
+            TRACEF("calloc failure for vqueue_mem_obj for iov\n");
+            ret = ERR_NO_MEMORY;
+            goto err;
+        }
+
+        /* map it */
         vqiovs->iovs[i].iov_base = NULL;
+        size = round_up(vqiovs->iovs[i].iov_len, PAGE_SIZE);
         ret = ext_mem_map_obj_id(vmm_get_kernel_aspace(), "vqueue-buf",
                                  client_id, vqiovs->shared_mem_id[i], 0, 0,
-                                 round_up(vqiovs->iovs[i].iov_len, PAGE_SIZE),
-                                 &vqiovs->iovs[i].iov_base, PAGE_SIZE_SHIFT, 0,
-                                 flags);
-        if (ret)
+                                 size, &vqiovs->iovs[i].iov_base,
+                                 PAGE_SIZE_SHIFT, 0, flags);
+        if (ret) {
+            free(obj);
             goto err;
+        }
+
+        vqueue_mem_obj_initialize(obj, client_id, vqiovs->shared_mem_id[i],
+                                  vqiovs->iovs[i].iov_base, size);
+
+        mutex_acquire(&mapped_list->lock);
+        if (unlikely(!vqueue_mem_insert(&mapped_list->list, obj)))
+            panic("Unhandled duplicate entry in ext_mem for iov\n");
+        mutex_release(&mapped_list->lock);
+
+        LTRACEF("iov saved %s id= %lu (base= %p, size= %lu)\n",
+                mapped_list->in_direction ? "IN" : "OUT",
+                (unsigned long)vqiovs->shared_mem_id[i],
+                vqiovs->iovs[i].iov_base, (unsigned long)size);
     }
 
     return NO_ERROR;
@@ -245,7 +345,10 @@ err:
     return ret;
 }
 
-void vqueue_unmap_iovs(struct vqueue_iovs* vqiovs) {
+void vqueue_unmap_iovs(struct vqueue_iovs* vqiovs,
+                       struct vqueue_mapped_list* mapped_list) {
+    struct vqueue_mem_obj* obj;
+
     DEBUG_ASSERT(vqiovs);
     DEBUG_ASSERT(vqiovs->shared_mem_id);
     DEBUG_ASSERT(vqiovs->iovs);
@@ -257,7 +360,64 @@ void vqueue_unmap_iovs(struct vqueue_iovs* vqiovs) {
         vmm_free_region(vmm_get_kernel_aspace(),
                         (vaddr_t)vqiovs->iovs[i].iov_base);
         vqiovs->iovs[i].iov_base = NULL;
+
+        /* remove from list since it has been unmapped */
+        mutex_acquire(&mapped_list->lock);
+        obj = vqueue_mem_lookup(&mapped_list->list, vqiovs->shared_mem_id[i]);
+        if (obj) {
+            LTRACEF("iov removed %s id= %lu (base= %p, size= %lu)\n",
+                    mapped_list->in_direction ? "IN" : "OUT",
+                    (unsigned long)vqiovs->shared_mem_id[i],
+                    vqiovs->iovs[i].iov_base,
+                    (unsigned long)vqiovs->iovs[i].iov_len);
+            vqueue_mem_delete(&mapped_list->list, obj);
+            free(obj);
+        } else {
+            TRACEF("iov mapping not found for id= %lu (base= %p, size= %lu)\n",
+                   (unsigned long)vqiovs->shared_mem_id[i],
+                   vqiovs->iovs[i].iov_base,
+                   (unsigned long)vqiovs->iovs[i].iov_len);
+        }
+        mutex_release(&mapped_list->lock);
+    }
+}
+
+int vqueue_unmap_memid(ext_mem_obj_id_t id,
+                       struct vqueue_mapped_list* mapped_list[],
+                       int list_cnt) {
+    struct vqueue_mapped_list* mapped;
+    struct vqueue_mem_obj* obj;
+    struct vqueue_iovs fake_vqiovs;
+    ext_mem_obj_id_t fake_shared_mem_id[1];
+    struct iovec_kern fake_iovs[1];
+
+    /* determine which list this entry is in */
+    for (int i = 0; i < list_cnt; i++) {
+        mapped = mapped_list[i];
+        obj = vqueue_mem_lookup(&mapped->list, id);
+        if (obj)
+            break;
+        mapped = NULL;
+    }
+
+    if (mapped) {
+        /* fake a vqueue_iovs struct to use common interface */
+        memset(&fake_vqiovs, 0, sizeof(fake_vqiovs));
+        fake_vqiovs.iovs = fake_iovs;
+        fake_vqiovs.shared_mem_id = fake_shared_mem_id;
+        fake_vqiovs.used = 1;
+        fake_vqiovs.cnt = 1;
+        fake_vqiovs.iovs[0].iov_base = obj->iov_base;
+        fake_vqiovs.iovs[0].iov_len = obj->size;
+        fake_vqiovs.shared_mem_id[0] = id;
+
+        /* unmap */
+        vqueue_unmap_iovs(&fake_vqiovs, mapped);
+
+        return NO_ERROR;
     }
+
+    return ERR_NOT_FOUND;
 }
 
 static int _vqueue_add_buf_locked(struct vqueue* vq,
diff --git a/lib/trusty/vqueue.h b/lib/trusty/vqueue.h
index 6c75e67..05c095f 100644
--- a/lib/trusty/vqueue.h
+++ b/lib/trusty/vqueue.h
@@ -26,6 +26,7 @@
 
 #include <arch/ops.h>
 #include <kernel/event.h>
+#include <kernel/mutex.h>
 #include <lib/extmem/extmem.h>
 #include <lib/trusty/uio.h>
 #include <stdint.h>
@@ -74,6 +75,12 @@ struct vqueue_buf {
     struct vqueue_iovs out_iovs;
 };
 
+struct vqueue_mapped_list {
+    struct bst_root list;
+    mutex_t lock;
+    bool in_direction;
+};
+
 int vqueue_init(struct vqueue* vq,
                 uint32_t id,
                 ext_mem_client_id_t client_id,
@@ -90,8 +97,14 @@ int vqueue_get_avail_buf(struct vqueue* vq, struct vqueue_buf* iovbuf);
 
 int vqueue_map_iovs(ext_mem_client_id_t client_id,
                     struct vqueue_iovs* vqiovs,
-                    u_int flags);
-void vqueue_unmap_iovs(struct vqueue_iovs* vqiovs);
+                    u_int flags,
+                    struct vqueue_mapped_list* mapped_list);
+void vqueue_unmap_iovs(struct vqueue_iovs* vqiovs,
+                       struct vqueue_mapped_list* mapped_list);
+
+int vqueue_unmap_memid(ext_mem_obj_id_t id,
+                       struct vqueue_mapped_list* mapped_list[],
+                       int list_cnt);
 
 int vqueue_add_buf(struct vqueue* vq, struct vqueue_buf* buf, uint32_t len);