From de16475969ca7dfaecf7144ece32b929cf193861 Mon Sep 17 00:00:00 2001 From: John Scheible Date: Tue, 26 Oct 2021 13:45:19 -0700 Subject: gxp: First import from development branch Squash at commit 48fe5786f1779890538d14f709b26063698c3711 Signed-off-by: John Scheible Change-Id: I208434dddde40c08b9fc4d5da072ba10541992a2 --- gxp-mapping.c | 344 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 gxp-mapping.c (limited to 'gxp-mapping.c') diff --git a/gxp-mapping.c b/gxp-mapping.c new file mode 100644 index 0000000..95873c0 --- /dev/null +++ b/gxp-mapping.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Records the mapped device addresses. + * + * Copyright (C) 2021 Google LLC + */ + +#include +#include +#include + +#include "gxp-dma.h" +#include "gxp-internal.h" +#include "gxp-mapping.h" +#include "mm-backport.h" + +int gxp_mapping_init(struct gxp_dev *gxp) +{ + gxp->mappings = + devm_kzalloc(gxp->dev, sizeof(*gxp->mappings), GFP_KERNEL); + if (!gxp->mappings) + return -ENOMEM; + + gxp->mappings->rb = RB_ROOT; + mutex_init(&gxp->mappings->lock); + + return 0; +} + +struct gxp_mapping *gxp_mapping_create(struct gxp_dev *gxp, uint core_list, + u64 user_address, size_t size, u32 flags, + enum dma_data_direction dir) +{ + struct gxp_mapping *mapping = NULL; + uint num_pages = 0; + struct page **pages; + ulong offset; + int ret, i; + struct vm_area_struct *vma; + unsigned int foll_flags = FOLL_LONGTERM | FOLL_WRITE; + + /* + * The host pages might be read-only and could fail if we attempt to pin + * it with FOLL_WRITE. + * default to read/write if find_extend_vma returns NULL + */ + vma = find_extend_vma(current->mm, user_address & PAGE_MASK); + if (vma && !(vma->vm_flags & VM_WRITE)) { + foll_flags &= ~FOLL_WRITE; + if (dir != DMA_TO_DEVICE) { + dev_err(gxp->dev, + "Unable to map read-only pages as anything but DMA_TO_DEVICE\n"); + return ERR_PTR(-EINVAL); + } + } + + /* Pin the user pages */ + offset = user_address & (PAGE_SIZE - 1); + if (unlikely((size + offset) / PAGE_SIZE >= UINT_MAX - 1 || + size + offset < size)) + return ERR_PTR(-ENOMEM); + num_pages = (size + offset) / PAGE_SIZE; + if ((size + offset) % PAGE_SIZE) + num_pages++; + + pages = kcalloc(num_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + return ERR_PTR(-ENOMEM); + + /* Provide protection around `pin_user_pages_fast` since it fails if + * called by more than one thread simultaneously. + */ + mutex_lock(&gxp->mappings->lock); + ret = pin_user_pages_fast(user_address & PAGE_MASK, num_pages, + foll_flags, pages); + mutex_unlock(&gxp->mappings->lock); + if (ret < 0 || ret < num_pages) { + dev_dbg(gxp->dev, + "Get user pages failed: user_add=%pK, num_pages=%u, ret=%d\n", + (void *)user_address, num_pages, ret); + num_pages = ret < 0 ? 0 : ret; + ret = ret >= 0 ? -EFAULT : ret; + goto error_unpin_pages; + } + + /* Initialize mapping book-keeping */ + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + ret = -ENOMEM; + goto error_unpin_pages; + } + mapping->host_address = user_address; + mapping->core_list = core_list; + mapping->size = size; + mapping->map_count = 1; + mapping->gxp_dma_flags = flags; + mapping->dir = dir; + ret = sg_alloc_table_from_pages(&mapping->sgt, pages, num_pages, 0, + num_pages * PAGE_SIZE, GFP_KERNEL); + if (ret) { + dev_dbg(gxp->dev, "Failed to alloc sgt for mapping (ret=%d)\n", + ret); + goto error_free_sgt; + } + + /* map the user pages */ + ret = gxp_dma_map_sg(gxp, mapping->core_list, mapping->sgt.sgl, + mapping->sgt.nents, mapping->dir, + DMA_ATTR_SKIP_CPU_SYNC, mapping->gxp_dma_flags); + if (!ret) { + dev_dbg(gxp->dev, "Failed to map sgt (ret=%d)\n", ret); + ret = -EINVAL; + goto error_free_sgt; + } + mapping->sgt.nents = ret; + mapping->device_address = + sg_dma_address(mapping->sgt.sgl) + offset; + + kfree(pages); + return mapping; + +error_free_sgt: + sg_free_table(&mapping->sgt); + kfree(mapping); +error_unpin_pages: + for (i = 0; i < num_pages; i++) + unpin_user_page(pages[i]); + kfree(pages); + + return ERR_PTR(ret); +} + +void gxp_mapping_destroy(struct gxp_dev *gxp, struct gxp_mapping *mapping) +{ + struct sg_page_iter sg_iter; + struct page *page; + + /* + * Unmap the user pages + * + * Normally on unmap, the entire mapping is synced back to the CPU. + * Since mappings are made at a page granularity regardless of the + * underlying buffer's size, they can cover other data as well. If a + * user requires a mapping be synced before unmapping, they are + * responsible for calling `gxp_mapping_sync()` before hand. + */ + gxp_dma_unmap_sg(gxp, mapping->core_list, mapping->sgt.sgl, + mapping->sgt.orig_nents, mapping->dir, + DMA_ATTR_SKIP_CPU_SYNC); + + /* Unpin the user pages */ + for_each_sg_page(mapping->sgt.sgl, &sg_iter, mapping->sgt.orig_nents, + 0) { + page = sg_page_iter_page(&sg_iter); + if (mapping->dir == DMA_FROM_DEVICE || + mapping->dir == DMA_BIDIRECTIONAL) { + set_page_dirty(page); + } + + unpin_user_page(page); + } + + /* Free the mapping book-keeping */ + sg_free_table(&mapping->sgt); + kfree(mapping); +} + +int gxp_mapping_sync(struct gxp_dev *gxp, struct gxp_mapping *mapping, + u32 offset, u32 size, bool for_cpu) +{ + struct scatterlist *sg, *start_sg = NULL, *end_sg = NULL; + int nelems = 0, cur_offset = 0, ret = 0, i; + u64 start, end; + unsigned int start_diff = 0, end_diff = 0; + + /* + * Valid input requires + * - size > 0 (offset + size != offset) + * - offset + size does not overflow (offset + size > offset) + * - the mapped range falls within [0 : mapping->size] + */ + if (offset + size <= offset || + offset + size > mapping->size) + return -EINVAL; + + /* + * Mappings are created at a PAGE_SIZE granularity, however other data + * which is not part of the mapped buffer may be present in the first + * and last pages of the buffer's scattergather list. + * + * To ensure only the intended data is actually synced, iterate through + * the scattergather list, to find the first and last `scatterlist`s + * which contain the range of the buffer to sync. + * + * After those links are found, change their offset/lengths so that + * `dma_map_sg_for_*()` will only sync the requested region. + */ + start = (mapping->host_address & ~PAGE_MASK) + offset; + end = start + size; + for_each_sg(mapping->sgt.sgl, sg, mapping->sgt.orig_nents, i) { + if (end <= cur_offset) + break; + if (cur_offset <= start && start < cur_offset + sg->length) { + start_sg = sg; + start_diff = start - cur_offset; + } + if (start_sg) + nelems++; + cur_offset += sg->length; + end_sg = sg; + } + end_diff = cur_offset - end; + + /* Make sure a valid starting scatterlist was found for the start */ + if (!start_sg) + return -EINVAL; + + /* + * Since the scatter-gather list of the mapping is modified while it is + * being synced, only one sync for a given mapping can occur at a time. + * Rather than maintain a mutex for every mapping, lock the mapping list + * mutex, making all syncs mutually exclusive. + */ + mutex_lock(&gxp->mappings->lock); + + start_sg->offset += start_diff; + start_sg->dma_address += start_diff; + start_sg->length -= start_diff; + start_sg->dma_length -= start_diff; + end_sg->length -= end_diff; + end_sg->dma_length -= end_diff; + + if (for_cpu) + gxp_dma_sync_sg_for_cpu(gxp, start_sg, nelems, mapping->dir); + else + gxp_dma_sync_sg_for_device(gxp, start_sg, nelems, mapping->dir); + + /* + * Return the start and end scatterlists' offset/lengths to their + * original values for the next time they need to be synced/unmapped. + */ + end_sg->length += end_diff; + end_sg->dma_length += end_diff; + start_sg->offset -= start_diff; + start_sg->dma_address -= start_diff; + start_sg->length += start_diff; + start_sg->dma_length += start_diff; + + mutex_unlock(&gxp->mappings->lock); + + return ret; +} + +int gxp_mapping_put(struct gxp_dev *gxp, struct gxp_mapping *map) +{ + struct rb_node **link; + struct rb_node *parent = NULL; + u64 device_address = map->device_address; + struct gxp_mapping *this; + + link = &gxp->mappings->rb.rb_node; + + mutex_lock(&gxp->mappings->lock); + + /* Figure out where to put new node */ + while (*link) { + parent = *link; + this = rb_entry(parent, struct gxp_mapping, node); + + if (this->device_address > device_address) + link = &(*link)->rb_left; + else if (this->device_address < device_address) + link = &(*link)->rb_right; + else + goto out; + } + + /* Add new node and rebalance tree. */ + rb_link_node(&map->node, parent, link); + rb_insert_color(&map->node, &gxp->mappings->rb); + + mutex_unlock(&gxp->mappings->lock); + + return 0; + +out: + mutex_unlock(&gxp->mappings->lock); + dev_err(gxp->dev, "Duplicate mapping: 0x%llx", map->device_address); + return -EINVAL; +} + +struct gxp_mapping *gxp_mapping_get(struct gxp_dev *gxp, u64 device_address) +{ + struct rb_node *node; + struct gxp_mapping *this; + + mutex_lock(&gxp->mappings->lock); + + node = gxp->mappings->rb.rb_node; + + while (node) { + this = rb_entry(node, struct gxp_mapping, node); + + if (this->device_address > device_address) { + node = node->rb_left; + } else if (this->device_address < device_address) { + node = node->rb_right; + } else { + mutex_unlock(&gxp->mappings->lock); + return this; /* Found it */ + } + } + + mutex_unlock(&gxp->mappings->lock); + + dev_err(gxp->dev, "Mapping not found: 0x%llx", device_address); + return NULL; +} + +struct gxp_mapping *gxp_mapping_get_host(struct gxp_dev *gxp, u64 host_address) +{ + struct rb_node *node; + struct gxp_mapping *this; + + mutex_lock(&gxp->mappings->lock); + + /* Iterate through the elements in the rbtree */ + for (node = rb_first(&gxp->mappings->rb); node; node = rb_next(node)) { + this = rb_entry(node, struct gxp_mapping, node); + if (this->host_address == host_address) { + mutex_unlock(&gxp->mappings->lock); + return this; + } + } + + mutex_unlock(&gxp->mappings->lock); + + return NULL; +} + +void gxp_mapping_remove(struct gxp_dev *gxp, struct gxp_mapping *map) +{ + rb_erase(&map->node, &gxp->mappings->rb); +} -- cgit v1.2.3