path: root/mali_pixel/protected_memory_allocator.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2021 Google LLC.
 *
 * Protected memory allocator driver for allocation and release of pages of
 * protected memory for use by Mali GPU device drivers.
 */

#include <linux/dma-buf.h>
#include <linux/dma-heap.h>
#include <linux/of.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/protected_memory_allocator.h>
#include <linux/slab.h>

#define MALI_PMA_DMA_HEAP_NAME "vframe-secure"
#define MALI_PMA_SLAB_SIZE (1 << 16)
#define MALI_PMA_SLAB_BLOCK_SIZE (PAGE_SIZE)
#define MALI_PMA_SLAB_BLOCK_COUNT \
	(MALI_PMA_SLAB_SIZE / MALI_PMA_SLAB_BLOCK_SIZE)
#define MALI_PMA_MAX_ALLOC_SIZE (MALI_PMA_SLAB_SIZE)

/**
 * struct mali_pma_dev - Structure for managing a Mali protected memory
 *                       allocator device.
 *
 * @pma_dev: The base protected memory allocator device.
 * @dev: The device for which to allocate protected memory.
 * @dma_heap: The DMA buffer heap from which to allocate protected memory.
 * @slab_list: List of allocated slabs of protected memory.
 * @slab_mutex: Mutex used to serialize access to the slab list.
 */
struct mali_pma_dev {
	struct protected_memory_allocator_device pma_dev;
	struct device *dev;
	struct dma_heap *dma_heap;
	struct list_head slab_list;
	struct mutex slab_mutex;
};

/**
 * struct mali_protected_memory_allocation - Structure for tracking a Mali
 *                                           protected memory allocation.
 *
 * @pma: The base protected memory allocation record.
 * @slab: Protected memory slab used for allocation.
 * @first_block_index: Index of first memory block allocated from the slab.
 * @block_count: Count of the number of blocks allocated from the slab.
 */
struct mali_protected_memory_allocation {
	struct protected_memory_allocation pma;
	struct mali_pma_slab *slab;
	int first_block_index;
	int block_count;
};

/**
 * struct mali_pma_slab - Structure for managing a slab of Mali protected
 *                        memory.
 *
 * @list_entry: Entry in slab list.
 * @base: Physical base address of slab memory.
 * @dma_buf: The DMA buffer allocated for the slab . A reference to the DMA
 *           buffer is held by this pointer.
 * @dma_attachment: The DMA buffer device attachment.
 * @dma_sg_table: The DMA buffer scatter/gather table.
 * @allocated_block_map: Bit map of allocated blocks in the slab.
 */
struct mali_pma_slab {
	struct list_head list_entry;
	phys_addr_t base;
	struct dma_buf *dma_buf;
	struct dma_buf_attachment *dma_attachment;
	struct sg_table *dma_sg_table;
	uint64_t allocated_block_map;
};
static_assert(8 * sizeof(((struct mali_pma_slab *) 0)->allocated_block_map) >=
		MALI_PMA_SLAB_BLOCK_COUNT);

static struct protected_memory_allocation *mali_pma_alloc_page(
	struct protected_memory_allocator_device *pma_dev,
	unsigned int order);

static phys_addr_t mali_pma_get_phys_addr(
	struct protected_memory_allocator_device *pma_dev,
	struct protected_memory_allocation *pma);

static void mali_pma_free_page(
	struct protected_memory_allocator_device *pma_dev,
	struct protected_memory_allocation *pma);

static bool mali_pma_slab_alloc(
	struct mali_pma_dev* mali_pma_dev,
	struct mali_protected_memory_allocation *mali_pma, size_t size);

static void mali_pma_slab_dealloc(
	struct mali_pma_dev* mali_pma_dev,
	struct mali_protected_memory_allocation *mali_pma);

static bool mali_pma_slab_find_available(
	struct mali_pma_dev* mali_pma_dev, size_t size,
	struct mali_pma_slab** p_slab, int* p_block_index);

static struct mali_pma_slab* mali_pma_slab_add(
	struct mali_pma_dev* mali_pma_dev);

static void mali_pma_slab_remove(
	struct mali_pma_dev* mali_pma_dev, struct mali_pma_slab* slab);

static int protected_memory_allocator_probe(struct platform_device *pdev);

static int protected_memory_allocator_remove(struct platform_device *pdev);

/**
 * mali_pma_alloc_page - Allocate protected memory pages
 *
 * @pma_dev: The protected memory allocator the request is being made
 *           through.
 * @order:   How many pages to allocate, as a base-2 logarithm.
 *
 * Return: Pointer to allocated memory, or NULL if allocation failed.
 */
static struct protected_memory_allocation *mali_pma_alloc_page(
	struct protected_memory_allocator_device *pma_dev,
	unsigned int order) {
	struct mali_pma_dev *mali_pma_dev;
	struct protected_memory_allocation* pma = NULL;
	struct mali_protected_memory_allocation *mali_pma;
	size_t alloc_size;
	bool succeeded = false;

	/* Get the Mali protected memory allocator device record. */
	mali_pma_dev = container_of(pma_dev, struct mali_pma_dev, pma_dev);

	/* Check requested size against the maximum size. */
	alloc_size = 1 << (PAGE_SHIFT + order);
	if (alloc_size > MALI_PMA_MAX_ALLOC_SIZE) {
		dev_err(mali_pma_dev->dev,
			"Protected memory allocation size %zu too big\n",
			alloc_size);
		goto out;
	}

	/* Allocate a Mali protected memory allocation record. */
	mali_pma = devm_kzalloc(
		mali_pma_dev->dev, sizeof(*mali_pma), GFP_KERNEL);
	if (!mali_pma) {
		dev_err(mali_pma_dev->dev,
			"Failed to allocate a Mali protected memory allocation "
			"record\n");
		goto out;
	}
	pma = &(mali_pma->pma);
	pma->order = order;

	/* Allocate Mali protected memory from a slab. */
	if (!mali_pma_slab_alloc(mali_pma_dev, mali_pma, alloc_size)) {
		dev_err(mali_pma_dev->dev,
			"Failed to allocate Mali protected memory.\n");
		goto out;
	}

	/* Mark the allocation as successful. */
	succeeded = true;

out:
	/* Clean up on error. */
	if (!succeeded) {
		if (pma) {
			mali_pma_free_page(pma_dev, pma);
			pma = NULL;
		}
	}

	return pma;
}

/**
 * mali_pma_get_phys_addr - Get the physical address of the protected memory
 *                          allocation
 *
 * @pma_dev: The protected memory allocator the request is being made
 *           through.
 * @pma:     The protected memory allocation whose physical address
 *           shall be retrieved
 *
 * Return: The physical address of the given allocation.
 */
static phys_addr_t mali_pma_get_phys_addr(
	struct protected_memory_allocator_device *pma_dev,
	struct protected_memory_allocation *pma) {
	return pma->pa;
}

/**
 * mali_pma_free_page - Free a page of memory
 *
 * @pma_dev: The protected memory allocator the request is being made
 *           through.
 * @pma:     The protected memory allocation to free.
 */
static void mali_pma_free_page(
	struct protected_memory_allocator_device *pma_dev,
	struct protected_memory_allocation *pma) {
	struct mali_pma_dev *mali_pma_dev;
	struct mali_protected_memory_allocation *mali_pma;

	/*
	 * Get the Mali protected memory allocator device record and allocation
	 * record.
	 */
	mali_pma_dev = container_of(pma_dev, struct mali_pma_dev, pma_dev);
	mali_pma =
		container_of(pma, struct mali_protected_memory_allocation, pma);

	/* Deallocate Mali protected memory from the slab. */
	mali_pma_slab_dealloc(mali_pma_dev, mali_pma);

	/* Deallocate the Mali protected memory allocation record. */
	devm_kfree(mali_pma_dev->dev, mali_pma);
}

/**
 * mali_pma_slab_alloc - Allocate protected memory from a slab
 *
 * @mali_pma_dev: Mali protected memory allocator device.
 * @mali_pma: Mali protected memory allocation record to hold the slab memory.
 * @size: Size in bytes of memory to allocate.
 *
 * Return: True if memory was successfully allocated.
 */
static bool mali_pma_slab_alloc(
	struct mali_pma_dev *mali_pma_dev,
	struct mali_protected_memory_allocation *mali_pma, size_t size) {
	struct mali_pma_slab *slab;
	int start_block;
	int block_count;
	bool succeeded = false;

	/* Lock the slab list. */
	mutex_lock(&(mali_pma_dev->slab_mutex));

	/*
	 * Try finding an existing slab from which to allocate. If none are
	 * available, add a new slab and allocate from it.
	 */
	if (!mali_pma_slab_find_available(
		mali_pma_dev, size, &slab, &start_block)) {
		slab = mali_pma_slab_add(mali_pma_dev);
		if (!slab) {
			goto out;
		}
		start_block = 0;
	}

	/* Allocate a contiguous set of blocks from the slab. */
	block_count = DIV_ROUND_UP(size, MALI_PMA_SLAB_BLOCK_SIZE);
	bitmap_set((unsigned long *) &(slab->allocated_block_map),
			start_block, block_count);

	/*
	 * Use the allocated slab memory for the Mali protected memory
	 * allocation.
	 */
	mali_pma->pma.pa =
		slab->base + (start_block * MALI_PMA_SLAB_BLOCK_SIZE);
	mali_pma->slab = slab;
	mali_pma->first_block_index = start_block;
	mali_pma->block_count = block_count;

	/* Mark the allocation as successful. */
	succeeded = true;

out:
	/* Unlock the slab list. */
	mutex_unlock(&(mali_pma_dev->slab_mutex));

	return succeeded;
}

/**
 * mali_pma_slab_dealloc - Deallocate protected memory from a slab
 *
 * @mali_pma_dev: Mali protected memory allocator device.
 * @mali_pma: Mali protected memory allocation record holding slab memory to
 *            deallocate.
 */
static void mali_pma_slab_dealloc(
	struct mali_pma_dev *mali_pma_dev,
	struct mali_protected_memory_allocation *mali_pma) {
	struct mali_pma_slab *slab;

	/* Lock the slab list. */
	mutex_lock(&(mali_pma_dev->slab_mutex));

	/* Get the slab. */
	slab = mali_pma->slab;

	/* Deallocate the slab. */
	if (slab != NULL) {
		/* Deallocate all the blocks in the slab. */
		bitmap_clear((unsigned long *) &(slab->allocated_block_map),
				mali_pma->first_block_index,
				mali_pma->block_count);

		/* If no slab blocks remain allocated, remove the slab. */
		if (bitmap_empty(
			(unsigned long *) &(slab->allocated_block_map),
			MALI_PMA_SLAB_BLOCK_COUNT)) {
			mali_pma_slab_remove(mali_pma_dev, slab);
		}
	}

	/* Unlock the slab list. */
	mutex_unlock(&(mali_pma_dev->slab_mutex));
}

/**
 * mali_pma_slab_find_available - Find a slab with available memory
 *
 * Must be called with the slab list mutex locked.
 *
 * @mali_pma_dev: Mali protected memory allocator device.
 * @size: Size in bytes of requested memory.
 * @p_slab: Returned slab with requested memory available.
 * @p_block_index: Returned starting block index of available memory.
 *
 * Return: True if a slab was found with the requested memory available.
 */
static bool mali_pma_slab_find_available(
	struct mali_pma_dev *mali_pma_dev, size_t size,
	struct mali_pma_slab **p_slab, int *p_block_index) {
	struct mali_pma_slab *slab;
	int block_count;
	int start_block;
	bool found = false;

	/* Ensure the slab list mutex is locked. */
	lockdep_assert_held(&(mali_pma_dev->slab_mutex));

	/* Search slabs for a contiguous set of blocks of the requested size. */
	block_count = DIV_ROUND_UP(size, MALI_PMA_SLAB_BLOCK_SIZE);
	list_for_each_entry(slab, &(mali_pma_dev->slab_list), list_entry) {
		start_block = bitmap_find_next_zero_area_off(
			(unsigned long *) &(slab->allocated_block_map),
			MALI_PMA_SLAB_BLOCK_COUNT, 0, block_count, 0, 0);
		if (start_block < MALI_PMA_SLAB_BLOCK_COUNT) {
			found = true;
			break;
		}
	}

	/* Return results if found. */
	if (found) {
		*p_slab = slab;
		*p_block_index = start_block;
	}

	return found;
}

/**
 * mali_pma_slab_add - Allocate and add a new slab
 *
 * Must be called with the slab list mutex locked.
 *
 * @mali_pma_dev: Mali protected memory allocator device.
 *
 * Return: Newly added slab.
 */
static struct mali_pma_slab *mali_pma_slab_add(
	struct mali_pma_dev *mali_pma_dev) {
	struct mali_pma_slab *slab = NULL;
	struct dma_buf *dma_buf;
	struct dma_buf_attachment *dma_attachment;
	struct sg_table *dma_sg_table;
	bool succeeded = false;

	/* Ensure the slab list mutex is locked. */
	lockdep_assert_held(&(mali_pma_dev->slab_mutex));

	/* Allocate and initialize a Mali protected memory slab record. */
	slab = devm_kzalloc(mali_pma_dev->dev, sizeof(*slab), GFP_KERNEL);
	if (!slab) {
		dev_err(mali_pma_dev->dev,
			"Failed to allocate a Mali protected memory slab.\n");
		goto out;
	}
	INIT_LIST_HEAD(&(slab->list_entry));

	/* Allocate a DMA buffer. */
	dma_buf = dma_heap_buffer_alloc(
		mali_pma_dev->dma_heap, MALI_PMA_SLAB_SIZE, O_RDWR, 0);
	if (IS_ERR(dma_buf)) {
		dev_err(mali_pma_dev->dev,
			"Failed to allocate a DMA buffer of size %d\n",
			MALI_PMA_SLAB_SIZE);
		goto out;
	}
	slab->dma_buf = dma_buf;

	/* Attach the device to the DMA buffer. */
	dma_attachment = dma_buf_attach(dma_buf, mali_pma_dev->dev);
	if (IS_ERR(dma_attachment)) {
		dev_err(mali_pma_dev->dev,
			"Failed to attach the device to the DMA buffer\n");
		goto out;
	}
	slab->dma_attachment = dma_attachment;

	/* Map the DMA buffer into the attached device address space. */
	dma_sg_table =
		dma_buf_map_attachment(dma_attachment, DMA_BIDIRECTIONAL);
	if (IS_ERR(dma_sg_table)) {
		dev_err(mali_pma_dev->dev, "Failed to map the DMA buffer\n");
		goto out;
	}
	slab->dma_sg_table = dma_sg_table;
	slab->base = page_to_phys(sg_page(dma_sg_table->sgl));

	/* Add the slab to the slab list. */
	list_add(&(slab->list_entry), &(mali_pma_dev->slab_list));

	/* Mark that the slab was successfully added. */
	succeeded = true;

out:
	/* Clean up on failure. */
	if (!succeeded && (slab != NULL)) {
		mali_pma_slab_remove(mali_pma_dev, slab);
		slab = NULL;
	}

	return slab;
}

/**
 * mali_pma_slab_remove - Remove and deallocate a slab
 *
 * Must be called with the slab list mutex locked.
 *
 * @mali_pma_dev: Mali protected memory allocator device.
 * @slab: Slab to remove and deallocate.
 */
static void mali_pma_slab_remove(
	struct mali_pma_dev *mali_pma_dev, struct mali_pma_slab *slab) {
	/* Ensure the slab list mutex is locked. */
	lockdep_assert_held(&(mali_pma_dev->slab_mutex));

	/* Free the Mali protected memory slab allocation. */
	if (slab->dma_sg_table) {
		dma_buf_unmap_attachment(
			slab->dma_attachment,
	 		slab->dma_sg_table, DMA_BIDIRECTIONAL);
	}
	if (slab->dma_attachment) {
		dma_buf_detach(slab->dma_buf, slab->dma_attachment);
	}
	if (slab->dma_buf) {
		dma_buf_put(slab->dma_buf);
	}

	/* Remove the slab from the slab list. */
	list_del(&(slab->list_entry));

	/* Deallocate the Mali protected memory slab record. */
	devm_kfree(mali_pma_dev->dev, slab);
}

/**
 * protected_memory_allocator_probe - Probe the protected memory allocator
 *                                    device
 *
 * @pdev: The platform device to probe.
 */
static int protected_memory_allocator_probe(struct platform_device *pdev)
{
	struct dma_heap *pma_heap;
	struct mali_pma_dev *mali_pma_dev;
	struct protected_memory_allocator_device *pma_dev;
	int ret = 0;

	/* Try locating a PMA heap, defer if not present (yet). */
	pma_heap = dma_heap_find(MALI_PMA_DMA_HEAP_NAME);
	if (!pma_heap) {
		dev_warn(&(pdev->dev),
			"Failed to find \"%s\" DMA buffer heap. Deferring.\n",
			MALI_PMA_DMA_HEAP_NAME);
		ret = -EPROBE_DEFER;
		goto out;
	}

	/* Create a Mali protected memory allocator device record. */
	mali_pma_dev = kzalloc(sizeof(*mali_pma_dev), GFP_KERNEL);
	if (!mali_pma_dev) {
		dev_err(&(pdev->dev),
			"Failed to create a Mali protected memory allocator "
			"device record\n");
		dma_heap_put(pma_heap);
		ret = -ENOMEM;
		goto out;
	}
	pma_dev = &(mali_pma_dev->pma_dev);
	platform_set_drvdata(pdev, pma_dev);

	/* Initialize the slab list. */
	INIT_LIST_HEAD(&(mali_pma_dev->slab_list));
	mutex_init(&(mali_pma_dev->slab_mutex));

	/* Configure the Mali protected memory allocator. */
	mali_pma_dev->dev = &(pdev->dev);
	pma_dev->owner = THIS_MODULE;
	pma_dev->ops.pma_alloc_page = mali_pma_alloc_page;
	pma_dev->ops.pma_get_phys_addr = mali_pma_get_phys_addr;
	pma_dev->ops.pma_free_page = mali_pma_free_page;

	/* Assign the DMA buffer heap. */
	mali_pma_dev->dma_heap = pma_heap;

	/* Log that the protected memory allocator was successfully probed. */
	dev_info(&(pdev->dev),
		"Protected memory allocator probed successfully\n");

out:
	return ret;
}

/**
 * protected_memory_allocator_remove - Remove the protected memory allocator
 *                                     device
 *
 * @pdev: The protected memory allocator platform device to remove.
 */
static int protected_memory_allocator_remove(struct platform_device *pdev)
{
	struct protected_memory_allocator_device *pma_dev;
	struct mali_pma_dev *mali_pma_dev;

	/* Get the Mali protected memory allocator device record. */
	pma_dev = platform_get_drvdata(pdev);
	if (!pma_dev) {
		return 0;
	}
	mali_pma_dev = container_of(pma_dev, struct mali_pma_dev, pma_dev);

	/* Warn if there are any outstanding protected memory slabs. */
	if (!list_empty(&(mali_pma_dev->slab_list))) {
		dev_warn(&(pdev->dev),
			"Some protected memory has been left allocated\n");
	}

	/* Release the DMA buffer heap. */
	if (mali_pma_dev->dma_heap) {
		dma_heap_put(mali_pma_dev->dma_heap);
	}

	/* Free the Mali protected memory allocator device record. */
	kfree(mali_pma_dev);

	return 0;
}

static const struct of_device_id protected_memory_allocator_dt_ids[] = {
	{ .compatible = "arm,protected-memory-allocator" },
	{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, protected_memory_allocator_dt_ids);

struct platform_driver protected_memory_allocator_driver = {
	.probe = protected_memory_allocator_probe,
	.remove = protected_memory_allocator_remove,
	.driver = {
		.name = "mali-pma",
		.owner = THIS_MODULE,
		.of_match_table = of_match_ptr(protected_memory_allocator_dt_ids),
		.suppress_bind_attrs = true,
	}
};