diff options
author | Sidath Senanayake <sidaths@google.com> | 2017-05-18 14:43:17 +0200 |
---|---|---|
committer | Sidath Senanayake <sidaths@google.com> | 2017-05-18 14:43:17 +0200 |
commit | 6f5ab3baed824941f168ab133469f997d4450146 (patch) | |
tree | 95dcbadaa979f84a8d75c0919af1b85c5afd1924 | |
parent | 48f3554a4abf9ce182253fb5415a1a26b0790998 (diff) | |
download | gpu-6f5ab3baed824941f168ab133469f997d4450146.tar.gz |
Mali Bifrost DDK r6p0 KMD
Provenance:
b67d8663a (collaborate/EAC/b_r6p0)
BX304L01B-BU-00000-r6p0-01rel0
BX304L06A-BU-00000-r6p0-01rel0
BX304X07X-BU-00000-r6p0-01rel0
Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: I6b19113374d523be9a75a252d672731600e991d6
99 files changed, 7921 insertions, 2556 deletions
diff --git a/mali_kbase/Kbuild b/mali_kbase/Kbuild index 3e78928..eb01993 100644 --- a/mali_kbase/Kbuild +++ b/mali_kbase/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -15,7 +15,7 @@ # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= "r5p0-01rel0" +MALI_RELEASE_NAME ?= "r6p0-01rel0" # Paths required for build KBASE_PATH = $(src) @@ -77,6 +77,7 @@ SRC := \ mali_kbase_cache_policy.c \ mali_kbase_mem.c \ mali_kbase_mmu.c \ + mali_kbase_ctx_sched.c \ mali_kbase_jd.c \ mali_kbase_jd_debugfs.c \ mali_kbase_jm.c \ @@ -97,8 +98,6 @@ SRC := \ mali_kbase_gpu_memory_debugfs.c \ mali_kbase_mem_linux.c \ mali_kbase_core_linux.c \ - mali_kbase_sync.c \ - mali_kbase_sync_user.c \ mali_kbase_replay.c \ mali_kbase_mem_profile_debugfs.c \ mali_kbase_mmu_mode_lpae.c \ @@ -143,16 +142,6 @@ ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y) ccflags-y += -I$(src)/platform/rtsm_ve endif - ifeq ($(CONFIG_MALI_PLATFORM_JUNO),y) - SRC += platform/juno/mali_kbase_config_vexpress.c - ccflags-y += -I$(src)/platform/juno - endif - - ifeq ($(CONFIG_MALI_PLATFORM_JUNO_SOC),y) - SRC += platform/juno_soc/mali_kbase_config_juno_soc.c - ccflags-y += -I$(src)/platform/juno_soc - endif - ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_1XV7_A57),y) SRC += platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c ccflags-y += -I$(src)/platform/vexpress_1xv7_a57 @@ -179,11 +168,22 @@ ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y) include $(src)/$(MALI_PLATFORM_THIRDPARTY_DIR)/Kbuild endif -ifeq ($(CONFIG_DEVFREQ_THERMAL),y) - include $(src)/ipa/Kbuild +ifeq ($(CONFIG_MALI_DEVFREQ),y) + ifeq ($(CONFIG_DEVFREQ_THERMAL),y) + include $(src)/ipa/Kbuild + endif endif -mali_kbase-$(CONFIG_MALI_DMA_FENCE) += mali_kbase_dma_fence.o +mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \ + mali_kbase_dma_fence.o \ + mali_kbase_fence.o +mali_kbase-$(CONFIG_SYNC) += \ + mali_kbase_sync_android.o \ + mali_kbase_sync_common.o +mali_kbase-$(CONFIG_SYNC_FILE) += \ + mali_kbase_sync_file.o \ + mali_kbase_sync_common.o \ + mali_kbase_fence.o MALI_BACKEND_PATH ?= backend CONFIG_MALI_BACKEND ?= gpu @@ -213,3 +213,6 @@ mali_kbase-$(CONFIG_MALI_PLATFORM_DEVICETREE) += \ platform/devicetree/mali_kbase_runtime_pm.o \ platform/devicetree/mali_kbase_config_devicetree.o ccflags-$(CONFIG_MALI_PLATFORM_DEVICETREE) += -I$(src)/platform/devicetree + +# For kutf and mali_kutf_irq_latency_test +obj-$(CONFIG_MALI_KUTF) += tests/ diff --git a/mali_kbase/Kconfig b/mali_kbase/Kconfig index c1acaf0..54565bb 100644 --- a/mali_kbase/Kconfig +++ b/mali_kbase/Kconfig @@ -169,7 +169,7 @@ config MALI_DEBUG config MALI_FENCE_DEBUG bool "Debug sync fence usage" - depends on MALI_MIDGARD && MALI_EXPERT && SYNC + depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE) default y if MALI_DEBUG help Select this option to enable additional checking and reporting on the @@ -244,3 +244,4 @@ config MALI_PWRSOFT_765 parts of the code. source "drivers/gpu/arm/midgard/platform/Kconfig" +source "drivers/gpu/arm/midgard/tests/Kconfig" diff --git a/mali_kbase/Makefile b/mali_kbase/Makefile index e1625e6..9aa242c 100644 --- a/mali_kbase/Makefile +++ b/mali_kbase/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/backend/gpu/mali_kbase_devfreq.c b/mali_kbase/backend/gpu/mali_kbase_devfreq.c index 574eb3e..e280322 100644 --- a/mali_kbase/backend/gpu/mali_kbase_devfreq.c +++ b/mali_kbase/backend/gpu/mali_kbase_devfreq.c @@ -40,6 +40,7 @@ #define dev_pm_opp_get_voltage opp_get_voltage #define dev_pm_opp_get_opp_count opp_get_opp_count #define dev_pm_opp_find_freq_ceil opp_find_freq_ceil +#define dev_pm_opp_find_freq_floor opp_find_freq_floor #endif /* Linux >= 3.13 */ /** @@ -362,7 +363,7 @@ int kbase_devfreq_init(struct kbase_device *kbdev) kbdev->devfreq_cooling = of_devfreq_cooling_register_power( kbdev->dev->of_node, kbdev->devfreq, - &power_model_ops); + &kbase_ipa_power_model_ops); if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) { err = PTR_ERR(kbdev->devfreq_cooling); dev_err(kbdev->dev, diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_as.c b/mali_kbase/backend/gpu/mali_kbase_jm_as.c index 202dcfa..92358f2 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_as.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_as.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,6 +22,7 @@ #include <mali_kbase.h> #include <mali_kbase_hwaccess_jm.h> +#include <mali_kbase_ctx_sched.h> /** * assign_and_activate_kctx_addr_space - Assign an AS to a context @@ -47,65 +48,20 @@ static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, struct kbase_as *current_as) { struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbasep_js_per_as_data *js_per_as_data; - int as_nr = current_as->number; lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); lockdep_assert_held(&js_devdata->runpool_mutex); lockdep_assert_held(&kbdev->hwaccess_lock); - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - /* Attribute handling */ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); - /* Assign addr space */ - kctx->as_nr = as_nr; - - /* If the GPU is currently powered, activate this address space on the - * MMU */ - if (kbdev->pm.backend.gpu_powered) - kbase_mmu_update(kctx); - /* If the GPU was not powered then the MMU will be reprogrammed on the - * next pm_context_active() */ - /* Allow it to run jobs */ kbasep_js_set_submit_allowed(js_devdata, kctx); - /* Book-keeping */ - js_per_as_data->kctx = kctx; - js_per_as_data->as_busy_refcount = 0; - kbase_js_runpool_inc_context_count(kbdev, kctx); } -/** - * release_addr_space - Release an address space - * @kbdev: Kbase device - * @kctx_as_nr: Address space of context to release - * @kctx: Context being released - * - * Context: kbasep_js_device_data.runpool_mutex must be held - * - * Release an address space, making it available for being picked again. - */ -static void release_addr_space(struct kbase_device *kbdev, int kctx_as_nr, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - u16 as_bit = (1u << kctx_as_nr); - - js_devdata = &kbdev->js_data; - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* The address space must not already be free */ - KBASE_DEBUG_ASSERT(!(js_devdata->as_free & as_bit)); - - js_devdata->as_free |= as_bit; - - kbase_js_runpool_dec_context_count(kbdev, kctx); -} - bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, struct kbase_context *kctx) { @@ -117,10 +73,7 @@ bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, } for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - struct kbasep_js_per_as_data *js_per_as_data = - &kbdev->js_data.runpool_irq.per_as_data[i]; - - if (js_per_as_data->kctx == kctx) { + if (kbdev->as_to_kctx[i] == kctx) { /* Context already has ASID - mark as active */ return true; } @@ -133,7 +86,6 @@ bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, struct kbase_context *kctx) { - struct kbasep_js_per_as_data *js_per_as_data; int as_nr = kctx->as_nr; if (as_nr == KBASEP_AS_NR_INVALID) { @@ -143,100 +95,24 @@ void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, lockdep_assert_held(&kbdev->hwaccess_lock); - js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr]; - if (js_per_as_data->as_busy_refcount != 0) { + if (atomic_read(&kctx->refcount) != 1) { WARN(1, "Attempting to release active ASID\n"); return; } - /* Release context from address space */ - js_per_as_data->kctx = NULL; - kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); - /* If the GPU is currently powered, de-activate this address space on - * the MMU */ - if (kbdev->pm.backend.gpu_powered) - kbase_mmu_disable(kctx); - /* If the GPU was not powered then the MMU will be reprogrammed on the - * next pm_context_active() */ - - release_addr_space(kbdev, as_nr, kctx); - kctx->as_nr = KBASEP_AS_NR_INVALID; -} - -void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ -} - -void kbase_backend_release_free_address_space(struct kbase_device *kbdev, - int as_nr) -{ - struct kbasep_js_device_data *js_devdata; - - js_devdata = &kbdev->js_data; - - lockdep_assert_held(&js_devdata->runpool_mutex); - js_devdata->as_free |= (1 << as_nr); + kbase_ctx_sched_release_ctx(kctx); + kbase_js_runpool_dec_context_count(kbdev, kctx); } -/** - * check_is_runpool_full - check whether the runpool is full for a specified - * context - * @kbdev: Kbase device - * @kctx: Kbase context - * - * If kctx == NULL, then this makes the least restrictive check on the - * runpool. A specific context that is supplied immediately after could fail - * the check, even under the same conditions. - * - * Therefore, once a context is obtained you \b must re-check it with this - * function, since the return value could change to false. - * - * Context: - * In all cases, the caller must hold kbasep_js_device_data.runpool_mutex. - * When kctx != NULL the caller must hold the - * kbasep_js_kctx_info.ctx.jsctx_mutex. - * When kctx == NULL, then the caller need not hold any jsctx_mutex locks (but - * it doesn't do any harm to do so). - * - * Return: true if the runpool is full - */ -static bool check_is_runpool_full(struct kbase_device *kbdev, +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, struct kbase_context *kctx) { - struct kbasep_js_device_data *js_devdata; - bool is_runpool_full; - - js_devdata = &kbdev->js_data; - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* Regardless of whether a context is submitting or not, can't have more - * than there are HW address spaces */ - is_runpool_full = (bool) (js_devdata->nr_all_contexts_running >= - kbdev->nr_hw_address_spaces); - - if (kctx && !kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); - /* Contexts that submit might use less of the address spaces - * available, due to HW workarounds. In which case, the runpool - * is also full when the number of submitting contexts exceeds - * the number of submittable address spaces. - * - * Both checks must be made: can have nr_user_address_spaces == - * nr_hw_address spaces, and at the same time can have - * nr_user_contexts_running < nr_all_contexts_running. */ - is_runpool_full |= (bool) - (js_devdata->nr_user_contexts_running >= - kbdev->nr_user_address_spaces); - } - - return is_runpool_full; } -int kbase_backend_find_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx) +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx) { struct kbasep_js_device_data *js_devdata; struct kbasep_js_kctx_info *js_kctx_info; @@ -249,37 +125,23 @@ int kbase_backend_find_free_address_space(struct kbase_device *kbdev, mutex_lock(&js_kctx_info->ctx.jsctx_mutex); mutex_lock(&js_devdata->runpool_mutex); - /* First try to find a free address space */ - if (check_is_runpool_full(kbdev, kctx)) - i = -1; - else - i = ffs(js_devdata->as_free) - 1; - - if (i >= 0 && i < kbdev->nr_hw_address_spaces) { - js_devdata->as_free &= ~(1 << i); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return i; - } - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - /* No address space currently free, see if we can release one */ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - struct kbasep_js_per_as_data *js_per_as_data; struct kbasep_js_kctx_info *as_js_kctx_info; struct kbase_context *as_kctx; - js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[i]; - as_kctx = js_per_as_data->kctx; + as_kctx = kbdev->as_to_kctx[i]; as_js_kctx_info = &as_kctx->jctx.sched_info; /* Don't release privileged or active contexts, or contexts with - * jobs running */ + * jobs running. + * Note that a context will have at least 1 reference (which + * was previously taken by kbasep_js_schedule_ctx()) until + * descheduled. + */ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && - js_per_as_data->as_busy_refcount == 0) { + atomic_read(&as_kctx->refcount) == 1) { if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, as_kctx)) { WARN(1, "Failed to retain active context\n"); @@ -314,8 +176,6 @@ int kbase_backend_find_free_address_space(struct kbase_device *kbdev, as_kctx, true); - js_devdata->as_free &= ~(1 << i); - mutex_unlock(&js_devdata->runpool_mutex); mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); @@ -353,16 +213,15 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev, js_devdata = &kbdev->js_data; js_kctx_info = &kctx->jctx.sched_info; - if (kbdev->hwaccess.active_kctx == kctx || - kctx->as_nr != KBASEP_AS_NR_INVALID || - as_nr == KBASEP_AS_NR_INVALID) { - WARN(1, "Invalid parameters to use_ctx()\n"); + if (kbdev->hwaccess.active_kctx == kctx) { + WARN(1, "Context is already scheduled in\n"); return false; } new_address_space = &kbdev->as[as_nr]; lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->mmu_hw_mutex); lockdep_assert_held(&kbdev->hwaccess_lock); assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c index ef7497d..a6fb097 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c @@ -29,6 +29,7 @@ #include <mali_kbase_vinstr.h> #include <mali_kbase_hw.h> #include <mali_kbase_hwaccess_jm.h> +#include <mali_kbase_ctx_sched.h> #include <backend/gpu/mali_kbase_device_internal.h> #include <backend/gpu/mali_kbase_irq_internal.h> #include <backend/gpu/mali_kbase_js_affinity.h> @@ -1155,7 +1156,6 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) { unsigned long flags; struct kbase_device *kbdev; - int i; ktime_t end_timestamp = ktime_get(); struct kbasep_js_device_data *js_devdata; bool try_schedule = false; @@ -1193,6 +1193,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) KBASE_RESET_GPU_NOT_PENDING); kbase_disjoint_state_down(kbdev); wake_up(&kbdev->hwaccess.backend.reset_wait); + kbase_vinstr_resume(kbdev->vinstr_ctx); return; } @@ -1265,18 +1266,9 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) mutex_lock(&js_devdata->runpool_mutex); mutex_lock(&kbdev->mmu_hw_mutex); - /* Reprogram the GPU's MMU */ - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (js_devdata->runpool_irq.per_as_data[i].kctx) - kbase_mmu_update( - js_devdata->runpool_irq.per_as_data[i].kctx); - else - kbase_mmu_disable_as(kbdev, i); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->mmu_hw_mutex); kbase_pm_enable_interrupts(kbdev); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c index 418ae1f..1e9b9e5 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c @@ -672,10 +672,9 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { kbase_vinstr_resume(kbdev->vinstr_ctx); -#ifdef CONFIG_DEVFREQ_THERMAL + /* Go back to configured model for IPA */ kbase_ipa_model_use_configured_locked(kbdev); -#endif } @@ -784,7 +783,8 @@ static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) if (kbdev->protected_ops) { /* Switch GPU to protected mode */ - err = kbdev->protected_ops->protected_mode_enter(kbdev); + err = kbdev->protected_ops->protected_mode_enable( + kbdev->protected_dev); if (err) dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", @@ -806,6 +806,8 @@ static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) if (!kbdev->protected_ops) return -EINVAL; + /* The protected mode disable callback will be called as part of reset + */ kbase_reset_gpu_silent(kbdev); return 0; @@ -841,10 +843,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, return -EAGAIN; } -#ifdef CONFIG_DEVFREQ_THERMAL /* Use generic model for IPA in protected mode */ kbase_ipa_model_use_fallback_locked(kbdev); -#endif /* Once reaching this point GPU must be * switched to protected mode or vinstr @@ -907,10 +907,9 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, kbase_gpu_dequeue_atom(kbdev, js, NULL); kbase_jm_return_atom_to_js(kbdev, katom[idx]); } -#ifdef CONFIG_DEVFREQ_THERMAL + /* Go back to configured model for IPA */ kbase_ipa_model_use_configured_locked(kbdev); -#endif return -EINVAL; } @@ -990,10 +989,9 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, } kbase_vinstr_resume(kbdev->vinstr_ctx); -#ifdef CONFIG_DEVFREQ_THERMAL + /* Use generic model for IPA in protected mode */ kbase_ipa_model_use_fallback_locked(kbdev); -#endif return -EINVAL; } @@ -1516,6 +1514,8 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) { KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); + kbase_vinstr_resume(kbdev->vinstr_ctx); + /* protected mode sanity checks */ KBASE_DEBUG_ASSERT_MSG( kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), diff --git a/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c b/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c index 4e5a74f..aa1817c 100644 --- a/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c +++ b/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -200,15 +200,15 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) KBASE_MMU_FAULT_TYPE_BUS : KBASE_MMU_FAULT_TYPE_PAGE; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - as->fault_extra_addr = kbase_reg_read(kbdev, - MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), - kctx); - as->fault_extra_addr <<= 32; - as->fault_extra_addr |= kbase_reg_read(kbdev, - MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), - kctx); -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + as->fault_extra_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), + kctx); + as->fault_extra_addr <<= 32; + as->fault_extra_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), + kctx); + } if (kbase_as_has_bus_fault(as)) { /* Mark bus fault as handled. @@ -248,34 +248,32 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, struct kbase_mmu_setup *current_setup = &as->current_setup; u32 transcfg = 0; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - transcfg = current_setup->transcfg & 0xFFFFFFFFUL; - - /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ - /* Clear PTW_MEMATTR bits */ - transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; - /* Enable correct PTW_MEMATTR bits */ - transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; - - if (kbdev->system_coherency == COHERENCY_ACE) { - /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ - /* Clear PTW_SH bits */ - transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); - /* Enable correct PTW_SH bits */ - transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); - } - - kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), - transcfg, kctx); - kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), - (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, kctx); - -#else /* CONFIG_MALI_GPU_MMU_AARCH64 */ - - if (kbdev->system_coherency == COHERENCY_ACE) - current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + transcfg = current_setup->transcfg & 0xFFFFFFFFUL; + + /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ + /* Clear PTW_MEMATTR bits */ + transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; + /* Enable correct PTW_MEMATTR bits */ + transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ + /* Clear PTW_SH bits */ + transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); + /* Enable correct PTW_SH bits */ + transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); + } -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), + transcfg, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), + (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, + kctx); + } else { + if (kbdev->system_coherency == COHERENCY_ACE) + current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; + } kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), current_setup->transtab & 0xFFFFFFFFUL, kctx); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c index ac63aa5..ed19a8a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c @@ -32,6 +32,7 @@ #include <mali_kbase_config_defaults.h> #include <mali_kbase_smc.h> #include <mali_kbase_hwaccess_jm.h> +#include <mali_kbase_ctx_sched.h> #include <backend/gpu/mali_kbase_cache_policy_backend.h> #include <backend/gpu/mali_kbase_device_internal.h> #include <backend/gpu/mali_kbase_irq_internal.h> @@ -1025,7 +1026,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) bool reset_required = is_resume; struct kbasep_js_device_data *js_devdata = &kbdev->js_data; unsigned long flags; - int i; KBASE_DEBUG_ASSERT(NULL != kbdev); lockdep_assert_held(&js_devdata->runpool_mutex); @@ -1067,18 +1067,9 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) } mutex_lock(&kbdev->mmu_hw_mutex); - /* Reprogram the GPU's MMU */ - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (js_devdata->runpool_irq.per_as_data[i].kctx) - kbase_mmu_update( - js_devdata->runpool_irq.per_as_data[i].kctx); - else - kbase_mmu_disable_as(kbdev, i); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->mmu_hw_mutex); /* Lastly, enable the interrupts */ @@ -1359,7 +1350,7 @@ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) } } -static int kbase_pm_reset_do_normal(struct kbase_device *kbdev) +static int kbase_pm_do_reset(struct kbase_device *kbdev) { struct kbasep_reset_timeout_data rtdata; @@ -1439,14 +1430,29 @@ static int kbase_pm_reset_do_normal(struct kbase_device *kbdev) return -EINVAL; } -static int kbase_pm_reset_do_protected(struct kbase_device *kbdev) +static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) { - KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); - KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); + struct kbase_device *kbdev = pdev->data; + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SET_PROTECTED_MODE, NULL); + return 0; +} - return kbdev->protected_ops->protected_mode_reset(kbdev); +static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + lockdep_assert_held(&kbdev->pm.lock); + + return kbase_pm_do_reset(kbdev); } +struct protected_mode_ops kbase_native_protected_ops = { + .protected_mode_enable = kbasep_protected_mode_enable, + .protected_mode_disable = kbasep_protected_mode_disable +}; + int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) { unsigned long irq_flags; @@ -1490,19 +1496,17 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); /* Soft reset the GPU */ - if (kbdev->protected_mode_support && - kbdev->protected_ops->protected_mode_reset) - err = kbase_pm_reset_do_protected(kbdev); + if (kbdev->protected_mode_support) + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); else - err = kbase_pm_reset_do_normal(kbdev); + err = kbase_pm_do_reset(kbdev); spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); if (kbdev->protected_mode) resume_vinstr = true; kbdev->protected_mode = false; -#ifdef CONFIG_DEVFREQ_THERMAL kbase_ipa_model_use_configured_locked(kbdev); -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h index 58f615d..6804f45 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -486,11 +486,11 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); */ void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); -#ifdef CONFIG_PM_DEVFREQ +#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, unsigned long *total, unsigned long *busy); void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); -#endif +#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */ #ifdef CONFIG_MALI_MIDGARD_DVFS diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c index 7613e1d..024248c 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -155,7 +155,7 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, kbdev->pm.backend.metrics.time_period_start = now; } -#if defined(CONFIG_PM_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) +#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) /* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this * function. */ diff --git a/mali_kbase/ipa/Kbuild b/mali_kbase/ipa/Kbuild index b3f8761..03dec47 100644 --- a/mali_kbase/ipa/Kbuild +++ b/mali_kbase/ipa/Kbuild @@ -18,3 +18,7 @@ mali_kbase-y += \ ipa/mali_kbase_ipa.o mali_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o + +ifneq ($(wildcard $(src)/ipa/mali_kbase_ipa_tmix.c),) + mali_kbase-y += ipa/mali_kbase_ipa_tmix.o +endif diff --git a/mali_kbase/ipa/mali_kbase_ipa.c b/mali_kbase/ipa/mali_kbase_ipa.c index d289ea8..ba897d7 100644 --- a/mali_kbase/ipa/mali_kbase_ipa.c +++ b/mali_kbase/ipa/mali_kbase_ipa.c @@ -32,6 +32,10 @@ #define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" +static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { + &kbase_simple_ipa_model_ops, +}; + int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) { int err = 0; @@ -50,57 +54,20 @@ int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) return err; } -int kbase_ipa_model_ops_register(struct kbase_device *kbdev, - struct kbase_ipa_model_ops *new_model_ops) -{ - struct kbase_ipa_model *new_model; - - lockdep_assert_held(&kbdev->ipa.lock); - - new_model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); - if (!new_model) - return -ENOMEM; - - new_model->kbdev = kbdev; - new_model->ops = new_model_ops; - - list_add(&new_model->link, &kbdev->ipa.power_models); - - return 0; -} - -static int kbase_ipa_internal_models_append_list(struct kbase_device *kbdev) -{ - int err; - - INIT_LIST_HEAD(&kbdev->ipa.power_models); - - /* Always have the simple IPA model */ - err = kbase_ipa_model_ops_register(kbdev, &kbase_simple_ipa_model_ops); - - if (err) - return err; - - return err; -} - -struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, - const char *name) +static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, + const char *name) { - /* Search registered power models first */ - struct list_head *it; + int i; - lockdep_assert_held(&kbdev->ipa.lock); + for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { + struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; - list_for_each(it, &kbdev->ipa.power_models) { - struct kbase_ipa_model *model = - list_entry(it, - struct kbase_ipa_model, - link); - if (strcmp(model->ops->name, name) == 0) - return model; + if (!strcmp(ops->name, name)) + return ops; } + dev_err(kbdev->dev, "power model \'%s\' not found\n", name); + return NULL; } @@ -109,7 +76,6 @@ void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) atomic_set(&kbdev->ipa_use_configured_model, false); } - void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) { atomic_set(&kbdev->ipa_use_configured_model, true); @@ -142,72 +108,54 @@ static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, NULL, compat_string); - if (!model_dt_node) { + if (!model_dt_node && !model->missing_dt_node_warning) { dev_warn(model->kbdev->dev, "Couldn't find power_model DT node matching \'%s\'\n", compat_string); + model->missing_dt_node_warning = true; } return model_dt_node; } -int kbase_ipa_model_add_param_u32_def(struct kbase_ipa_model *model, - const char *name, u32 *addr, - bool has_default, u32 default_value) -{ - int err; - struct device_node *model_dt_node = get_model_dt_node(model); - - err = of_property_read_u32(model_dt_node, name, addr); - - if (err && !has_default) { - dev_err(model->kbdev->dev, - "No DT entry or default found for %s.%s, err = %d\n", - model->ops->name, name, err); - goto exit; - } else if (err && has_default) { - *addr = default_value; - dev_dbg(model->kbdev->dev, "%s.%s = %u (default)\n", - model->ops->name, name, *addr); - err = 0; - } else /* !err */ { - dev_dbg(model->kbdev->dev, "%s.%s = %u (DT)\n", - model->ops->name, name, *addr); - } - - err = kbase_ipa_model_param_add(model, name, addr, sizeof(u32), - PARAM_TYPE_U32); -exit: - return err; -} - -int kbase_ipa_model_add_param_s32_array(struct kbase_ipa_model *model, - const char *name, s32 *addr, - size_t num_elems) +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required) { int err, i; struct device_node *model_dt_node = get_model_dt_node(model); + char *origin; err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); - if (err) { - dev_err(model->kbdev->dev, - "No DT entry found for %s.%s, err = %d\n", - model->ops->name, name, err); - goto exit; - } else { - for (i = 0; i < num_elems; ++i) - dev_dbg(model->kbdev->dev, "%s.%s[%u] = %d (DT)\n", - model->ops->name, name, i, *(addr + i)); + if (err && dt_required) { + memset(addr, 0, sizeof(s32) * num_elems); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = %zu*[0]\n", + err, model->ops->name, name, num_elems); + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + origin = "DT"; } /* Create a unique debugfs entry for each element */ for (i = 0; i < num_elems; ++i) { char elem_name[32]; - snprintf(elem_name, sizeof(elem_name), "%s.%d", name, i); - err = kbase_ipa_model_param_add(model, elem_name, &addr[i], - sizeof(s32), PARAM_TYPE_S32); + if (num_elems == 1) + snprintf(elem_name, sizeof(elem_name), "%s", name); + else + snprintf(elem_name, sizeof(elem_name), "%s.%d", + name, i); + + dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", + model->ops->name, elem_name, addr[i], origin); + + err = kbase_ipa_model_param_add(model, elem_name, + &addr[i], sizeof(s32), + PARAM_TYPE_S32); if (err) goto exit; } @@ -217,32 +165,41 @@ exit: int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, const char *name, char *addr, - size_t len) + size_t size, bool dt_required) { int err; struct device_node *model_dt_node = get_model_dt_node(model); const char *string_prop_value; + char *origin; err = of_property_read_string(model_dt_node, name, &string_prop_value); - if (err) { - dev_err(model->kbdev->dev, - "No DT entry found for %s.%s, err = %d\n", - model->ops->name, name, err); - goto exit; - } else { - strncpy(addr, string_prop_value, len); - dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (DT)\n", - model->ops->name, name, string_prop_value); + if (err && dt_required) { + strncpy(addr, "", size - 1); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = \'%s\'\n", + err, model->ops->name, name, addr); + err = 0; + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + strncpy(addr, string_prop_value, size - 1); + origin = "DT"; } - err = kbase_ipa_model_param_add(model, name, addr, len, + addr[size - 1] = '\0'; + + dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", + model->ops->name, name, string_prop_value, origin); + + err = kbase_ipa_model_param_add(model, name, addr, size, PARAM_TYPE_STRING); -exit: + return err; } -static void term_model(struct kbase_ipa_model *model) +void kbase_ipa_term_model(struct kbase_ipa_model *model) { if (!model) return; @@ -253,42 +210,49 @@ static void term_model(struct kbase_ipa_model *model) model->ops->term(model); kbase_ipa_model_param_free_all(model); + + kfree(model); } +KBASE_EXPORT_TEST_API(kbase_ipa_term_model); -static struct kbase_ipa_model *init_model(struct kbase_device *kbdev, - const char *model_name) +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops) { struct kbase_ipa_model *model; int err; lockdep_assert_held(&kbdev->ipa.lock); - model = kbase_ipa_get_model(kbdev, model_name); - if (!model) { - dev_err(kbdev->dev, "power model \'%s\' not found\n", - model_name); + if (!ops || !ops->name) return NULL; - } + model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); + if (!model) + return NULL; + + model->kbdev = kbdev; + model->ops = ops; INIT_LIST_HEAD(&model->params); err = model->ops->init(model); if (err) { dev_err(kbdev->dev, "init of power model \'%s\' returned error %d\n", - model_name, err); - term_model(model); - return NULL; + ops->name, err); + goto term_model; } err = kbase_ipa_model_recalculate(model); - if (err) { - term_model(model); - return NULL; - } + if (err) + goto term_model; return model; + +term_model: + kbase_ipa_term_model(model); + return NULL; } +KBASE_EXPORT_TEST_API(kbase_ipa_init_model); static void kbase_ipa_term_locked(struct kbase_device *kbdev) { @@ -296,24 +260,18 @@ static void kbase_ipa_term_locked(struct kbase_device *kbdev) /* Clean up the models */ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) - term_model(kbdev->ipa.configured_model); - term_model(kbdev->ipa.fallback_model); - - /* Clean up the list */ - if (!list_empty(&kbdev->ipa.power_models)) { - struct kbase_ipa_model *model_p, *model_n; + kbase_ipa_term_model(kbdev->ipa.configured_model); + kbase_ipa_term_model(kbdev->ipa.fallback_model); - list_for_each_entry_safe(model_p, model_n, &kbdev->ipa.power_models, link) { - list_del(&model_p->link); - kfree(model_p); - } - } + kbdev->ipa.configured_model = NULL; + kbdev->ipa.fallback_model = NULL; } int kbase_ipa_init(struct kbase_device *kbdev) { const char *model_name; + struct kbase_ipa_model_ops *ops; struct kbase_ipa_model *default_model = NULL; int err; @@ -324,11 +282,18 @@ int kbase_ipa_init(struct kbase_device *kbdev) */ mutex_lock(&kbdev->ipa.lock); - /* Add default ones to the list */ - err = kbase_ipa_internal_models_append_list(kbdev); - /* The simple IPA model must *always* be present.*/ - default_model = init_model(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); + ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); + + if (!ops->do_utilization_scaling_in_framework) { + dev_err(kbdev->dev, + "Fallback IPA model %s should not account for utilization\n", + ops->name); + err = -EINVAL; + goto end; + } + + default_model = kbase_ipa_init_model(kbdev, ops); if (!default_model) { err = -EINVAL; goto end; @@ -344,10 +309,18 @@ int kbase_ipa_init(struct kbase_device *kbdev) gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; model_name = kbase_ipa_model_name_from_id(gpu_id); + dev_dbg(kbdev->dev, + "Inferring model from GPU ID 0x%x: \'%s\'\n", + gpu_id, model_name); + } else { + dev_dbg(kbdev->dev, + "Using ipa-model parameter from DT: \'%s\'\n", + model_name); } if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { - kbdev->ipa.configured_model = init_model(kbdev, model_name); + ops = kbase_ipa_model_ops_find(kbdev, model_name); + kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); if (!kbdev->ipa.configured_model) { err = -EINVAL; goto end; @@ -365,12 +338,13 @@ end: else dev_info(kbdev->dev, "Using configured power model %s, and fallback %s\n", - kbdev->ipa.fallback_model->ops->name, - kbdev->ipa.configured_model->ops->name); + kbdev->ipa.configured_model->ops->name, + kbdev->ipa.fallback_model->ops->name); mutex_unlock(&kbdev->ipa.lock); return err; } +KBASE_EXPORT_TEST_API(kbase_ipa_init); void kbase_ipa_term(struct kbase_device *kbdev) { @@ -378,6 +352,7 @@ void kbase_ipa_term(struct kbase_device *kbdev) kbase_ipa_term_locked(kbdev); mutex_unlock(&kbdev->ipa.lock); } +KBASE_EXPORT_TEST_API(kbase_ipa_term); /** * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP @@ -393,21 +368,20 @@ void kbase_ipa_term(struct kbase_device *kbdev) * * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) */ -static inline unsigned long kbase_scale_dynamic_power(const unsigned long c, - const unsigned long freq, - const unsigned long voltage) +static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, + const u32 voltage) { /* Range: 2^8 < v2 < 2^16 m(V^2) */ - const unsigned long v2 = (voltage * voltage) / 1000; + const u32 v2 = (voltage * voltage) / 1000; /* Range: 2^3 < f_MHz < 2^10 MHz */ - const unsigned long f_MHz = freq / 1000000; + const u32 f_MHz = freq / 1000000; /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ - const unsigned long v2f_big = v2 * f_MHz; + const u32 v2f_big = v2 * f_MHz; /* Range: 2^1 < v2f < 2^16 MHz V^2 */ - const unsigned long v2f = v2f_big / 1000; + const u32 v2f = v2f_big / 1000; /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. * Must be < 2^42 to avoid overflowing the return value. */ @@ -425,17 +399,16 @@ static inline unsigned long kbase_scale_dynamic_power(const unsigned long c, * * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) */ -unsigned long kbase_scale_static_power(const unsigned long c, - const unsigned long voltage) +u32 kbase_scale_static_power(const u32 c, const u32 voltage) { /* Range: 2^8 < v2 < 2^16 m(V^2) */ - const unsigned long v2 = (voltage * voltage) / 1000; + const u32 v2 = (voltage * voltage) / 1000; /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ - const unsigned long v3_big = v2 * voltage; + const u32 v3_big = v2 * voltage; /* Range: 2^7 < v3 < 2^19 m(V^3) */ - const unsigned long v3 = v3_big / 1000; + const u32 v3 = v3_big / 1000; /* * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. @@ -457,6 +430,29 @@ static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) return kbdev->ipa.fallback_model; } +static u32 get_static_power_locked(struct kbase_device *kbdev, + struct kbase_ipa_model *model, + unsigned long voltage) +{ + u32 power = 0; + int err; + u32 power_coeff; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (!model->ops->get_static_coeff) + model = kbdev->ipa.fallback_model; + + if (model->ops->get_static_coeff) { + err = model->ops->get_static_coeff(model, &power_coeff); + if (!err) + power = kbase_scale_static_power(power_coeff, + (u32) voltage); + } + + return power; +} + #ifdef CONFIG_MALI_PWRSOFT_765 static unsigned long kbase_get_static_power(struct devfreq *df, unsigned long voltage) @@ -465,7 +461,7 @@ static unsigned long kbase_get_static_power(unsigned long voltage) #endif { struct kbase_ipa_model *model; - unsigned long power_coeff = 0, power = 0; + u32 power = 0; #ifdef CONFIG_MALI_PWRSOFT_765 struct kbase_device *kbdev = dev_get_drvdata(&df->dev); #else @@ -475,15 +471,7 @@ static unsigned long kbase_get_static_power(unsigned long voltage) mutex_lock(&kbdev->ipa.lock); model = get_current_model(kbdev); - - if (model) { - power_coeff = model->ops->get_static_power(model); - power = kbase_scale_static_power(power_coeff, voltage); - } else { - dev_err(kbdev->dev, "%s: No current IPA model set", __func__); - } - - kbdev->ipa.last_static_power_coeff = power_coeff; + power = get_static_power_locked(kbdev, model, voltage); mutex_unlock(&kbdev->ipa.lock); @@ -504,7 +492,8 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, #endif { struct kbase_ipa_model *model; - unsigned long power_coeff = 0, power = 0; + u32 power_coeff = 0, power = 0; + int err = 0; #ifdef CONFIG_MALI_PWRSOFT_765 struct kbase_device *kbdev = dev_get_drvdata(&df->dev); #else @@ -513,16 +502,16 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, mutex_lock(&kbdev->ipa.lock); - model = get_current_model(kbdev); + model = kbdev->ipa.fallback_model; - if (model) { - power_coeff = model->ops->get_dynamic_power(model); - power = kbase_scale_dynamic_power(power_coeff, freq, voltage); - } else { - dev_err(kbdev->dev, "%s: No current IPA model set", __func__); - } + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); - kbdev->ipa.last_model_dyn_power_coeff = power_coeff; + if (!err) + power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + else + dev_err_ratelimited(kbdev->dev, + "Model %s returned error code %d\n", + model->ops->name, err); mutex_unlock(&kbdev->ipa.lock); @@ -533,90 +522,60 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, return power; } -unsigned long kbase_power_to_state(struct devfreq *df, u32 target_power) +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage) { + struct kbase_ipa_model *model; + u32 power_coeff = 0; + int err = 0; struct kbase_device *kbdev = dev_get_drvdata(&df->dev); - struct device *dev = df->dev.parent; - unsigned long i, state = -1; - unsigned long dyn_coeff, static_coeff; mutex_lock(&kbdev->ipa.lock); - dyn_coeff = kbdev->ipa.last_model_dyn_power_coeff; - static_coeff = kbdev->ipa.last_static_power_coeff; + model = get_current_model(kbdev); - mutex_unlock(&kbdev->ipa.lock); + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); - /* OPPs are sorted from highest frequency to lowest */ - for (i = 0; i < df->profile->max_state - 1; i++) { - struct dev_pm_opp *opp; - unsigned int freq; - unsigned long dyn_power, static_power, voltage; + /* If we switch to protected model between get_current_model() and + * get_dynamic_coeff(), counter reading could fail. If that happens + * (unlikely, but possible), revert to the fallback model. */ + if (err && model != kbdev->ipa.fallback_model) { + model = kbdev->ipa.fallback_model; + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + } - freq = df->profile->freq_table[i]; /* Hz */ + if (err) + goto exit_unlock; - rcu_read_lock(); - opp = dev_pm_opp_find_freq_exact(dev, freq, true); - /* Allow unavailable frequencies too in case we can enable a - * higher one. */ - if (PTR_ERR(opp) == -ERANGE) - opp = dev_pm_opp_find_freq_exact(dev, freq, false); + *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); - if (IS_ERR(opp)) { - rcu_read_unlock(); - return PTR_ERR(opp); - } + if (model->ops->do_utilization_scaling_in_framework) { + struct devfreq_dev_status *status = &df->last_status; + unsigned long total_time = max(status->total_time, 1ul); + u64 busy_time = min(status->busy_time, total_time); - voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */ - rcu_read_unlock(); + *power = ((u64) *power * (u64) busy_time) / total_time; + } - dyn_power = kbase_scale_dynamic_power(dyn_coeff, freq, voltage); - static_power = kbase_scale_static_power(static_coeff, voltage); + *power += get_static_power_locked(kbdev, model, voltage); - if (target_power >= dyn_power + static_power) - break; - } - state = i; +exit_unlock: + mutex_unlock(&kbdev->ipa.lock); - return state; + return err; } -KBASE_EXPORT_TEST_API(kbase_power_to_state); +KBASE_EXPORT_TEST_API(kbase_get_real_power); #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -struct devfreq_cooling_ops power_model_ops = { +struct devfreq_cooling_ops kbase_ipa_power_model_ops = { #else -struct devfreq_cooling_power power_model_ops = { +struct devfreq_cooling_power kbase_ipa_power_model_ops = { #endif .get_static_power = &kbase_get_static_power, .get_dynamic_power = &kbase_get_dynamic_power, #ifdef CONFIG_MALI_PWRSOFT_765 - .power2state = &kbase_power_to_state, + .get_real_power = &kbase_get_real_power, #endif }; - -unsigned long kbase_ipa_dynamic_power(struct kbase_device *kbdev, - unsigned long freq, - unsigned long voltage) -{ -#ifdef CONFIG_MALI_PWRSOFT_765 - struct devfreq *df = kbdev->devfreq; - - return kbase_get_dynamic_power(df, freq, voltage); -#else - return kbase_get_dynamic_power(freq, voltage); -#endif -} -KBASE_EXPORT_TEST_API(kbase_ipa_dynamic_power); - -unsigned long kbase_ipa_static_power(struct kbase_device *kbdev, - unsigned long voltage) -{ -#ifdef CONFIG_MALI_PWRSOFT_765 - struct devfreq *df = kbdev->devfreq; - - return kbase_get_static_power(df, voltage); -#else - return kbase_get_static_power(voltage); -#endif -} -KBASE_EXPORT_TEST_API(kbase_ipa_static_power); +KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); diff --git a/mali_kbase/ipa/mali_kbase_ipa.h b/mali_kbase/ipa/mali_kbase_ipa.h index 540d34a..b2d3db1 100644 --- a/mali_kbase/ipa/mali_kbase_ipa.h +++ b/mali_kbase/ipa/mali_kbase_ipa.h @@ -18,29 +18,51 @@ #ifndef _KBASE_IPA_H_ #define _KBASE_IPA_H_ +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +struct devfreq; + struct kbase_ipa_model { struct list_head link; struct kbase_device *kbdev; void *model_data; struct kbase_ipa_model_ops *ops; struct list_head params; + bool missing_dt_node_warning; }; -int kbase_ipa_model_add_param_u32_def(struct kbase_ipa_model *model, - const char *name, u32 *addr, - bool has_default, u32 default_value); - -int kbase_ipa_model_add_param_s32_array(struct kbase_ipa_model *model, - const char *name, s32 *addr, - size_t num_elems); +/** + * kbase_ipa_model_add_param_s32 - Add an integer model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @num_elems: number of elements (1 if not an array) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required); +/** + * kbase_ipa_model_add_param_string - Add a string model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @size: size, in bytes, of the value storage (so the maximum string + * length is size - 1) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, const char *name, char *addr, - size_t len); - -#define kbase_ipa_model_add_param_u32(MODEL, NAME, ADDR) \ - kbase_ipa_model_add_param_u32_def((MODEL), (NAME), (ADDR), \ - false, 0) + size_t size, bool dt_required); struct kbase_ipa_model_ops { char *name; @@ -56,64 +78,71 @@ struct kbase_ipa_model_ops { * recalculated. */ int (*recalculate)(struct kbase_ipa_model *model); void (*term)(struct kbase_ipa_model *model); - /* get_dynamic_power() - return a coefficient with units pW/(Hz V^2), - * which is scaled by the IPA framework according to the current OPP's - * frequency and voltage. */ - unsigned long (*get_dynamic_power)(struct kbase_ipa_model *model); - /* get_static_power() - return a coefficient with units uW/(V^3), - * which is scaled by the IPA framework according to the current OPP's - * voltage. */ - unsigned long (*get_static_power)(struct kbase_ipa_model *model); + /* + * get_dynamic_coeff() - calculate dynamic power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * @current_freq: frequency the GPU has been running at for the + * previous sampling period. + * + * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which + * is then scaled by the IPA framework according to the current OPP's + * frequency and voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq); + /* + * get_static_coeff() - calculate static power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * + * Calculate a static power coefficient, with units uW/(V^3), which is + * scaled by the IPA framework according to the current OPP's voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); + /* If false, the model's get_dynamic_coeff() method accounts for how + * long the GPU was active over the sample period. If true, the + * framework will scale the calculated power according to the + * utilization stats recorded by devfreq in get_real_power(). */ + bool do_utilization_scaling_in_framework; }; /* Models can be registered only in the platform's platform_init_func call */ int kbase_ipa_model_ops_register(struct kbase_device *kbdev, struct kbase_ipa_model_ops *new_model_ops); +struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, + const char *name); int kbase_ipa_init(struct kbase_device *kbdev); void kbase_ipa_term(struct kbase_device *kbdev); void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops); +void kbase_ipa_term_model(struct kbase_ipa_model *model); extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; -/** - * kbase_ipa_dynamic_power() - Calculate dynamic power component - * @kbdev: Pointer to kbase device. - * @freq: Frequency, in Hz. - * @voltage: Voltage, in mV. - * - * Return: Dynamic power consumption, in mW. - */ -unsigned long kbase_ipa_dynamic_power(struct kbase_device *kbdev, - unsigned long freq, - unsigned long voltage); - -/** - * kbase_ipa_static_power() - Calculate static power component - * @kbdev: Pointer to kbase device. - * @voltage: Voltage, in mV. - * - * Return: Static power consumption, in mW. - */ -unsigned long kbase_ipa_static_power(struct kbase_device *kbdev, - unsigned long voltage); - -/** - * kbase_power_to_state() - Find the OPP which consumes target_power - * @df: Pointer to devfreq device. - * @target_power: Maximum power consumption, in mW. - * - * Return: The index of the most performant OPP whose power consumption is less - * than target_power. - */ -unsigned long kbase_power_to_state(struct devfreq *df, u32 target_power); #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -extern struct devfreq_cooling_ops power_model_ops; +extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; #else -extern struct devfreq_cooling_power power_model_ops; +extern struct devfreq_cooling_power kbase_ipa_power_model_ops; #endif +#else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) +{ } + +static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) +{ } + +#endif /* (defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + #endif diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c index 78bdcef..eafc140 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c +++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c @@ -31,7 +31,6 @@ struct kbase_ipa_model_param { char *name; union { void *voidp; - u32 *u32p; s32 *s32p; char *str; } addr; @@ -46,10 +45,7 @@ static int param_int_get(void *data, u64 *val) struct kbase_ipa_model_param *param = data; mutex_lock(¶m->model->kbdev->ipa.lock); - if (param->type == PARAM_TYPE_S32) - *(s64 *) val = *param->addr.s32p; - else - *val = *param->addr.u32p; + *(s64 *) val = *param->addr.s32p; mutex_unlock(¶m->model->kbdev->ipa.lock); return 0; @@ -62,28 +58,17 @@ static int param_int_set(void *data, u64 val) s64 sval = (s64) val; int err = 0; - switch (param->type) { - case PARAM_TYPE_U32: - if (sval < 0 || val > U32_MAX) - return -ERANGE; - break; - case PARAM_TYPE_S32: - if (sval < S32_MIN || sval > S32_MAX) - return -ERANGE; - break; - default: - return -EINVAL; - } + if (sval < S32_MIN || sval > S32_MAX) + return -ERANGE; mutex_lock(¶m->model->kbdev->ipa.lock); - *param->addr.u32p = val; + *param->addr.s32p = val; err = kbase_ipa_model_recalculate(model); mutex_unlock(¶m->model->kbdev->ipa.lock); return err; } -DEFINE_DEBUGFS_ATTRIBUTE(fops_u32, param_int_get, param_int_set, "%llu\n"); DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); static ssize_t param_string_get(struct file *file, char __user *user_buf, @@ -206,9 +191,6 @@ static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) case PARAM_TYPE_S32: fops = &fops_s32; break; - case PARAM_TYPE_U32: - fops = &fops_u32; - break; case PARAM_TYPE_STRING: fops = &fops_string; break; diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h index d7fc908..ec06e20 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h +++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h @@ -20,7 +20,6 @@ enum kbase_ipa_model_param_type { PARAM_TYPE_S32 = 1, - PARAM_TYPE_U32, PARAM_TYPE_STRING, }; diff --git a/mali_kbase/ipa/mali_kbase_ipa_simple.c b/mali_kbase/ipa/mali_kbase_ipa_simple.c index f8d46e1..deca1bc 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_simple.c +++ b/mali_kbase/ipa/mali_kbase_ipa_simple.c @@ -85,7 +85,7 @@ static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) return clamp(res_unclamped, (s64) 0, (s64) 10000000); } -static unsigned long model_static_power(struct kbase_ipa_model *model) +static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) { #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) unsigned long temp; @@ -113,15 +113,21 @@ static unsigned long model_static_power(struct kbase_ipa_model *model) temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, temp); - return (model_data->static_coefficient * temp_scaling_factor) / 1000000; + *coeffp = model_data->static_coefficient * temp_scaling_factor; + *coeffp /= 1000000; + + return 0; } -static unsigned long model_dynamic_power(struct kbase_ipa_model *model) +static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq) { struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *) model->model_data; - return model_data->dynamic_coefficient; + *coeffp = model_data->dynamic_coefficient; + + return 0; } static int add_params(struct kbase_ipa_model *model) @@ -130,24 +136,26 @@ static int add_params(struct kbase_ipa_model *model) struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *)model->model_data; - err = kbase_ipa_model_add_param_u32(model, "static-coefficient", - &model_data->static_coefficient); + err = kbase_ipa_model_add_param_s32(model, "static-coefficient", + &model_data->static_coefficient, + 1, true); if (err) goto end; - err = kbase_ipa_model_add_param_u32(model, "dynamic-coefficient", - &model_data->dynamic_coefficient); + err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", + &model_data->dynamic_coefficient, + 1, true); if (err) goto end; - err = kbase_ipa_model_add_param_s32_array(model, "ts", - model_data->ts, 4); + err = kbase_ipa_model_add_param_s32(model, "ts", + model_data->ts, 4, true); if (err) goto end; err = kbase_ipa_model_add_param_string(model, "thermal-zone", model_data->tz_name, - sizeof(model_data->tz_name)); + sizeof(model_data->tz_name), true); end: return err; @@ -175,13 +183,18 @@ static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *)model->model_data; - model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name); - if (IS_ERR(model_data->gpu_tz)) { - pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", - PTR_ERR(model_data->gpu_tz), - model_data->tz_name); + if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { model_data->gpu_tz = NULL; - return -EPROBE_DEFER; + } else { + model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name); + + if (IS_ERR(model_data->gpu_tz)) { + pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", + PTR_ERR(model_data->gpu_tz), + model_data->tz_name); + model_data->gpu_tz = NULL; + return -EPROBE_DEFER; + } } return 0; @@ -200,6 +213,7 @@ struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { .init = &kbase_simple_power_model_init, .recalculate = &kbase_simple_power_model_recalculate, .term = &kbase_simple_power_model_term, - .get_dynamic_power = &model_dynamic_power, - .get_static_power = &model_static_power, + .get_dynamic_coeff = &model_dynamic_coeff, + .get_static_coeff = &model_static_coeff, + .do_utilization_scaling_in_framework = true, }; diff --git a/mali_kbase/mali_base_hwconfig_features.h b/mali_kbase/mali_base_hwconfig_features.h index 8ee19fc..6be0a33 100644 --- a/mali_kbase/mali_base_hwconfig_features.h +++ b/mali_kbase/mali_base_hwconfig_features.h @@ -51,6 +51,7 @@ enum base_hw_feature { BASE_HW_FEATURE_PROTECTED_MODE, BASE_HW_FEATURE_COHERENCY_REG, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_AARCH64_MMU, BASE_HW_FEATURE_END }; @@ -247,4 +248,64 @@ static const enum base_hw_feature base_hw_features_tSIx[] = { }; +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_feature base_hw_features_tKAx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_feature base_hw_features_tTRx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + #endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/mali_kbase/mali_base_hwconfig_issues.h b/mali_kbase/mali_base_hwconfig_issues.h index 53f3804..6d7e5c5 100644 --- a/mali_kbase/mali_base_hwconfig_issues.h +++ b/mali_kbase/mali_base_hwconfig_issues.h @@ -956,9 +956,11 @@ static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_END }; @@ -975,6 +977,7 @@ static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_END }; @@ -1001,6 +1004,15 @@ static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { BASE_HW_ISSUE_END }; +static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + static const enum base_hw_issue base_hw_issues_model_tHEx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, @@ -1041,4 +1053,46 @@ static const enum base_hw_issue base_hw_issues_model_tSIx[] = { +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_issue base_hw_issues_tKAx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_issue base_hw_issues_model_tKAx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_issue base_hw_issues_model_tTRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + #endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/mali_kbase/mali_base_kernel.h b/mali_kbase/mali_base_kernel.h index 0c80e03..d5c8cbc 100644 --- a/mali_kbase/mali_base_kernel.h +++ b/mali_kbase/mali_base_kernel.h @@ -118,33 +118,44 @@ typedef union kbase_pointer { */ /** - * @brief Memory allocation, access/hint flags + * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. * * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator * in order to determine the best cache policy. Some combinations are - * of course invalid (eg @c MEM_PROT_CPU_WR | @c MEM_HINT_CPU_RD), - * which defines a @a write-only region on the CPU side, which is + * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), + * which defines a write-only region on the CPU side, which is * heavily read by the CPU... * Other flags are only meaningful to a particular allocator. * More flags can be added to this list, as long as they don't clash - * (see ::BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). + * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). */ typedef u32 base_mem_alloc_flags; -/** - * @brief Memory allocation, access/hint flags - * - * See ::base_mem_alloc_flags. +/* Memory allocation, access/hint flags. * + * See base_mem_alloc_flags. */ -enum { + /* IN */ - BASE_MEM_PROT_CPU_RD = (1U << 0), /**< Read access CPU side */ - BASE_MEM_PROT_CPU_WR = (1U << 1), /**< Write access CPU side */ - BASE_MEM_PROT_GPU_RD = (1U << 2), /**< Read access GPU side */ - BASE_MEM_PROT_GPU_WR = (1U << 3), /**< Write access GPU side */ - BASE_MEM_PROT_GPU_EX = (1U << 4), /**< Execute allowed on the GPU - side */ +/* Read access CPU side + */ +#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) + +/* Write access CPU side + */ +#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) + +/* Read access GPU side + */ +#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) + +/* Write access GPU side + */ +#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) + +/* Execute allowed on the GPU side + */ +#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) /* BASE_MEM_HINT flags have been removed, but their values are reserved * for backwards compatibility with older user-space drivers. The values @@ -157,54 +168,66 @@ enum { * RESERVED: (1U << 8) */ - BASE_MEM_GROW_ON_GPF = (1U << 9), /**< Grow backing store on GPU - Page Fault */ +/* Grow backing store on GPU Page Fault + */ +#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) - BASE_MEM_COHERENT_SYSTEM = (1U << 10), /**< Page coherence Outer - shareable, if available */ - BASE_MEM_COHERENT_LOCAL = (1U << 11), /**< Page coherence Inner - shareable */ - BASE_MEM_CACHED_CPU = (1U << 12), /**< Should be cached on the - CPU */ +/* Page coherence Outer shareable, if available + */ +#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) + +/* Page coherence Inner shareable + */ +#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) + +/* Should be cached on the CPU + */ +#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) /* IN/OUT */ - BASE_MEM_SAME_VA = (1U << 13), /**< Must have same VA on both the GPU - and the CPU */ +/* Must have same VA on both the GPU and the CPU + */ +#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) + /* OUT */ - BASE_MEM_NEED_MMAP = (1U << 14), /**< Must call mmap to acquire a GPU - address for the alloc */ +/* Must call mmap to acquire a GPU address for the alloc + */ +#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) + /* IN */ - BASE_MEM_COHERENT_SYSTEM_REQUIRED = (1U << 15), /**< Page coherence - Outer shareable, required. */ - BASE_MEM_SECURE = (1U << 16), /**< Secure memory */ - BASE_MEM_DONT_NEED = (1U << 17), /**< Not needed physical - memory */ - BASE_MEM_IMPORT_SHARED = (1U << 18), /**< Must use shared CPU/GPU zone - (SAME_VA zone) but doesn't - require the addresses to - be the same */ -}; +/* Page coherence Outer shareable, required. + */ +#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) -/** - * @brief Number of bits used as flags for base memory management +/* Secure memory + */ +#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) + +/* Not needed physical memory + */ +#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) + +/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the + * addresses to be the same + */ +#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) + +/* Number of bits used as flags for base memory management * - * Must be kept in sync with the ::base_mem_alloc_flags flags + * Must be kept in sync with the base_mem_alloc_flags flags */ #define BASE_MEM_FLAGS_NR_BITS 19 -/** - * A mask for all output bits, excluding IN/OUT bits. - */ +/* A mask for all output bits, excluding IN/OUT bits. + */ #define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP -/** - * A mask for all input bits, including IN/OUT bits. - */ +/* A mask for all input bits, including IN/OUT bits. + */ #define BASE_MEM_FLAGS_INPUT_MASK \ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) -/** - * A mask for all the flags which are modifiable via the base_mem_set_flags +/* A mask for all the flags which are modifiable via the base_mem_set_flags * interface. */ #define BASE_MEM_FLAGS_MODIFIABLE \ @@ -294,7 +317,6 @@ struct base_mem_import_user_buffer { */ typedef enum base_backing_threshold_status { BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ - BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE = -1, /**< Not a growable tmem object */ BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ } base_backing_threshold_status; @@ -1429,9 +1451,9 @@ struct mali_base_gpu_core_props { /** * Status of the GPU release. - * No defined values, but starts at 0 and increases by one for each release - * status (alpha, beta, EAC, etc.). - * 4 bit values (0-15). + * No defined values, but starts at 0 and increases by one for each + * release status (alpha, beta, EAC, etc.). + * 4 bit values (0-15). */ u16 version_status; @@ -1450,8 +1472,11 @@ struct mali_base_gpu_core_props { u16 padding; /** - * @usecase GPU clock speed is not specified in the Midgard Architecture, but is - * <b>necessary for OpenCL's clGetDeviceInfo() function</b>. + * This property is deprecated since it has not contained the real current + * value of GPU clock speed. It is kept here only for backwards compatibility. + * For the new ioctl interface, it is ignored and is treated as a padding + * to keep the structure of the same size and retain the placement of its + * members. */ u32 gpu_speed_mhz; @@ -1459,6 +1484,7 @@ struct mali_base_gpu_core_props { * @usecase GPU clock max/min speed is required for computing best/worst case * in tasks as job scheduling ant irq_throttling. (It is not specified in the * Midgard Architecture). + * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. */ u32 gpu_freq_khz_max; u32 gpu_freq_khz_min; diff --git a/mali_kbase/mali_kbase.h b/mali_kbase/mali_kbase.h index 2289f80..baefe8a 100644 --- a/mali_kbase/mali_kbase.h +++ b/mali_kbase/mali_kbase.h @@ -66,9 +66,7 @@ #include "mali_kbase_jm.h" #include "mali_kbase_vinstr.h" -#ifdef CONFIG_DEVFREQ_THERMAL #include "ipa/mali_kbase_ipa.h" -#endif #ifdef CONFIG_GPU_TRACEPOINTS #include <trace/events/gpu.h> @@ -106,22 +104,27 @@ void kbase_release_device(struct kbase_device *kbdev); void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); -u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control); - struct kbase_context * kbase_create_context(struct kbase_device *kbdev, bool is_compat); void kbase_destroy_context(struct kbase_context *kctx); int kbase_jd_init(struct kbase_context *kctx); void kbase_jd_exit(struct kbase_context *kctx); -#ifdef BASE_LEGACY_UK6_SUPPORT -int kbase_jd_submit(struct kbase_context *kctx, - const struct kbase_uk_job_submit *submit_data, - int uk6_atom); -#else + +/** + * kbase_jd_submit - Submit atoms to the job dispatcher + * + * @kctx: The kbase context to submit to + * @user_addr: The address in user space of the struct base_jd_atom_v2 array + * @nr_atoms: The number of atoms in the array + * @stride: sizeof(struct base_jd_atom_v2) + * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) + * + * Return: 0 on success or error code + */ int kbase_jd_submit(struct kbase_context *kctx, - const struct kbase_uk_job_submit *submit_data); -#endif + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom); /** * kbase_jd_done_worker - Handle a job completion @@ -195,8 +198,10 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom); void kbase_finish_soft_job(struct kbase_jd_atom *katom); void kbase_cancel_soft_job(struct kbase_jd_atom *katom); void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); -void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom); void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); +#endif int kbase_soft_event_update(struct kbase_context *kctx, u64 event, unsigned char new_status); diff --git a/mali_kbase/mali_kbase_10969_workaround.c b/mali_kbase/mali_kbase_10969_workaround.c index 933aa28..fde0f8f 100644 --- a/mali_kbase/mali_kbase_10969_workaround.c +++ b/mali_kbase/mali_kbase_10969_workaround.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2015, 2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_config_defaults.h b/mali_kbase/mali_kbase_config_defaults.h index f6eeac8..69079e7 100644 --- a/mali_kbase/mali_kbase_config_defaults.h +++ b/mali_kbase/mali_kbase_config_defaults.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -170,7 +170,7 @@ enum { * Default timeout for some software jobs, after which the software event wait * jobs will be cancelled. */ -#define DEFAULT_JS_SOFT_JOB_TIMEOUT ((u32)3000) /* 3s */ +#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ /* * Default minimum number of scheduling ticks before the GPU is reset to clear a diff --git a/mali_kbase/mali_kbase_context.c b/mali_kbase/mali_kbase_context.c index 1401380..4e802bc 100644 --- a/mali_kbase/mali_kbase_context.c +++ b/mali_kbase/mali_kbase_context.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,6 +24,8 @@ #include <mali_kbase.h> #include <mali_midg_regmap.h> #include <mali_kbase_mem_linux.h> +#include <mali_kbase_dma_fence.h> +#include <mali_kbase_ctx_sched.h> /** * kbase_create_context() - Create a kernel base context. @@ -53,6 +55,7 @@ kbase_create_context(struct kbase_device *kbdev, bool is_compat) kctx->kbdev = kbdev; kctx->as_nr = KBASEP_AS_NR_INVALID; + atomic_set(&kctx->refcount, 0); if (is_compat) kbase_ctx_flag_set(kctx, KCTX_COMPAT); #ifdef CONFIG_MALI_TRACE_TIMELINE @@ -210,6 +213,7 @@ void kbase_destroy_context(struct kbase_context *kctx) struct kbase_device *kbdev; int pages; unsigned long pending_regions_to_clean; + unsigned long flags; KBASE_DEBUG_ASSERT(NULL != kctx); @@ -277,6 +281,12 @@ void kbase_destroy_context(struct kbase_context *kctx) kbase_dma_fence_term(kctx); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + kbase_mmu_term(kctx); pages = atomic_read(&kctx->used_pages); diff --git a/mali_kbase/mali_kbase_core_linux.c b/mali_kbase/mali_kbase_core_linux.c index e335c0d..c076300 100644 --- a/mali_kbase/mali_kbase_core_linux.c +++ b/mali_kbase/mali_kbase_core_linux.c @@ -22,6 +22,7 @@ #include <mali_kbase_gator.h> #include <mali_kbase_mem_linux.h> #ifdef CONFIG_MALI_DEVFREQ +#include <linux/devfreq.h> #include <backend/gpu/mali_kbase_devfreq.h> #ifdef CONFIG_DEVFREQ_THERMAL #include <ipa/mali_kbase_ipa_debugfs.h> @@ -40,7 +41,9 @@ #include "mali_kbase_regs_history_debugfs.h" #include <mali_kbase_hwaccess_backend.h> #include <mali_kbase_hwaccess_jm.h> +#include <mali_kbase_ctx_sched.h> #include <backend/gpu/mali_kbase_device_internal.h> +#include "mali_kbase_ioctl.h" #ifdef CONFIG_KDS #include <linux/kds.h> @@ -55,6 +58,7 @@ #include <linux/errno.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/of_platform.h> #include <linux/miscdevice.h> #include <linux/list.h> #include <linux/semaphore.h> @@ -73,12 +77,9 @@ #ifdef CONFIG_MALI_PLATFORM_FAKE #include <platform/mali_kbase_platform_fake.h> #endif /*CONFIG_MALI_PLATFORM_FAKE */ -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) #include <mali_kbase_sync.h> -#endif /* CONFIG_SYNC */ -#ifdef CONFIG_PM_DEVFREQ -#include <linux/devfreq.h> -#endif /* CONFIG_PM_DEVFREQ */ +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ #include <linux/clk.h> #include <linux/delay.h> @@ -116,60 +117,8 @@ static inline void __compile_time_asserts(void) CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); } -static void kbase_create_timeline_objects(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned int lpu_id; - unsigned int as_nr; - struct kbasep_kctx_list_element *element; - - /* Create LPU objects. */ - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - u32 *lpu = - &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; - KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); - } - - /* Create Address Space objects. */ - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); - - /* Create GPU object and make it retain all LPUs and address spaces. */ - KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( - kbdev, - kbdev->gpu_props.props.raw_props.gpu_id, - kbdev->gpu_props.num_cores); - - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - void *lpu = - &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; - KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); - } - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( - &kbdev->as[as_nr], - kbdev); - - /* Create object for each known context. */ - mutex_lock(&kbdev->kctx_list_lock); - list_for_each_entry(element, &kbdev->kctx_list, link) { - KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( - element->kctx, - (u32)(element->kctx->id), - (u32)(element->kctx->tgid)); - } - /* Before releasing the lock, reset body stream buffers. - * This will prevent context creation message to be directed to both - * summary and body stream. */ - kbase_tlstream_reset_body_streams(); - mutex_unlock(&kbdev->kctx_list_lock); - /* Static object are placed into summary packet that needs to be - * transmitted first. Flush all streams to make it available to - * user space. */ - kbase_tlstream_flush_streams(); -} - -static void kbase_api_handshake(struct uku_version_check_args *version) +static int kbase_api_handshake(struct kbase_context *kctx, + struct kbase_ioctl_version_check *version) { switch (version->major) { #ifdef BASE_LEGACY_UK6_SUPPORT @@ -218,6 +167,11 @@ static void kbase_api_handshake(struct uku_version_check_args *version) version->minor = BASE_UK_VERSION_MINOR; break; } + + /* save the proposed version number for later use */ + kctx->api_version = KBASE_API_VERSION(version->major, version->minor); + + return 0; } /** @@ -262,7 +216,9 @@ enum { inited_registers_map = (1u << 17), inited_io_history = (1u << 18), inited_power_control = (1u << 19), - inited_buslogger = (1u << 20) + inited_buslogger = (1u << 20), + inited_protected = (1u << 21), + inited_ctx_sched = (1u << 22) }; @@ -281,7 +237,18 @@ void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); #endif /* CONFIG_MALI_DEBUG */ -static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 args_size) +/** + * kbase_legacy_dispatch - UKK dispatch function + * + * This is the dispatch function for the legacy UKK ioctl interface. No new + * ioctls should be added to this function, see kbase_ioctl instead. + * + * @kctx: The kernel context structure + * @args: Pointer to the data structure passed from/to user space + * @args_size: Size of the data structure + */ +static int kbase_legacy_dispatch(struct kbase_context *kctx, + void * const args, u32 args_size) { struct kbase_device *kbdev; union uk_header *ukh = args; @@ -301,16 +268,20 @@ static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 arg if (UKP_FUNC_ID_CHECK_VERSION == id) { struct uku_version_check_args *version_check; + struct kbase_ioctl_version_check version; if (args_size != sizeof(struct uku_version_check_args)) { ukh->ret = MALI_ERROR_FUNCTION_FAILED; return 0; } version_check = (struct uku_version_check_args *)args; - kbase_api_handshake(version_check); - /* save the proposed version number for later use */ - kctx->api_version = KBASE_API_VERSION(version_check->major, - version_check->minor); + version.minor = version_check->minor; + version.major = version_check->major; + + kbase_api_handshake(kctx, &version); + + version_check->minor = version.minor; + version_check->major = version.major; ukh->ret = MALI_ERROR_NONE; return 0; } @@ -409,6 +380,7 @@ static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 arg (enum base_mem_import_type) mem_import->type, phandle, + 0, &mem_import->gpu_va, &mem_import->va_pages, &mem_import->flags)) { @@ -470,21 +442,26 @@ copy_failed: case KBASE_FUNC_MEM_COMMIT: { struct kbase_uk_mem_commit *commit = args; + int ret; if (sizeof(*commit) != args_size) goto bad_size; - if (commit->gpu_addr & ~PAGE_MASK) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_COMMIT: commit->gpu_addr: passed parameter is invalid"); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } + ret = kbase_mem_commit(kctx, commit->gpu_addr, + commit->pages); - if (kbase_mem_commit(kctx, commit->gpu_addr, - commit->pages, - (base_backing_threshold_status *) - &commit->result_subcode) != 0) - ukh->ret = MALI_ERROR_FUNCTION_FAILED; + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; + + if (ret == 0) { + ukh->ret = MALI_ERROR_NONE; + commit->result_subcode = + BASE_BACKING_THRESHOLD_OK; + } else if (ret == -ENOMEM) { + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_OOM; + } break; } @@ -496,19 +473,6 @@ copy_failed: if (sizeof(*query) != args_size) goto bad_size; - if (query->gpu_addr & ~PAGE_MASK) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_QUERY: query->gpu_addr: passed parameter is invalid"); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } - if (query->query != KBASE_MEM_QUERY_COMMIT_SIZE && - query->query != KBASE_MEM_QUERY_VA_SIZE && - query->query != KBASE_MEM_QUERY_FLAGS) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_QUERY: query->query = %lld unknown", (unsigned long long)query->query); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } - if (kbase_mem_query(kctx, query->gpu_addr, query->query, &query->value) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; @@ -525,12 +489,6 @@ copy_failed: if (sizeof(*fc) != args_size) goto bad_size; - if ((fc->gpu_va & ~PAGE_MASK) && (fc->gpu_va >= PAGE_SIZE)) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_FLAGS_CHANGE: mem->gpu_va: passed parameter is invalid"); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } - if (kbase_mem_flags_change(kctx, fc->gpu_va, fc->flags, fc->mask) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; @@ -544,12 +502,6 @@ copy_failed: if (sizeof(*mem) != args_size) goto bad_size; - if ((mem->gpu_addr & ~PAGE_MASK) && (mem->gpu_addr >= PAGE_SIZE)) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_FREE: mem->gpu_addr: passed parameter is invalid"); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } - if (kbase_mem_free(kctx, mem->gpu_addr) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; break; @@ -562,11 +514,10 @@ copy_failed: if (sizeof(*job) != args_size) goto bad_size; -#ifdef BASE_LEGACY_UK6_SUPPORT - if (kbase_jd_submit(kctx, job, 0) != 0) -#else - if (kbase_jd_submit(kctx, job) != 0) -#endif /* BASE_LEGACY_UK6_SUPPORT */ + if (kbase_jd_submit(kctx, job->addr.value, + job->nr_atoms, + job->stride, + false) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; break; } @@ -579,7 +530,10 @@ copy_failed: if (sizeof(*job) != args_size) goto bad_size; - if (kbase_jd_submit(kctx, job, 1) != 0) + if (kbase_jd_submit(kctx, job->addr.value, + job->nr_atoms, + job->stride, + true) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; break; } @@ -592,14 +546,8 @@ copy_failed: if (sizeof(*sn) != args_size) goto bad_size; - if (sn->sset.basep_sset.mem_handle.basep.handle & ~PAGE_MASK) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_SYNC: sn->sset.basep_sset.mem_handle: passed parameter is invalid"); - ukh->ret = MALI_ERROR_FUNCTION_FAILED; - break; - } - #ifndef CONFIG_MALI_COH_USER - if (kbase_sync_now(kctx, &sn->sset) != 0) + if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; #endif break; @@ -693,7 +641,7 @@ copy_failed: goto bad_size; if (find->gpu_addr & ~PAGE_MASK) { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); + dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); goto out_bad; } @@ -728,7 +676,7 @@ copy_failed: case KBASE_FUNC_STREAM_CREATE: { -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; if (sizeof(*screate) != args_size) @@ -740,28 +688,29 @@ copy_failed: break; } - if (kbase_stream_create(screate->name, &screate->fd) != 0) + if (kbase_sync_fence_stream_create(screate->name, + &screate->fd) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; else ukh->ret = MALI_ERROR_NONE; -#else /* CONFIG_SYNC */ +#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -#endif /* CONFIG_SYNC */ +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ break; } case KBASE_FUNC_FENCE_VALIDATE: { -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; if (sizeof(*fence_validate) != args_size) goto bad_size; - if (kbase_fence_validate(fence_validate->fd) != 0) + if (kbase_sync_fence_validate(fence_validate->fd) != 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; else ukh->ret = MALI_ERROR_NONE; -#endif /* CONFIG_SYNC */ +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ break; } @@ -818,7 +767,7 @@ copy_failed: #ifdef BASE_LEGACY_UK8_SUPPORT case KBASE_FUNC_KEEP_GPU_POWERED: { - dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n"); + dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n"); ukh->ret = MALI_ERROR_FUNCTION_FAILED; break; } @@ -834,7 +783,8 @@ copy_failed: goto bad_size; for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) - controls->profiling_controls[i] = kbase_get_profiling_control(kbdev, i); + controls->profiling_controls[i] = + kbdev->kbase_profiling_controls[i]; break; } @@ -890,7 +840,6 @@ copy_failed: if (kbasep_mem_profile_debugfs_insert(kctx, buf, add_data->len)) { ukh->ret = MALI_ERROR_FUNCTION_FAILED; - kfree(buf); goto out_bad; } @@ -914,20 +863,17 @@ copy_failed: { struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire = args; + int ret; if (sizeof(*tlstream_acquire) != args_size) goto bad_size; - if (0 != kbase_tlstream_acquire( - kctx, - &tlstream_acquire->fd, 0)) { + ret = kbase_tlstream_acquire( + kctx, 0); + if (ret < 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; - } else if (0 <= tlstream_acquire->fd) { - /* Summary stream was cleared during acquire. - * Create static timeline objects that will be - * read by client. */ - kbase_create_timeline_objects(kctx); - } + else + tlstream_acquire->fd = ret; break; } #endif /* BASE_LEGACY_UK10_4_SUPPORT */ @@ -935,6 +881,7 @@ copy_failed: { struct kbase_uk_tlstream_acquire *tlstream_acquire = args; + int ret; if (sizeof(*tlstream_acquire) != args_size) goto bad_size; @@ -942,17 +889,12 @@ copy_failed: if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) goto out_bad; - if (0 != kbase_tlstream_acquire( - kctx, - &tlstream_acquire->fd, - tlstream_acquire->flags)) { + ret = kbase_tlstream_acquire( + kctx, tlstream_acquire->flags); + if (ret < 0) ukh->ret = MALI_ERROR_FUNCTION_FAILED; - } else if (0 <= tlstream_acquire->fd) { - /* Summary stream was cleared during acquire. - * Create static timeline objects that will be - * read by client. */ - kbase_create_timeline_objects(kctx); - } + else + tlstream_acquire->fd = ret; break; } case KBASE_FUNC_TLSTREAM_FLUSH: @@ -1128,10 +1070,12 @@ void kbase_release_device(struct kbase_device *kbdev) } EXPORT_SYMBOL(kbase_release_device); -#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ + !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) /* * Older versions, before v4.6, of the kernel doesn't have - * kstrtobool_from_user(). + * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 */ static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) { @@ -1320,7 +1264,8 @@ static int kbase_release(struct inode *inode, struct file *filp) #define CALL_MAX_SIZE 536 -static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) { u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ u32 size = _IOC_SIZE(cmd); @@ -1334,7 +1279,7 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return -EFAULT; } - if (kbase_dispatch(kctx, &msg, size) != 0) + if (kbase_legacy_dispatch(kctx, &msg, size) != 0) return -EFAULT; if (0 != copy_to_user((void __user *)arg, &msg, size)) { @@ -1344,6 +1289,600 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return 0; } +static int kbase_api_set_flags(struct kbase_context *kctx, + struct kbase_ioctl_set_flags *flags) +{ + int err; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) + return -EINVAL; + + err = kbase_context_set_create_flags(kctx, flags->create_flags); + /* if bad flags, will stay stuck in setup mode */ + if (err) + return err; + + atomic_set(&kctx->setup_complete, 1); + return 0; +} + +static int kbase_api_job_submit(struct kbase_context *kctx, + struct kbase_ioctl_job_submit *submit) +{ + return kbase_jd_submit(kctx, submit->addr.value, submit->nr_atoms, + submit->stride, false); +} + +static int kbase_api_get_gpuprops(struct kbase_context *kctx, + struct kbase_ioctl_get_gpuprops *get_props) +{ + struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; + int err; + + if (get_props->flags != 0) { + dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); + return -EINVAL; + } + + if (get_props->size == 0) + return kprops->prop_buffer_size; + if (get_props->size < kprops->prop_buffer_size) + return -EINVAL; + + err = copy_to_user(get_props->buffer.value, kprops->prop_buffer, + kprops->prop_buffer_size); + if (err) + return err; + return kprops->prop_buffer_size; +} + +static int kbase_api_post_term(struct kbase_context *kctx) +{ + kbase_event_close(kctx); + return 0; +} + +static int kbase_api_mem_alloc(struct kbase_context *kctx, + union kbase_ioctl_mem_alloc *alloc) +{ + struct kbase_va_region *reg; + u64 flags = alloc->in.flags; + u64 gpu_va; + +#if defined(CONFIG_64BIT) + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* force SAME_VA if a 64-bit client */ + flags |= BASE_MEM_SAME_VA; + } +#endif + + reg = kbase_mem_alloc(kctx, alloc->in.va_pages, + alloc->in.commit_pages, + alloc->in.extent, + &flags, &gpu_va); + + if (!reg) + return -ENOMEM; + + alloc->out.flags = flags; + alloc->out.gpu_va = gpu_va; + + return 0; +} + +static int kbase_api_mem_query(struct kbase_context *kctx, + union kbase_ioctl_mem_query *query) +{ + return kbase_mem_query(kctx, query->in.gpu_addr, + query->in.query, &query->out.value); +} + +static int kbase_api_mem_free(struct kbase_context *kctx, + struct kbase_ioctl_mem_free *free) +{ + return kbase_mem_free(kctx, free->gpu_addr); +} + +static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_reader_setup *setup) +{ + int ret; + struct kbase_uk_hwcnt_reader_setup args = { + .buffer_count = setup->buffer_count, + .jm_bm = setup->jm_bm, + .shader_bm = setup->shader_bm, + .tiler_bm = setup->tiler_bm, + .mmu_l2_bm = setup->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + if (ret) + return ret; + return args.fd; +} + +static int kbase_api_hwcnt_enable(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_enable *enable) +{ + int ret; + struct kbase_uk_hwcnt_setup args = { + .dump_buffer = enable->dump_buffer, + .jm_bm = enable->jm_bm, + .shader_bm = enable->shader_bm, + .tiler_bm = enable->tiler_bm, + .mmu_l2_bm = enable->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, + &kctx->vinstr_cli, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_dump(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_clear(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_disjoint_query(struct kbase_context *kctx, + struct kbase_ioctl_disjoint_query *query) +{ + query->counter = kbase_disjoint_event_get(kctx->kbdev); + + return 0; +} + +static int kbase_api_get_ddk_version(struct kbase_context *kctx, + struct kbase_ioctl_get_ddk_version *version) +{ + int ret; + int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + + if (version->version_buffer.value == NULL) + return len; + + if (version->size < len) + return -EOVERFLOW; + + ret = copy_to_user(version->version_buffer.value, + KERNEL_SIDE_DDK_VERSION_STRING, + sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + + if (ret) + return ret; + + return len; +} + +static int kbase_api_mem_jit_init(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init *jit_init) +{ + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); +} + +static int kbase_api_mem_sync(struct kbase_context *kctx, + struct kbase_ioctl_mem_sync *sync) +{ +#ifdef CONFIG_MALI_COH_USER + return 0; +#endif + struct basep_syncset sset = { + .mem_handle.basep.handle = sync->handle, + .user_addr = sync->user_addr, + .size = sync->size, + .type = sync->type + }; + + return kbase_sync_now(kctx, &sset); +} + +static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, + union kbase_ioctl_mem_find_cpu_offset *find) +{ + return kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->in.cpu_addr, + find->in.size, + &find->out.offset); +} + +static int kbase_api_get_context_id(struct kbase_context *kctx, + struct kbase_ioctl_get_context_id *info) +{ + info->id = kctx->id; + + return 0; +} + +static int kbase_api_tlstream_acquire(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_acquire *acquire) +{ + return kbase_tlstream_acquire(kctx, acquire->flags); +} + +static int kbase_api_tlstream_flush(struct kbase_context *kctx) +{ + kbase_tlstream_flush_streams(); + + return 0; +} + +static int kbase_api_mem_commit(struct kbase_context *kctx, + struct kbase_ioctl_mem_commit *commit) +{ + return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); +} + +static int kbase_api_mem_alias(struct kbase_context *kctx, + union kbase_ioctl_mem_alias *alias) +{ + struct base_mem_aliasing_info *ai; + u64 flags; + int err; + + if (alias->in.nents == 0 || alias->in.nents > 2048) + return -EINVAL; + + ai = vmalloc(sizeof(*ai) * alias->in.nents); + if (!ai) + return -ENOMEM; + + err = copy_from_user(ai, alias->in.aliasing_info.value, + sizeof(*ai) * alias->in.nents); + if (err) { + vfree(ai); + return err; + } + + flags = alias->in.flags; + + alias->out.gpu_va = kbase_mem_alias(kctx, &flags, + alias->in.stride, alias->in.nents, + ai, &alias->out.va_pages); + + alias->out.flags = flags; + + vfree(ai); + + if (alias->out.gpu_va == 0) + return -ENOMEM; + + return 0; +} + +static int kbase_api_mem_import(struct kbase_context *kctx, + union kbase_ioctl_mem_import *import) +{ + int ret; + u64 flags = import->in.flags; + + ret = kbase_mem_import(kctx, + import->in.type, + import->in.phandle.value, + import->in.padding, + &import->out.gpu_va, + &import->out.va_pages, + &flags); + + import->out.flags = flags; + + return ret; +} + +static int kbase_api_mem_flags_change(struct kbase_context *kctx, + struct kbase_ioctl_mem_flags_change *change) +{ + return kbase_mem_flags_change(kctx, change->gpu_va, + change->flags, change->mask); +} + +static int kbase_api_stream_create(struct kbase_context *kctx, + struct kbase_ioctl_stream_create *stream) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + int fd, ret; + + /* Name must be NULL-terminated and padded with NULLs, so check last + * character is NULL + */ + if (stream->name[sizeof(stream->name)-1] != 0) + return -EINVAL; + + ret = kbase_sync_fence_stream_create(stream->name, &fd); + + if (ret) + return ret; + return fd; +#else + return -ENOENT; +#endif +} + +static int kbase_api_fence_validate(struct kbase_context *kctx, + struct kbase_ioctl_fence_validate *validate) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + return kbase_sync_fence_validate(validate->fd); +#else + return -ENOENT; +#endif +} + +static int kbase_api_get_profiling_controls(struct kbase_context *kctx, + struct kbase_ioctl_get_profiling_controls *controls) +{ + if (controls->count > FBDUMP_CONTROL_MAX) + return -EINVAL; + + return copy_to_user(controls->buffer.value, + &kctx->kbdev->kbase_profiling_controls[ + FBDUMP_CONTROL_MIN], + controls->count * sizeof(u32)); +} + +static int kbase_api_mem_profile_add(struct kbase_context *kctx, + struct kbase_ioctl_mem_profile_add *data) +{ + char *buf; + int err; + + if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); + return -EINVAL; + } + + buf = kmalloc(data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + return -ENOMEM; + + err = copy_from_user(buf, data->buffer.value, data->len); + if (err) { + kfree(buf); + return err; + } + + return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); +} + +static int kbase_api_soft_event_update(struct kbase_context *kctx, + struct kbase_ioctl_soft_event_update *update) +{ + if (update->flags != 0) + return -EINVAL; + + return kbase_soft_event_update(kctx, update->event, update->new_status); +} + +#if MALI_UNIT_TEST +static int kbase_api_tlstream_test(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_test *test) +{ + kbase_tlstream_test( + test->tpw_count, + test->msg_delay, + test->msg_count, + test->aux_msg); + + return 0; +} + +static int kbase_api_tlstream_stats(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_stats *stats) +{ + kbase_tlstream_stats( + &stats->bytes_collected, + &stats->bytes_generated); + + return 0; +} +#endif /* MALI_UNIT_TEST */ + +#define KBASE_HANDLE_IOCTL(cmd, function) \ + case cmd: \ + do { \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + return function(kctx); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return function(kctx, ¶m); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct kbase_context *kctx = filp->private_data; + struct kbase_device *kbdev = kctx->kbdev; + void __user *uarg = (void __user *)arg; + + /* The UK ioctl values overflow the cmd field causing the type to be + * incremented + */ + if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* The UK version check IOCTL doesn't overflow the cmd field, so is + * handled separately here + */ + if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, + UKP_FUNC_ID_CHECK_VERSION, + sizeof(struct uku_version_check_args))) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* Only these ioctls are available until setup is complete */ + switch (cmd) { + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, + kbase_api_handshake, + struct kbase_ioctl_version_check); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, + kbase_api_set_flags, + struct kbase_ioctl_set_flags); + } + + /* Block call until version handshake and setup is complete */ + if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) + return -EINVAL; + + /* Normal ioctls */ + switch (cmd) { + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, + kbase_api_job_submit, + struct kbase_ioctl_job_submit); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, + kbase_api_get_gpuprops, + struct kbase_ioctl_get_gpuprops); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, + kbase_api_post_term); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, + kbase_api_mem_alloc, + union kbase_ioctl_mem_alloc); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, + kbase_api_mem_query, + union kbase_ioctl_mem_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, + kbase_api_mem_free, + struct kbase_ioctl_mem_free); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, + kbase_api_hwcnt_reader_setup, + struct kbase_ioctl_hwcnt_reader_setup); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, + kbase_api_hwcnt_enable, + struct kbase_ioctl_hwcnt_enable); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, + kbase_api_hwcnt_dump); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, + kbase_api_hwcnt_clear); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, + kbase_api_disjoint_query, + struct kbase_ioctl_disjoint_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, + kbase_api_get_ddk_version, + struct kbase_ioctl_get_ddk_version); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, + kbase_api_mem_jit_init, + struct kbase_ioctl_mem_jit_init); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, + kbase_api_mem_sync, + struct kbase_ioctl_mem_sync); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, + kbase_api_mem_find_cpu_offset, + union kbase_ioctl_mem_find_cpu_offset); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, + kbase_api_get_context_id, + struct kbase_ioctl_get_context_id); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, + kbase_api_tlstream_acquire, + struct kbase_ioctl_tlstream_acquire); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, + kbase_api_tlstream_flush); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, + kbase_api_mem_commit, + struct kbase_ioctl_mem_commit); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, + kbase_api_mem_alias, + union kbase_ioctl_mem_alias); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, + kbase_api_mem_import, + union kbase_ioctl_mem_import); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, + kbase_api_mem_flags_change, + struct kbase_ioctl_mem_flags_change); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, + kbase_api_stream_create, + struct kbase_ioctl_stream_create); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, + kbase_api_fence_validate, + struct kbase_ioctl_fence_validate); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, + kbase_api_get_profiling_controls, + struct kbase_ioctl_get_profiling_controls); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, + kbase_api_mem_profile_add, + struct kbase_ioctl_mem_profile_add); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, + kbase_api_soft_event_update, + struct kbase_ioctl_soft_event_update); + +#if MALI_UNIT_TEST + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, + kbase_api_tlstream_test, + struct kbase_ioctl_tlstream_test); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, + kbase_api_tlstream_stats, + struct kbase_ioctl_tlstream_stats); +#endif + } + + dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + + return -ENOIOCTLCMD; +} + static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct kbase_context *kctx = filp->private_data; @@ -1576,6 +2115,7 @@ static unsigned long kbase_get_unmapped_area(struct file *filp, int gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; bool is_shader_code = false; + unsigned long ret; /* err on fixed address */ if ((flags & MAP_FIXED) || addr) @@ -1634,7 +2174,19 @@ static unsigned long kbase_get_unmapped_area(struct file *filp, info.align_offset = align_offset; info.align_mask = align_mask; - return kbase_unmapped_area_topdown(&info, is_shader_code); + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + + if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && + high_limit < (kctx->same_va_end << PAGE_SHIFT)) { + /* Retry above mmap_base */ + info.low_limit = mm->mmap_base; + info.high_limit = min_t(u64, TASK_SIZE, + (kctx->same_va_end << PAGE_SHIFT)); + + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + } + + return ret; } static const struct file_operations kbase_fops = { @@ -3132,42 +3684,74 @@ static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { #endif /* CONFIG_DEBUG_FS */ -static int kbasep_protected_mode_enter(struct kbase_device *kbdev) +static int kbasep_protected_mode_init(struct kbase_device *kbdev) { - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_SET_PROTECTED_MODE, NULL); - return 0; -} +#ifdef CONFIG_OF + struct device_node *protected_node; + struct platform_device *pdev; + struct protected_mode_device *protected_dev; +#endif -static bool kbasep_protected_mode_supported(struct kbase_device *kbdev) -{ - return true; -} + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { + /* Use native protected ops */ + kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), + GFP_KERNEL); + if (!kbdev->protected_dev) + return -ENOMEM; + kbdev->protected_dev->data = kbdev; + kbdev->protected_ops = &kbase_native_protected_ops; + kbdev->protected_mode_support = true; + return 0; + } -static struct kbase_protected_ops kbasep_protected_ops = { - .protected_mode_enter = kbasep_protected_mode_enter, - .protected_mode_reset = NULL, - .protected_mode_supported = kbasep_protected_mode_supported, -}; + kbdev->protected_mode_support = false; -static void kbasep_protected_mode_init(struct kbase_device *kbdev) -{ - kbdev->protected_ops = NULL; +#ifdef CONFIG_OF + protected_node = of_parse_phandle(kbdev->dev->of_node, + "protected-mode-switcher", 0); - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { - /* Use native protected ops */ - kbdev->protected_ops = &kbasep_protected_ops; + if (!protected_node) + protected_node = of_parse_phandle(kbdev->dev->of_node, + "secure-mode-switcher", 0); + + if (!protected_node) { + /* If protected_node cannot be looked up then we assume + * protected mode is not supported on this platform. */ + dev_info(kbdev->dev, "Protected mode not available\n"); + return 0; + } + + pdev = of_find_device_by_node(protected_node); + if (!pdev) + return -EINVAL; + + protected_dev = platform_get_drvdata(pdev); + if (!protected_dev) + return -EPROBE_DEFER; + + kbdev->protected_ops = &protected_dev->ops; + kbdev->protected_dev = protected_dev; + + if (kbdev->protected_ops) { + int err; + + /* Make sure protected mode is disabled on startup */ + mutex_lock(&kbdev->pm.lock); + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + mutex_unlock(&kbdev->pm.lock); + + /* protected_mode_disable() returns -EINVAL if not supported */ + kbdev->protected_mode_support = (err != -EINVAL); } -#ifdef PROTECTED_CALLBACKS - else - kbdev->protected_ops = PROTECTED_CALLBACKS; #endif + return 0; +} - if (kbdev->protected_ops) - kbdev->protected_mode_support = - kbdev->protected_ops->protected_mode_supported(kbdev); - else - kbdev->protected_mode_support = false; +static void kbasep_protected_mode_term(struct kbase_device *kbdev) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) + kfree(kbdev->protected_dev); } #ifdef CONFIG_MALI_NO_MALI @@ -3649,6 +4233,8 @@ static int kbase_platform_device_remove(struct platform_device *pdev) if (!kbdev) return -ENODEV; + kfree(kbdev->gpu_props.prop_buffer); + #ifdef CONFIG_MALI_FPGA_BUS_LOGGER if (kbdev->inited_subsys & inited_buslogger) { bl_core_client_unregister(kbdev->buslogger); @@ -3718,6 +4304,11 @@ static int kbase_platform_device_remove(struct platform_device *pdev) if (kbdev->inited_subsys & inited_mem) kbase_mem_halt(kbdev); + if (kbdev->inited_subsys & inited_protected) { + kbasep_protected_mode_term(kbdev); + kbdev->inited_subsys &= ~inited_protected; + } + if (kbdev->inited_subsys & inited_js) { kbasep_js_devdata_term(kbdev); kbdev->inited_subsys &= ~inited_js; @@ -3733,6 +4324,11 @@ static int kbase_platform_device_remove(struct platform_device *pdev) kbdev->inited_subsys &= ~inited_pm_runtime_init; } + if (kbdev->inited_subsys & inited_ctx_sched) { + kbase_ctx_sched_term(kbdev); + kbdev->inited_subsys &= ~inited_ctx_sched; + } + if (kbdev->inited_subsys & inited_device) { kbase_device_term(kbdev); kbdev->inited_subsys &= ~inited_device; @@ -3805,10 +4401,6 @@ static int kbase_platform_device_probe(struct platform_device *pdev) kbdev->dev = &pdev->dev; dev_set_drvdata(kbdev->dev, kbdev); -#ifdef CONFIG_DEVFREQ_THERMAL - INIT_LIST_HEAD(&kbdev->ipa.power_models); -#endif - #ifdef CONFIG_MALI_NO_MALI err = gpu_device_create(kbdev); if (err) { @@ -3877,6 +4469,15 @@ static int kbase_platform_device_probe(struct platform_device *pdev) } kbdev->inited_subsys |= inited_device; + err = kbase_ctx_sched_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", + err); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_ctx_sched; + if (kbdev->pm.callback_power_runtime_init) { err = kbdev->pm.callback_power_runtime_init(kbdev); if (err) { @@ -3902,7 +4503,13 @@ static int kbase_platform_device_probe(struct platform_device *pdev) kbase_device_coherency_init(kbdev, prod_id); - kbasep_protected_mode_init(kbdev); + err = kbasep_protected_mode_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_protected; dev_list = kbase_dev_list_get(); list_add(&kbdev->entry, &kbase_dev_list); @@ -4020,6 +4627,13 @@ static int kbase_platform_device_probe(struct platform_device *pdev) } #endif + err = kbase_gpuprops_populate_user_buffer(kbdev); + if (err) { + dev_err(&pdev->dev, "GPU property population failed"); + kbase_platform_device_remove(pdev); + return err; + } + dev_info(kbdev->dev, "Probed as %s\n", dev_name(kbdev->mdev.this_device)); @@ -4030,7 +4644,6 @@ static int kbase_platform_device_probe(struct platform_device *pdev) #undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE - /** * kbase_device_suspend - Suspend callback from the OS. * diff --git a/mali_kbase/mali_kbase_ctx_sched.c b/mali_kbase/mali_kbase_ctx_sched.c new file mode 100644 index 0000000..e2f7baa --- /dev/null +++ b/mali_kbase/mali_kbase_ctx_sched.c @@ -0,0 +1,203 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include <mali_kbase.h> +#include <mali_kbase_config_defaults.h> + +#include "mali_kbase_ctx_sched.h" + +int kbase_ctx_sched_init(struct kbase_device *kbdev) +{ + int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; + + /* These two must be recalculated if nr_hw_address_spaces changes + * (e.g. for HW workarounds) */ + kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { + bool use_workaround; + + use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; + if (use_workaround) { + dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); + kbdev->nr_user_address_spaces = 1; + } + } + + kbdev->as_free = as_present; /* All ASs initially free */ + + memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); + + return 0; +} + +void kbase_ctx_sched_term(struct kbase_device *kbdev) +{ + s8 i; + + /* Sanity checks */ + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + WARN_ON(kbdev->as_to_kctx[i] != NULL); + WARN_ON(!(kbdev->as_free & (1u << i))); + } +} + +/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space + * + * @kbdev: The context for which to find a free address space + * + * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID + * + * This function returns an address space available for use. It would prefer + * returning an AS that has been previously assigned to the context to + * avoid having to reprogram the MMU. + */ +static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int free_as; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* First check if the previously assigned AS is available */ + if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && + (kbdev->as_free & (1u << kctx->as_nr))) + return kctx->as_nr; + + /* The previously assigned AS was taken, we'll be returning any free + * AS at this point. + */ + free_as = ffs(kbdev->as_free) - 1; + if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) + return free_as; + + return KBASEP_AS_NR_INVALID; +} + +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + if (atomic_inc_return(&kctx->refcount) == 1) { + int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); + + if (free_as != KBASEP_AS_NR_INVALID) { + kbdev->as_free &= ~(1u << free_as); + /* Only program the MMU if the context has not been + * assigned the same address space before. + */ + if (free_as != kctx->as_nr) { + struct kbase_context *const prev_kctx = + kbdev->as_to_kctx[free_as]; + + if (prev_kctx) { + WARN_ON(atomic_read(&prev_kctx->refcount) != 0); + kbase_mmu_disable(prev_kctx); + prev_kctx->as_nr = KBASEP_AS_NR_INVALID; + } + + kctx->as_nr = free_as; + kbdev->as_to_kctx[free_as] = kctx; + kbase_mmu_update(kctx); + } + } else { + atomic_dec(&kctx->refcount); + + /* Failed to find an available address space, we must + * be returning an error at this point. + */ + WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); + } + } + + return kctx->as_nr; +} + +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) == 0); + WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); + WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); + + atomic_inc(&kctx->refcount); +} + +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_dec_return(&kctx->refcount) == 0) + kbdev->as_free |= (1u << kctx->as_nr); +} + +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(atomic_read(&kctx->refcount) != 0); + + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + if (kbdev->pm.backend.gpu_powered) + kbase_mmu_disable(kctx); + + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } +} + +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) +{ + s8 i; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + struct kbase_context *kctx; + + kctx = kbdev->as_to_kctx[i]; + if (kctx) { + if (atomic_read(&kctx->refcount)) { + WARN_ON(kctx->as_nr != i); + + kbase_mmu_update(kctx); + } else { + /* This context might have been assigned an + * AS before, clear it. + */ + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } + } else { + kbase_mmu_disable_as(kbdev, i); + } + } +} diff --git a/mali_kbase/mali_kbase_ctx_sched.h b/mali_kbase/mali_kbase_ctx_sched.h new file mode 100644 index 0000000..e551525 --- /dev/null +++ b/mali_kbase/mali_kbase_ctx_sched.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_CTX_SCHED_H_ +#define _KBASE_CTX_SCHED_H_ + +#include <mali_kbase.h> + +/* The Context Scheduler manages address space assignment and reference + * counting to kbase_context. The interface has been designed to minimise + * interactions between the Job Scheduler and Power Management/MMU to support + * both the existing Job Scheduler and Command Stream Frontend interface. + * + * The initial implementation of the Context Scheduler does not schedule + * contexts. Instead it relies on the Job Scheduler/CSF to make decisions of + * when to schedule/evict contexts if address spaces are starved. In the + * future, once an interface between the CS and JS/CSF have been devised to + * provide enough information about how each context is consuming GPU resources, + * those decisions can be made in the CS itself, thereby reducing duplicated + * code. + */ + +/* base_ctx_sched_init - Initialise the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * initialised + * + * Return: 0 for success, otherwise failure + * + * This must be called during device initilisation. The number of hardware + * address spaces must already be established before calling this function. + */ +int kbase_ctx_sched_init(struct kbase_device *kbdev); + +/* base_ctx_sched_term - Terminate the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * terminated + * + * This must be called during device termination after all contexts have been + * destroyed. + */ +void kbase_ctx_sched_term(struct kbase_device *kbdev); + +/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context + * + * @kctx: The context to which to retain a reference + * + * Return: The address space that the context has been assigned to or + * KBASEP_AS_NR_INVALID if no address space was available. + * + * This function should be called whenever an address space should be assigned + * to a context and programmed onto the MMU. It should typically be called + * when jobs are ready to be submitted to the GPU. + * + * It can be called as many times as necessary. The address space will be + * assigned to the context for as long as there is a reference to said context. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_retain_ctx_refcount + * + * @kctx: The context to which to retain a reference + * + * This function only retains a reference to the context. It must be called + * only when the context already has a reference. + * + * This is typically called inside an atomic session where we know the context + * is already scheduled in but want to take an extra reference to ensure that + * it doesn't get descheduled. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); + +/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context + * + * @kctx: The context from which to release a reference + * + * This function should be called whenever an address space could be unassigned + * from a context. When there are no more references to said context, the + * address space previously assigned to this context shall be reassigned to + * other contexts as needed. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space + * + * @kctx: The context to be removed + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces + * + * @kbdev: The device for which address spaces to be reprogrammed + * + * This function shall reprogram all address spaces previously assigned to + * contexts. It can be used after the GPU is reset. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); + +#endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/mali_kbase/mali_kbase_debug_mem_view.c b/mali_kbase/mali_kbase_debug_mem_view.c index dc75c86..6f2cbdf 100644 --- a/mali_kbase/mali_kbase_debug_mem_view.c +++ b/mali_kbase/mali_kbase_debug_mem_view.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_defs.h b/mali_kbase/mali_kbase_defs.h index 73f849c..8dabf7c 100644 --- a/mali_kbase/mali_kbase_defs.h +++ b/mali_kbase/mali_kbase_defs.h @@ -35,6 +35,7 @@ #include <mali_kbase_mmu_mode.h> #include <mali_kbase_instr_defs.h> #include <mali_kbase_pm.h> +#include <protected_mode_switcher.h> #include <linux/atomic.h> #include <linux/mempool.h> @@ -50,19 +51,19 @@ #include <linux/kds.h> #endif /* CONFIG_KDS */ -#ifdef CONFIG_SYNC -#include "sync.h" -#endif /* CONFIG_SYNC */ - -#include "mali_kbase_dma_fence.h" +#if defined(CONFIG_SYNC) +#include <sync.h> +#else +#include "mali_kbase_fence_defs.h" +#endif #ifdef CONFIG_DEBUG_FS #include <linux/debugfs.h> #endif /* CONFIG_DEBUG_FS */ -#ifdef CONFIG_PM_DEVFREQ +#ifdef CONFIG_MALI_DEVFREQ #include <linux/devfreq.h> -#endif /* CONFIG_DEVFREQ */ +#endif /* CONFIG_MALI_DEVFREQ */ #include <linux/clk.h> #include <linux/regulator/consumer.h> @@ -436,15 +437,26 @@ struct kbase_jd_atom { struct kds_resource_set *kds_rset; bool kds_dep_satisfied; #endif /* CONFIG_KDS */ -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) + /* Stores either an input or output fence, depending on soft-job type */ struct sync_fence *fence; struct sync_fence_waiter sync_waiter; #endif /* CONFIG_SYNC */ -#ifdef CONFIG_MALI_DMA_FENCE +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) struct { - /* This points to the dma-buf fence for this atom. If this is - * NULL then there is no fence for this atom and the other - * fields related to dma_fence may have invalid data. + /* Use the functions/API defined in mali_kbase_fence.h to + * when working with this sub struct */ +#if defined(CONFIG_SYNC_FILE) + /* Input fence */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence_in; +#else + struct dma_fence *fence_in; +#endif +#endif + /* This points to the dma-buf output fence for this atom. If + * this is NULL then there is no fence for this atom and the + * following fields related to dma_fence may have invalid data. * * The context and seqno fields contain the details for this * fence. @@ -453,7 +465,11 @@ struct kbase_jd_atom { * regardless of the event_code of the katom (signal also on * failure). */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) struct fence *fence; +#else + struct dma_fence *fence; +#endif /* The dma-buf fence context number for this atom. A unique * context number is allocated to each katom in the context on * context creation. @@ -492,7 +508,7 @@ struct kbase_jd_atom { */ atomic_t dep_count; } dma_fence; -#endif /* CONFIG_MALI_DMA_FENCE */ +#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE*/ /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ enum base_jd_event_code event_code; @@ -887,40 +903,6 @@ struct kbase_pm_device_data { }; /** - * struct kbase_protected_ops - Platform specific functions for GPU protected - * mode operations - * @protected_mode_enter: Callback to enter protected mode on the GPU - * @protected_mode_reset: Callback to reset the GPU and exit protected mode. - * @protected_mode_supported: Callback to check if protected mode is supported. - */ -struct kbase_protected_ops { - /** - * protected_mode_enter() - Enter protected mode on the GPU - * @kbdev: The kbase device - * - * Return: 0 on success, non-zero on error - */ - int (*protected_mode_enter)(struct kbase_device *kbdev); - - /** - * protected_mode_reset() - Reset the GPU and exit protected mode - * @kbdev: The kbase device - * - * Return: 0 on success, non-zero on error - */ - int (*protected_mode_reset)(struct kbase_device *kbdev); - - /** - * protected_mode_supported() - Check if protected mode is supported - * @kbdev: The kbase device - * - * Return: 0 on success, non-zero on error - */ - bool (*protected_mode_supported)(struct kbase_device *kbdev); -}; - - -/** * struct kbase_mem_pool - Page based memory pool for kctx/kbdev * @kbdev: Kbase device where memory is used * @cur_size: Number of free pages currently in the pool (may exceed @max_size @@ -1003,6 +985,14 @@ struct kbase_device { struct kbase_mmu_mode const *mmu_mode; struct kbase_as as[BASE_MAX_NR_AS]; + /* The below variables (as_free and as_to_kctx) are managed by the + * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must + * be held whilst accessing these. + */ + u16 as_free; /* Bitpattern of free Address Spaces */ + /* Mapping from active Address Spaces to kbase_context */ + struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; + spinlock_t mmu_mask_change; @@ -1097,7 +1087,7 @@ struct kbase_device { struct list_head kctx_list; struct mutex kctx_list_lock; -#ifdef CONFIG_PM_DEVFREQ +#ifdef CONFIG_MALI_DEVFREQ struct devfreq_dev_profile devfreq_profile; struct devfreq *devfreq; unsigned long current_freq; @@ -1117,14 +1107,11 @@ struct kbase_device { struct { /* Access to this struct must be with ipa.lock held */ struct mutex lock; - unsigned long last_model_dyn_power_coeff; - unsigned long last_static_power_coeff; - struct list_head power_models; struct kbase_ipa_model *configured_model; struct kbase_ipa_model *fallback_model; } ipa; -#endif -#endif +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_DEVFREQ */ #ifdef CONFIG_MALI_TRACE_TIMELINE @@ -1224,8 +1211,11 @@ struct kbase_device { u32 snoop_enable_smc; u32 snoop_disable_smc; - /* Protected operations */ - struct kbase_protected_ops *protected_ops; + /* Protected mode operations */ + struct protected_mode_ops *protected_ops; + + /* Protected device attached to this kbase device */ + struct protected_mode_device *protected_dev; /* * true when GPU is put into protected mode @@ -1411,6 +1401,14 @@ struct kbase_context { * you can take whilst doing this) */ int as_nr; + /* Keeps track of the number of users of this context. A user can be a + * job that is available for execution, instrumentation needing to 'pin' + * a context for counter collection, etc. If the refcount reaches 0 then + * this context is considered inactive and the previously programmed + * AS might be cleared at any point. + */ + atomic_t refcount; + /* NOTE: * * Flags are in jctx.sched_info.ctx.flags diff --git a/mali_kbase/mali_kbase_device.c b/mali_kbase/mali_kbase_device.c index e53f876..d635fcc 100644 --- a/mali_kbase/mali_kbase_device.c +++ b/mali_kbase/mali_kbase_device.c @@ -245,11 +245,10 @@ int kbase_device_init(struct kbase_device * const kbdev) kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); -#else - kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); + else + kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); #ifdef CONFIG_MALI_DEBUG init_waitqueue_head(&kbdev->driver_inactive_wait); @@ -656,28 +655,6 @@ void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 va } } -u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control) -{ - u32 ret_value = 0; - - switch (control) { - case FBDUMP_CONTROL_ENABLE: - /* fall through */ - case FBDUMP_CONTROL_RATE: - /* fall through */ - case SW_COUNTER_ENABLE: - /* fall through */ - case FBDUMP_CONTROL_RESIZE_FACTOR: - ret_value = kbdev->kbase_profiling_controls[control]; - break; - default: - dev_err(kbdev->dev, "Profiling control %d not found\n", control); - break; - } - - return ret_value; -} - /* * Called by gator to control the production of * profiling information at runtime diff --git a/mali_kbase/mali_kbase_dma_fence.c b/mali_kbase/mali_kbase_dma_fence.c index 97bb6c5..9197743 100644 --- a/mali_kbase/mali_kbase_dma_fence.c +++ b/mali_kbase/mali_kbase_dma_fence.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,7 +22,6 @@ #include "mali_kbase_dma_fence.h" #include <linux/atomic.h> -#include <linux/fence.h> #include <linux/list.h> #include <linux/lockdep.h> #include <linux/mutex.h> @@ -34,10 +33,6 @@ #include <mali_kbase.h> - -/* Spin lock protecting all Mali fences as fence->lock. */ -static DEFINE_SPINLOCK(kbase_dma_fence_lock); - static void kbase_dma_fence_work(struct work_struct *pwork); @@ -49,67 +44,12 @@ kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); } -void +static void kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) { list_del(&katom->queue); } -static const char * -kbase_dma_fence_get_driver_name(struct fence *fence) -{ - return kbase_drv_name; -} - -static const char * -kbase_dma_fence_get_timeline_name(struct fence *fence) -{ - return kbase_timeline_name; -} - -static bool -kbase_dma_fence_enable_signaling(struct fence *fence) -{ - /* If in the future we need to add code here remember to - * to get a reference to the fence and release it when signaling - * as stated in fence.h - */ - return true; -} - -static void -kbase_dma_fence_fence_value_str(struct fence *fence, char *str, int size) -{ - snprintf(str, size, "%u", fence->seqno); -} - -static const struct fence_ops kbase_dma_fence_ops = { - .get_driver_name = kbase_dma_fence_get_driver_name, - .get_timeline_name = kbase_dma_fence_get_timeline_name, - .enable_signaling = kbase_dma_fence_enable_signaling, - /* Use the default wait */ - .wait = fence_default_wait, - .fence_value_str = kbase_dma_fence_fence_value_str, -}; - -static struct fence * -kbase_dma_fence_new(unsigned int context, unsigned int seqno) -{ - struct fence *fence; - - fence = kzalloc(sizeof(*fence), GFP_KERNEL); - if (!fence) - return NULL; - - fence_init(fence, - &kbase_dma_fence_ops, - &kbase_dma_fence_lock, - context, - seqno); - - return fence; -} - static int kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, struct ww_acquire_ctx *ctx) @@ -189,59 +129,6 @@ kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) } /** - * kbase_dma_fence_free_callbacks - Free dma-fence callbacks on a katom - * @katom: Pointer to katom - * @queue_worker: Boolean indicating if fence worker is to be queued when - * dep_count reaches 0. - * - * This function will free all fence callbacks on the katom's list of - * callbacks. Callbacks that have not yet been called, because their fence - * hasn't yet signaled, will first be removed from the fence. - * - * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. - */ -static void -kbase_dma_fence_free_callbacks(struct kbase_jd_atom *katom, bool queue_worker) -{ - struct kbase_dma_fence_cb *cb, *tmp; - - lockdep_assert_held(&katom->kctx->jctx.lock); - - /* Clean up and free callbacks. */ - list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { - bool ret; - - /* Cancel callbacks that hasn't been called yet. */ - ret = fence_remove_callback(cb->fence, &cb->fence_cb); - if (ret) { - int ret; - - /* Fence had not signaled, clean up after - * canceling. - */ - ret = atomic_dec_return(&katom->dma_fence.dep_count); - - if (unlikely(queue_worker && ret == 0)) { - /* - * dep_count went to zero and queue_worker is - * true. Queue the worker to handle the - * completion of the katom. - */ - kbase_dma_fence_queue_work(katom); - } - } - - /* - * Release the reference taken in - * kbase_dma_fence_add_callback(). - */ - fence_put(cb->fence); - list_del(&cb->node); - kfree(cb); - } -} - -/** * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom * @katom: Katom to cancel * @@ -253,14 +140,12 @@ kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) lockdep_assert_held(&katom->kctx->jctx.lock); /* Cancel callbacks and clean up. */ - kbase_dma_fence_free_callbacks(katom, false); - - KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == 0); + kbase_fence_free_callbacks(katom); /* Mark the atom as handled in case all fences signaled just before * canceling the callbacks and the worker was queued. */ - atomic_set(&katom->dma_fence.dep_count, -1); + kbase_fence_dep_count_set(katom, -1); /* Prevent job_done_nolock from being called twice on an atom when * there is a race between job completion and cancellation. @@ -290,15 +175,15 @@ kbase_dma_fence_work(struct work_struct *pwork) ctx = &katom->kctx->jctx; mutex_lock(&ctx->lock); - if (atomic_read(&katom->dma_fence.dep_count) != 0) + if (kbase_fence_dep_count_read(katom) != 0) goto out; - atomic_set(&katom->dma_fence.dep_count, -1); + kbase_fence_dep_count_set(katom, -1); /* Remove atom from list of dma-fence waiting atoms. */ kbase_dma_fence_waiters_remove(katom); /* Cleanup callbacks. */ - kbase_dma_fence_free_callbacks(katom, false); + kbase_fence_free_callbacks(katom); /* * Queue atom on GPU, unless it has already completed due to a failing * dependency. Run jd_done_nolock() on the katom if it is completed. @@ -312,64 +197,15 @@ out: mutex_unlock(&ctx->lock); } -/** - * kbase_dma_fence_add_callback() - Add callback on @fence to block @katom - * @katom: Pointer to katom that will be blocked by @fence - * @fence: Pointer to fence on which to set up the callback - * @callback: Pointer to function to be called when fence is signaled - * - * Caller needs to hold a reference to @fence when calling this function, and - * the caller is responsible for releasing that reference. An additional - * reference to @fence will be taken when the callback was successfully set up - * and @fence needs to be kept valid until the callback has been called and - * cleanup have been done. - * - * Return: 0 on success: fence was either already signalled, or callback was - * set up. Negative error code is returned on error. - */ -static int -kbase_dma_fence_add_callback(struct kbase_jd_atom *katom, - struct fence *fence, - fence_func_t callback) -{ - int err = 0; - struct kbase_dma_fence_cb *kbase_fence_cb; - - kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); - if (!kbase_fence_cb) - return -ENOMEM; - - kbase_fence_cb->fence = fence; - kbase_fence_cb->katom = katom; - INIT_LIST_HEAD(&kbase_fence_cb->node); - - err = fence_add_callback(fence, &kbase_fence_cb->fence_cb, callback); - if (err == -ENOENT) { - /* Fence signaled, clear the error and return */ - err = 0; - kbase_fence_cb->fence = NULL; - kfree(kbase_fence_cb); - } else if (err) { - kfree(kbase_fence_cb); - } else { - /* - * Get reference to fence that will be kept until callback gets - * cleaned up in kbase_dma_fence_free_callbacks(). - */ - fence_get(fence); - atomic_inc(&katom->dma_fence.dep_count); - /* Add callback to katom's list of callbacks */ - list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); - } - - return err; -} - static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) +#else +kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +#endif { - struct kbase_dma_fence_cb *kcb = container_of(cb, - struct kbase_dma_fence_cb, + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, fence_cb); struct kbase_jd_atom *katom = kcb->katom; @@ -377,7 +213,8 @@ kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) * preventing this callback from ever scheduling work. Which in turn * would reschedule the atom. */ - if (atomic_dec_and_test(&katom->dma_fence.dep_count)) + + if (kbase_fence_dep_count_dec_and_test(katom)) kbase_dma_fence_queue_work(katom); } @@ -386,8 +223,13 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, struct reservation_object *resv, bool exclusive) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) struct fence *excl_fence = NULL; struct fence **shared_fences = NULL; +#else + struct dma_fence *excl_fence = NULL; + struct dma_fence **shared_fences = NULL; +#endif unsigned int shared_count = 0; int err, i; @@ -399,16 +241,16 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, return err; if (excl_fence) { - err = kbase_dma_fence_add_callback(katom, - excl_fence, - kbase_dma_fence_cb); + err = kbase_fence_add_callback(katom, + excl_fence, + kbase_dma_fence_cb); /* Release our reference, taken by reservation_object_get_fences_rcu(), * to the fence. We have set up our callback (if that was possible), * and it's the fence's owner is responsible for singling the fence * before allowing it to disappear. */ - fence_put(excl_fence); + dma_fence_put(excl_fence); if (err) goto out; @@ -416,9 +258,9 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, if (exclusive) { for (i = 0; i < shared_count; i++) { - err = kbase_dma_fence_add_callback(katom, - shared_fences[i], - kbase_dma_fence_cb); + err = kbase_fence_add_callback(katom, + shared_fences[i], + kbase_dma_fence_cb); if (err) goto out; } @@ -431,7 +273,7 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, */ out: for (i = 0; i < shared_count; i++) - fence_put(shared_fences[i]); + dma_fence_put(shared_fences[i]); kfree(shared_fences); if (err) { @@ -439,7 +281,7 @@ out: * On error, cancel and clean up all callbacks that was set up * before the error. */ - kbase_dma_fence_free_callbacks(katom, false); + kbase_fence_free_callbacks(katom); } return err; @@ -468,13 +310,16 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, struct kbase_dma_fence_resv_info *info) { int err, i; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) struct fence *fence; +#else + struct dma_fence *fence; +#endif struct ww_acquire_ctx ww_ctx; lockdep_assert_held(&katom->kctx->jctx.lock); - fence = kbase_dma_fence_new(katom->dma_fence.context, - atomic_inc_return(&katom->dma_fence.seqno)); + fence = kbase_fence_out_new(katom); if (!fence) { err = -ENOMEM; dev_err(katom->kctx->kbdev->dev, @@ -482,15 +327,14 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, return err; } - katom->dma_fence.fence = fence; - atomic_set(&katom->dma_fence.dep_count, 1); + kbase_fence_dep_count_set(katom, 1); err = kbase_dma_fence_lock_reservations(info, &ww_ctx); if (err) { dev_err(katom->kctx->kbdev->dev, "Error %d locking reservations.\n", err); - atomic_set(&katom->dma_fence.dep_count, -1); - fence_put(fence); + kbase_fence_dep_count_set(katom, -1); + kbase_fence_out_remove(katom); return err; } @@ -512,7 +356,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, goto end; } - reservation_object_add_shared_fence(obj, katom->dma_fence.fence); + reservation_object_add_shared_fence(obj, fence); } else { err = kbase_dma_fence_add_reservation_callback(katom, obj, true); if (err) { @@ -521,7 +365,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, goto end; } - reservation_object_add_excl_fence(obj, katom->dma_fence.fence); + reservation_object_add_excl_fence(obj, fence); } } @@ -530,9 +374,9 @@ end: if (likely(!err)) { /* Test if the callbacks are already triggered */ - if (atomic_dec_and_test(&katom->dma_fence.dep_count)) { - atomic_set(&katom->dma_fence.dep_count, -1); - kbase_dma_fence_free_callbacks(katom, false); + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_dep_count_set(katom, -1); + kbase_fence_free_callbacks(katom); } else { /* Add katom to the list of dma-buf fence waiting atoms * only if it is still waiting. @@ -545,8 +389,8 @@ end: * kill it for us), signal the fence, free callbacks and the * fence. */ - kbase_dma_fence_free_callbacks(katom, false); - atomic_set(&katom->dma_fence.dep_count, -1); + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); kbase_dma_fence_signal(katom); } @@ -569,7 +413,8 @@ void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) { /* Cancel callbacks and clean up. */ - kbase_dma_fence_free_callbacks(katom, true); + if (kbase_fence_free_callbacks(katom)) + kbase_dma_fence_queue_work(katom); } void kbase_dma_fence_signal(struct kbase_jd_atom *katom) @@ -577,14 +422,12 @@ void kbase_dma_fence_signal(struct kbase_jd_atom *katom) if (!katom->dma_fence.fence) return; - KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == -1); - /* Signal the atom's fence. */ - fence_signal(katom->dma_fence.fence); - fence_put(katom->dma_fence.fence); - katom->dma_fence.fence = NULL; + dma_fence_signal(katom->dma_fence.fence); + + kbase_fence_out_remove(katom); - kbase_dma_fence_free_callbacks(katom, false); + kbase_fence_free_callbacks(katom); } void kbase_dma_fence_term(struct kbase_context *kctx) diff --git a/mali_kbase/mali_kbase_dma_fence.h b/mali_kbase/mali_kbase_dma_fence.h index 3b0a69b..c9ab403 100644 --- a/mali_kbase/mali_kbase_dma_fence.h +++ b/mali_kbase/mali_kbase_dma_fence.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,9 +20,9 @@ #ifdef CONFIG_MALI_DMA_FENCE -#include <linux/fence.h> #include <linux/list.h> #include <linux/reservation.h> +#include <mali_kbase_fence.h> /* Forward declaration from mali_kbase_defs.h */ @@ -30,20 +30,6 @@ struct kbase_jd_atom; struct kbase_context; /** - * struct kbase_dma_fence_cb - Mali dma-fence callback data struct - * @fence_cb: Callback function - * @katom: Pointer to katom that is waiting on this callback - * @fence: Pointer to the fence object on which this callback is waiting - * @node: List head for linking this callback to the katom - */ -struct kbase_dma_fence_cb { - struct fence_cb fence_cb; - struct kbase_jd_atom *katom; - struct fence *fence; - struct list_head node; -}; - -/** * struct kbase_dma_fence_resv_info - Structure with list of reservation objects * @resv_objs: Array of reservation objects to attach the * new fence to. @@ -131,11 +117,6 @@ void kbase_dma_fence_term(struct kbase_context *kctx); */ int kbase_dma_fence_init(struct kbase_context *kctx); -/** - * kbase_dma_fence_waiters_remove()- Remove katom from dma-fence wait list - * @katom: Pointer to katom to remove from list - */ -void kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom); #else /* CONFIG_MALI_DMA_FENCE */ /* Dummy functions for when dma-buf fence isn't enabled. */ diff --git a/mali_kbase/mali_kbase_fence.c b/mali_kbase/mali_kbase_fence.c new file mode 100644 index 0000000..fcb3733 --- /dev/null +++ b/mali_kbase/mali_kbase_fence.c @@ -0,0 +1,196 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include <linux/atomic.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <mali_kbase_fence_defs.h> +#include <mali_kbase_fence.h> +#include <mali_kbase.h> + +/* Spin lock protecting all Mali fences as fence->lock. */ +static DEFINE_SPINLOCK(kbase_fence_lock); + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_driver_name(struct fence *fence) +#else +kbase_fence_get_driver_name(struct dma_fence *fence) +#endif +{ + return kbase_drv_name; +} + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_timeline_name(struct fence *fence) +#else +kbase_fence_get_timeline_name(struct dma_fence *fence) +#endif +{ + return kbase_timeline_name; +} + +static bool +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_enable_signaling(struct fence *fence) +#else +kbase_fence_enable_signaling(struct dma_fence *fence) +#endif +{ + return true; +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_fence_value_str(struct fence *fence, char *str, int size) +#else +kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) +#endif +{ + snprintf(str, size, "%u", fence->seqno); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +const struct fence_ops kbase_fence_ops = { + .wait = fence_default_wait, +#else +const struct dma_fence_ops kbase_fence_ops = { + .wait = dma_fence_default_wait, +#endif + .get_driver_name = kbase_fence_get_driver_name, + .get_timeline_name = kbase_fence_get_timeline_name, + .enable_signaling = kbase_fence_enable_signaling, + .fence_value_str = kbase_fence_fence_value_str +}; + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#else +struct dma_fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#endif +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + WARN_ON(katom->dma_fence.fence); + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + dma_fence_init(fence, + &kbase_fence_ops, + &kbase_fence_lock, + katom->dma_fence.context, + atomic_inc_return(&katom->dma_fence.seqno)); + + katom->dma_fence.fence = fence; + + return fence; +} + +bool +kbase_fence_free_callbacks(struct kbase_jd_atom *katom) +{ + struct kbase_fence_cb *cb, *tmp; + bool res = false; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Clean up and free callbacks. */ + list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { + bool ret; + + /* Cancel callbacks that hasn't been called yet. */ + ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); + if (ret) { + int ret; + + /* Fence had not signaled, clean up after + * canceling. + */ + ret = atomic_dec_return(&katom->dma_fence.dep_count); + + if (unlikely(ret == 0)) + res = true; + } + + /* + * Release the reference taken in + * kbase_fence_add_callback(). + */ + dma_fence_put(cb->fence); + list_del(&cb->node); + kfree(cb); + } + + return res; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback) +#else +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback) +#endif +{ + int err = 0; + struct kbase_fence_cb *kbase_fence_cb; + + if (!fence) + return -EINVAL; + + kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); + if (!kbase_fence_cb) + return -ENOMEM; + + kbase_fence_cb->fence = fence; + kbase_fence_cb->katom = katom; + INIT_LIST_HEAD(&kbase_fence_cb->node); + + err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, + callback); + if (err == -ENOENT) { + /* Fence signaled, clear the error and return */ + err = 0; + kfree(kbase_fence_cb); + } else if (err) { + kfree(kbase_fence_cb); + } else { + /* + * Get reference to fence that will be kept until callback gets + * cleaned up in kbase_fence_free_callbacks(). + */ + dma_fence_get(fence); + atomic_inc(&katom->dma_fence.dep_count); + /* Add callback to katom's list of callbacks */ + list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); + } + + return err; +} diff --git a/mali_kbase/mali_kbase_fence.h b/mali_kbase/mali_kbase_fence.h new file mode 100644 index 0000000..8d39299 --- /dev/null +++ b/mali_kbase/mali_kbase_fence.h @@ -0,0 +1,266 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_H_ +#define _KBASE_FENCE_H_ + +/* + * mali_kbase_fence.[hc] has common fence code used by both + * - CONFIG_MALI_DMA_FENCE - implicit DMA fences + * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel + */ + +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include <linux/list.h> +#include "mali_kbase_fence_defs.h" +#include "mali_kbase.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +extern const struct fence_ops kbase_fence_ops; +#else +extern const struct dma_fence_ops kbase_fence_ops; +#endif + +/** +* struct kbase_fence_cb - Mali dma-fence callback data struct +* @fence_cb: Callback function +* @katom: Pointer to katom that is waiting on this callback +* @fence: Pointer to the fence object on which this callback is waiting +* @node: List head for linking this callback to the katom +*/ +struct kbase_fence_cb { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence_cb fence_cb; + struct fence *fence; +#else + struct dma_fence_cb fence_cb; + struct dma_fence *fence; +#endif + struct kbase_jd_atom *katom; + struct list_head node; +}; + +/** + * kbase_fence_out_new() - Creates a new output fence and puts it on the atom + * @katom: Atom to create an output fence for + * + * return: A new fence object on success, NULL on failure. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#else +struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#endif + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_fence_in_set() - Assign input fence to atom + * @katom: Atom to assign input fence to + * @fence: Input fence to assign to atom + * + * This function will take ownership of one fence reference! + */ +#define kbase_fence_fence_in_set(katom, fence) \ + do { \ + WARN_ON((katom)->dma_fence.fence_in); \ + (katom)->dma_fence.fence_in = fence; \ + } while (0) +#endif + +/** + * kbase_fence_out_remove() - Removes the output fence from atom + * @katom: Atom to remove output fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence) { + dma_fence_put(katom->dma_fence.fence); + katom->dma_fence.fence = NULL; + } +} + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_out_remove() - Removes the input fence from atom + * @katom: Atom to remove input fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence_in) { + dma_fence_put(katom->dma_fence.fence_in); + katom->dma_fence.fence_in = NULL; + } +} +#endif + +/** + * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us + * @katom: Atom to check output fence for + * + * Return: true if fence exists and is valid, otherwise false + */ +static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) +{ + return katom->dma_fence.fence && + katom->dma_fence.fence->ops == &kbase_fence_ops; +} + +/** + * kbase_fence_out_signal() - Signal output fence of atom + * @katom: Atom to signal output fence for + * @status: Status to signal with (0 for success, < 0 for error) + * + * Return: 0 on success, < 0 on error + */ +static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, + int status) +{ + katom->dma_fence.fence->status = status; + return dma_fence_signal(katom->dma_fence.fence); +} + +/** + * kbase_fence_add_callback() - Add callback on @fence to block @katom + * @katom: Pointer to katom that will be blocked by @fence + * @fence: Pointer to fence on which to set up the callback + * @callback: Pointer to function to be called when fence is signaled + * + * Caller needs to hold a reference to @fence when calling this function, and + * the caller is responsible for releasing that reference. An additional + * reference to @fence will be taken when the callback was successfully set up + * and @fence needs to be kept valid until the callback has been called and + * cleanup have been done. + * + * Return: 0 on success: fence was either already signaled, or callback was + * set up. Negative error code is returned on error. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback); +#else +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback); +#endif + +/** + * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value + * @katom: Atom to set dep_count for + * @val: value to set dep_count to + * + * The dep_count is available to the users of this module so that they can + * synchronize completion of the wait with cancellation and adding of more + * callbacks. For instance, a user could do the following: + * + * dep_count set to 1 + * callback #1 added, dep_count is increased to 2 + * callback #1 happens, dep_count decremented to 1 + * since dep_count > 0, no completion is done + * callback #2 is added, dep_count is increased to 2 + * dep_count decremented to 1 + * callback #2 happens, dep_count decremented to 0 + * since dep_count now is zero, completion executes + * + * The dep_count can also be used to make sure that the completion only + * executes once. This is typically done by setting dep_count to -1 for the + * thread that takes on this responsibility. + */ +static inline void +kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) +{ + atomic_set(&katom->dma_fence.dep_count, val); +} + +/** + * kbase_fence_dep_count_dec_and_test() - Decrements dep_count + * @katom: Atom to decrement dep_count for + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: true if value was decremented to zero, otherwise false + */ +static inline bool +kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) +{ + return atomic_dec_and_test(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_dep_count_read() - Returns the current dep_count value + * @katom: Pointer to katom + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: The current dep_count value + */ +static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) +{ + return atomic_read(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom + * @katom: Pointer to katom + * + * This function will free all fence callbacks on the katom's list of + * callbacks. Callbacks that have not yet been called, because their fence + * hasn't yet signaled, will first be removed from the fence. + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + * + * Return: true if dep_count reached 0, otherwise false. + */ +bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_in_get() - Retrieve input fence for atom. + * @katom: Atom to get input fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no input fence for atom + */ +#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) +#endif + +/** + * kbase_fence_out_get() - Retrieve output fence for atom. + * @katom: Atom to get output fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no output fence for atom + */ +#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) + +/** + * kbase_fence_put() - Releases a reference to a fence + * @fence: Fence to release reference for. + */ +#define kbase_fence_put(fence) dma_fence_put(fence) + + +#endif /* CONFIG_MALI_DMA_FENCE || defined(CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_H_ */ diff --git a/mali_kbase/mali_kbase_fence_defs.h b/mali_kbase/mali_kbase_fence_defs.h new file mode 100644 index 0000000..fa2c6df --- /dev/null +++ b/mali_kbase/mali_kbase_fence_defs.h @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_DEFS_H_ +#define _KBASE_FENCE_DEFS_H_ + +/* + * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) + * This file hides the compatibility issues with this for the rest the driver + */ + +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include <linux/version.h> + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + +#include <linux/fence.h> + +#define dma_fence_context_alloc(a) fence_context_alloc(a) +#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) +#define dma_fence_get(a) fence_get(a) +#define dma_fence_put(a) fence_put(a) +#define dma_fence_signal(a) fence_signal(a) +#define dma_fence_is_signaled(a) fence_is_signaled(a) +#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) +#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) + +#else + +#include <linux/dma-fence.h> + +#endif /* < 4.10.0 */ + +#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_DEFS_H_ */ diff --git a/mali_kbase/mali_kbase_gator_hwcnt_names.h b/mali_kbase/mali_kbase_gator_hwcnt_names.h index e4003f6..cad19b6 100644 --- a/mali_kbase/mali_kbase_gator_hwcnt_names.h +++ b/mali_kbase/mali_kbase_gator_hwcnt_names.h @@ -2163,4 +2163,8 @@ static const char * const hardware_counters_mali_t88x[] = { #include "mali_kbase_gator_hwcnt_names_tsix.h" +#ifdef MALI_INCLUDE_TKAX +#include "mali_kbase_gator_hwcnt_names_tkax.h" +#endif /* MALI_INCLUDE_TKAX */ + #endif diff --git a/mali_kbase/mali_kbase_gpu_id.h b/mali_kbase/mali_kbase_gpu_id.h index 60b4371..42f0111 100644 --- a/mali_kbase/mali_kbase_gpu_id.h +++ b/mali_kbase/mali_kbase_gpu_id.h @@ -101,6 +101,12 @@ #define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) #define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) #define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) +#ifdef MALI_INCLUDE_TKAX +#define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(9u, 0) +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX +#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(10u, 0) +#endif /* MALI_INCLUDE_TTRX */ /* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ #define GPU_ID_S_15DEV0 0x1 diff --git a/mali_kbase/mali_kbase_gpuprops.c b/mali_kbase/mali_kbase_gpuprops.c index fc65223..e2f4209 100644 --- a/mali_kbase/mali_kbase_gpuprops.c +++ b/mali_kbase/mali_kbase_gpuprops.c @@ -26,6 +26,7 @@ #include <mali_kbase_gpuprops.h> #include <mali_kbase_config_defaults.h> #include <mali_kbase_hwaccess_gpuprops.h> +#include "mali_kbase_ioctl.h" #include <linux/clk.h> /** @@ -214,6 +215,14 @@ static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kb gpu_props->raw_props.thread_features = regdump.thread_features; } +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) +{ + gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); + gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); + gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); + gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); +} + /** * kbase_gpuprops_calculate_props - Calculate the derived properties * @gpu_props: The &base_gpu_props structure @@ -227,10 +236,7 @@ static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, str int i; /* Populate the base_gpu_props structure */ - gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); - gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); - gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); - gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); + kbase_gpuprops_update_core_props_gpu_id(gpu_props); gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT; @@ -327,3 +333,178 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev) gpu_props->raw_props.coherency_mode = regdump.coherency_features | COHERENCY_FEATURE_BIT(COHERENCY_NONE); } + +static struct { + u32 type; + size_t offset; + int size; +} gpu_property_mapping[] = { +#define PROP(name, member) \ + {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ + sizeof(((struct mali_base_gpu_props *)0)->member)} + PROP(PRODUCT_ID, core_props.product_id), + PROP(VERSION_STATUS, core_props.version_status), + PROP(MINOR_REVISION, core_props.minor_revision), + PROP(MAJOR_REVISION, core_props.major_revision), + PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), + PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), + PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), + PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), + PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), + PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), + PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), + PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), + + PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), + PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), + PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), + + PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), + PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), + + PROP(MAX_THREADS, thread_props.max_threads), + PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), + PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), + PROP(MAX_REGISTERS, thread_props.max_registers), + PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), + PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), + PROP(IMPL_TECH, thread_props.impl_tech), + + PROP(RAW_SHADER_PRESENT, raw_props.shader_present), + PROP(RAW_TILER_PRESENT, raw_props.tiler_present), + PROP(RAW_L2_PRESENT, raw_props.l2_present), + PROP(RAW_STACK_PRESENT, raw_props.stack_present), + PROP(RAW_L2_FEATURES, raw_props.l2_features), + PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), + PROP(RAW_MEM_FEATURES, raw_props.mem_features), + PROP(RAW_MMU_FEATURES, raw_props.mmu_features), + PROP(RAW_AS_PRESENT, raw_props.as_present), + PROP(RAW_JS_PRESENT, raw_props.js_present), + PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), + PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), + PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), + PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), + PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), + PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), + PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), + PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), + PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), + PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), + PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), + PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), + PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), + PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), + PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), + PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), + PROP(RAW_TILER_FEATURES, raw_props.tiler_features), + PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), + PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), + PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), + PROP(RAW_GPU_ID, raw_props.gpu_id), + PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), + PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, + raw_props.thread_max_workgroup_size), + PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), + PROP(RAW_THREAD_FEATURES, raw_props.thread_features), + PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), + + PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), + PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), + PROP(COHERENCY_COHERENCY, coherency_info.coherency), + PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), + PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), + PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), + PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), + PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), + PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), + PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), + PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), + PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), + PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), + PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), + PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), + PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), + PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), + PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), + PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), + +#undef PROP +}; + +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *kprops = &kbdev->gpu_props; + struct mali_base_gpu_props *props = &kprops->props; + u32 count = ARRAY_SIZE(gpu_property_mapping); + u32 i; + u32 size = 0; + u8 *p; + + for (i = 0; i < count; i++) { + /* 4 bytes for the ID, and the size of the property */ + size += 4 + gpu_property_mapping[i].size; + } + + kprops->prop_buffer_size = size; + kprops->prop_buffer = kmalloc(size, GFP_KERNEL); + + if (!kprops->prop_buffer) { + kprops->prop_buffer_size = 0; + return -ENOMEM; + } + + p = kprops->prop_buffer; + +#define WRITE_U8(v) (*p++ = (v) & 0xFF) +#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) +#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) +#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) + + for (i = 0; i < count; i++) { + u32 type = gpu_property_mapping[i].type; + u8 type_size; + void *field = ((u8 *)props) + gpu_property_mapping[i].offset; + + switch (gpu_property_mapping[i].size) { + case 1: + type_size = KBASE_GPUPROP_VALUE_SIZE_U8; + break; + case 2: + type_size = KBASE_GPUPROP_VALUE_SIZE_U16; + break; + case 4: + type_size = KBASE_GPUPROP_VALUE_SIZE_U32; + break; + case 8: + type_size = KBASE_GPUPROP_VALUE_SIZE_U64; + break; + default: + dev_err(kbdev->dev, + "Invalid gpu_property_mapping type=%d size=%d", + type, gpu_property_mapping[i].size); + return -EINVAL; + } + + WRITE_U32((type<<2) | type_size); + + switch (type_size) { + case KBASE_GPUPROP_VALUE_SIZE_U8: + WRITE_U8(*((u8 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U16: + WRITE_U16(*((u16 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U32: + WRITE_U32(*((u32 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U64: + WRITE_U64(*((u64 *)field)); + break; + default: /* Cannot be reached */ + WARN_ON(1); + return -EINVAL; + } + } + + return 0; +} diff --git a/mali_kbase/mali_kbase_gpuprops.h b/mali_kbase/mali_kbase_gpuprops.h index 44b5e9e..57b3eaf 100644 --- a/mali_kbase/mali_kbase_gpuprops.h +++ b/mali_kbase/mali_kbase_gpuprops.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2015, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -61,4 +61,24 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev); */ int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); +/** + * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer + * @kbdev: The kbase device + * + * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user + * space to read. + */ +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value + * @gpu_props: the &base_gpu_props structure + * + * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into + * separate fields (version_status, minor_revision, major_revision, product_id) + * stored in base_gpu_props::core_props. + */ +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); + + #endif /* _KBASE_GPUPROPS_H_ */ diff --git a/mali_kbase/mali_kbase_gpuprops_types.h b/mali_kbase/mali_kbase_gpuprops_types.h index b06e511..10794fc 100644 --- a/mali_kbase/mali_kbase_gpuprops_types.h +++ b/mali_kbase/mali_kbase_gpuprops_types.h @@ -84,6 +84,9 @@ struct kbase_gpu_props { /* Properties shared with userspace */ base_gpu_props props; + + u32 prop_buffer_size; + void *prop_buffer; }; #endif /* _KBASE_GPUPROPS_TYPES_H_ */ diff --git a/mali_kbase/mali_kbase_hw.c b/mali_kbase/mali_kbase_hw.c index 89fe0d8..9a390d2 100644 --- a/mali_kbase/mali_kbase_hw.c +++ b/mali_kbase/mali_kbase_hw.c @@ -48,6 +48,16 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) case GPU_ID2_PRODUCT_TSIX: features = base_hw_features_tSIx; break; +#ifdef MALI_INCLUDE_TKAX + case GPU_ID2_PRODUCT_TKAX: + features = base_hw_features_tKAx; + break; +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX + case GPU_ID2_PRODUCT_TTRX: + features = base_hw_features_tTRx; + break; +#endif /* MALI_INCLUDE_TTRX */ default: features = base_hw_features_generic; break; @@ -87,6 +97,142 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) set_bit(*features, &kbdev->hw_features_mask[0]); } +/** + * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID + * @kbdev: Device pointer + * + * Return: pointer to an array of hardware issues, terminated by + * BASE_HW_ISSUE_END. + * + * This function can only be used on new-format GPU IDs, i.e. those for which + * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU will + * be treated as the most recent known version not later than the actual + * version. In such circumstances, the GPU ID in @kbdev will also be replaced + * with the most recent known version. + * + * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() + * before calling this function. + */ +static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( + struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues = NULL; + + struct base_hw_product { + u32 product_model; + struct { + u32 version; + const enum base_hw_issue *issues; + } map[7]; + }; + + static const struct base_hw_product base_hw_products[] = { + {GPU_ID2_PRODUCT_TMIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 1), + base_hw_issues_tMIx_r0p0_05dev0}, + {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, + {U32_MAX /* sentinel value */, NULL} } }, + + {GPU_ID2_PRODUCT_THEX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TSIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 0, 1), base_hw_issues_tSIx_r1p0}, + {U32_MAX, NULL} } }, + + +#ifdef MALI_INCLUDE_TKAX + {GPU_ID2_PRODUCT_TKAX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tKAx_r0p0}, + {U32_MAX, NULL} } }, +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX + {GPU_ID2_PRODUCT_TTRX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, + {U32_MAX, NULL} } }, +#endif /* MALI_INCLUDE_TTRX */ + }; + + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; + const struct base_hw_product *product = NULL; + size_t p; + + /* Stop when we reach the end of the products array. */ + for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { + if (product_model == base_hw_products[p].product_model) { + product = &base_hw_products[p]; + break; + } + } + + if (product != NULL) { + /* Found a matching product. */ + const u32 version = gpu_id & GPU_ID2_VERSION; + u32 fallback_version = 0; + const enum base_hw_issue *fallback_issues = NULL; + size_t v; + + /* Stop when we reach the end of the map. */ + for (v = 0; product->map[v].version != U32_MAX; ++v) { + + if (version == product->map[v].version) { + /* Exact match so stop. */ + issues = product->map[v].issues; + break; + } + + /* Check whether this is a candidate for most recent + known version not later than the actual + version. */ + if ((version > product->map[v].version) && + (product->map[v].version >= fallback_version)) { + fallback_version = product->map[v].version; + fallback_issues = product->map[v].issues; + } + } + + if ((issues == NULL) && (fallback_issues != NULL)) { + /* Fall back to the issue set of the most recent known + version not later than the actual version. */ + issues = fallback_issues; + + dev_info(kbdev->dev, + "r%dp%d status %d is unknown; treating as r%dp%d status %d", + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT, + (fallback_version & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + + gpu_id &= ~GPU_ID2_VERSION; + gpu_id |= fallback_version; + kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; + + kbase_gpuprops_update_core_props_gpu_id(&kbdev->gpu_props.props); + } + } + return issues; +} + int kbase_hw_set_issues_mask(struct kbase_device *kbdev) { const enum base_hw_issue *issues; @@ -101,46 +247,17 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) if (impl_tech != IMPLEMENTATION_MODEL) { if (GPU_ID_IS_NEW_FORMAT(product_id)) { - switch (gpu_id) { - case GPU_ID2_MAKE(6, 0, 10, 0, 0, 0, 1): - issues = base_hw_issues_tMIx_r0p0_05dev0; - break; - case GPU_ID2_MAKE(6, 0, 10, 0, 0, 0, 2): - issues = base_hw_issues_tMIx_r0p0; - break; - default: - switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { - case GPU_ID2_PRODUCT_TMIX: - issues = base_hw_issues_tMIx_r0p0; - break; - case GPU_ID2_PRODUCT_THEX: - issues = base_hw_issues_tHEx_r0p0; - break; - case GPU_ID2_PRODUCT_TSIX: - switch (gpu_id & (GPU_ID2_VERSION_MAJOR | - GPU_ID2_VERSION_MINOR)) { - case GPU_ID2_VERSION_MAKE(0, 0, 0): - issues = base_hw_issues_tSIx_r0p0; - break; - case GPU_ID2_VERSION_MAKE(0, 1, 0): - issues = base_hw_issues_tSIx_r0p1; - break; - case GPU_ID2_VERSION_MAKE(1, 0, 0): - issues = base_hw_issues_tSIx_r1p0; - break; - default: - dev_err(kbdev->dev, - "Unknown GPU ID %x", - gpu_id); - return -EINVAL; - } - break; - default: - dev_err(kbdev->dev, - "Unknown GPU ID %x", gpu_id); - return -EINVAL; - } + issues = kbase_hw_get_issues_for_new_id(kbdev); + if (issues == NULL) { + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; } + + /* The GPU ID might have been replaced with the last + known version of the same GPU. */ + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + } else { switch (gpu_id) { case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): @@ -250,6 +367,16 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) case GPU_ID2_PRODUCT_TSIX: issues = base_hw_issues_model_tSIx; break; +#ifdef MALI_INCLUDE_TKAX + case GPU_ID2_PRODUCT_TKAX: + issues = base_hw_issues_model_tKAx; + break; +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX + case GPU_ID2_PRODUCT_TTRX: + issues = base_hw_issues_model_tTRx; + break; +#endif /* MALI_INCLUDE_TTRX */ default: dev_err(kbdev->dev, "Unknown GPU ID %x", gpu_id); @@ -289,7 +416,35 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) } } - dev_info(kbdev->dev, "GPU identified as 0x%04x r%dp%d status %d", (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, (gpu_id & GPU_ID_VERSION_STATUS) >> GPU_ID_VERSION_STATUS_SHIFT); + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + dev_info(kbdev->dev, + "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", + (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> + GPU_ID2_PRODUCT_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MAJOR) >> + GPU_ID2_ARCH_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MINOR) >> + GPU_ID2_ARCH_MINOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_REV) >> + GPU_ID2_ARCH_REV_SHIFT, + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + } else { + dev_info(kbdev->dev, + "GPU identified as 0x%04x r%dp%d status %d", + (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT, + (gpu_id & GPU_ID_VERSION_MAJOR) >> + GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> + GPU_ID_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID_VERSION_STATUS) >> + GPU_ID_VERSION_STATUS_SHIFT); + } for (; *issues != BASE_HW_ISSUE_END; issues++) set_bit(*issues, &kbdev->hw_issues_mask[0]); diff --git a/mali_kbase/mali_kbase_hw.h b/mali_kbase/mali_kbase_hw.h index fce7d29..754250c 100644 --- a/mali_kbase/mali_kbase_hw.h +++ b/mali_kbase/mali_kbase_hw.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,7 +40,20 @@ test_bit(feature, &(kbdev)->hw_features_mask[0]) /** - * @brief Set the HW issues mask depending on the GPU ID + * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID + * @kbdev: Device pointer + * + * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. + * + * The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU with a + * new-format ID will be treated as the most recent known version not later + * than the actual version. In such circumstances, the GPU ID in @kbdev will + * also be replaced with the most recent known version. + * + * Note: The GPU configuration must have been read by + * kbase_gpuprops_get_props() before calling this function. */ int kbase_hw_set_issues_mask(struct kbase_device *kbdev); diff --git a/mali_kbase/mali_kbase_hwaccess_jm.h b/mali_kbase/mali_kbase_hwaccess_jm.h index 821b68a..750fda2 100644 --- a/mali_kbase/mali_kbase_hwaccess_jm.h +++ b/mali_kbase/mali_kbase_hwaccess_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -45,33 +45,22 @@ void kbase_backend_run_atom(struct kbase_device *kbdev, void kbase_backend_slot_update(struct kbase_device *kbdev); /** - * kbase_backend_find_free_address_space() - Find a free address space. + * kbase_backend_find_and_release_free_address_space() - Release a free AS * @kbdev: Device pointer * @kctx: Context pointer * - * If no address spaces are currently free, then this function can evict an - * idle context from the runpool, freeing up the address space it was using. + * This function can evict an idle context from the runpool, freeing up the + * address space it was using. * * The address space is marked as in use. The caller must either assign a * context using kbase_gpu_use_ctx(), or release it using - * kbase_gpu_release_free_address_space() + * kbase_ctx_sched_release() * * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none * available */ -int kbase_backend_find_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx); - -/** - * kbase_backend_release_free_address_space() - Release an address space. - * @kbdev: Device pointer - * @as_nr: Address space to release - * - * The address space must have been returned by - * kbase_gpu_find_free_address_space(). - */ -void kbase_backend_release_free_address_space(struct kbase_device *kbdev, - int as_nr); +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx); /** * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the @@ -387,4 +376,6 @@ bool kbase_reset_gpu_active(struct kbase_device *kbdev); void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, struct kbase_jd_atom *target_katom); +extern struct protected_mode_ops kbase_native_protected_ops; + #endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/mali_kbase/mali_kbase_ioctl.h b/mali_kbase/mali_kbase_ioctl.h new file mode 100644 index 0000000..dcbed9c --- /dev/null +++ b/mali_kbase/mali_kbase_ioctl.h @@ -0,0 +1,656 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IOCTL_H_ +#define _KBASE_IOCTL_H_ + +#ifdef __cpluscplus +extern "C" { +#endif + +#include <linux/types.h> + +#define KBASE_IOCTL_TYPE 0x80 + +#ifdef ANDROID +/* Android's definition of ioctl is incorrect, specifying the type argument as + * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work + * round this by redefining _IOC to include a case to 'int'. + */ +#undef _IOC +#define _IOC(dir, type, nr, size) \ + ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) +#endif + +/** + * struct kbase_ioctl_version_check - Check version compatibility with kernel + * + * @major: Major version number + * @minor: Minor version number + */ +struct kbase_ioctl_version_check { + __u16 major; + __u16 minor; +}; + +#define KBASE_IOCTL_VERSION_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) + +/** + * struct kbase_ioctl_set_flags - Set kernel context creation flags + * + * @create_flags: Flags - see base_context_create_flags + */ +struct kbase_ioctl_set_flags { + __u32 create_flags; +}; + +#define KBASE_IOCTL_SET_FLAGS \ + _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) + +/** + * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel + * + * @addr: Memory address of an array of struct base_jd_atom_v2 + * @nr_atoms: Number of entries in the array + * @stride: sizeof(struct base_jd_atom_v2) + */ +struct kbase_ioctl_job_submit { + union kbase_pointer addr; + __u32 nr_atoms; + __u32 stride; +}; + +#define KBASE_IOCTL_JOB_SUBMIT \ + _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) + +/** + * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel + * + * @buffer: Pointer to the buffer to store properties into + * @size: Size of the buffer + * @flags: Flags - must be zero for now + * + * The ioctl will return the number of bytes stored into @buffer or an error + * on failure (e.g. @size is too small). If @size is specified as 0 then no + * data will be written but the return value will be the number of bytes needed + * for all the properties. + * + * @flags may be used in the future to request a different format for the + * buffer. With @flags == 0 the following format is used. + * + * The buffer will be filled with pairs of values, a u32 key identifying the + * property followed by the value. The size of the value is identified using + * the bottom bits of the key. The value then immediately followed the key and + * is tightly packed (there is no padding). All keys and values are + * little-endian. + * + * 00 = u8 + * 01 = u16 + * 10 = u32 + * 11 = u64 + */ +struct kbase_ioctl_get_gpuprops { + union kbase_pointer buffer; + __u32 size; + __u32 flags; +}; + +#define KBASE_IOCTL_GET_GPUPROPS \ + _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) + +#define KBASE_IOCTL_POST_TERM \ + _IO(KBASE_IOCTL_TYPE, 4) + +/** + * union kbase_ioctl_mem_alloc - Allocate memory on the GPU + * + * @va_pages: The number of pages of virtual address space to reserve + * @commit_pages: The number of physical pages to allocate + * @extent: The number of extra pages to allocate on each GPU fault which grows + * the region + * @flags: Flags + * @gpu_va: The GPU virtual address which is allocated + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alloc { + struct { + __u64 va_pages; + __u64 commit_pages; + __u64 extent; + __u64 flags; + } in; + struct { + __u64 flags; + __u64 gpu_va; + } out; +}; + +#define KBASE_IOCTL_MEM_ALLOC \ + _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) + +/** + * struct kbase_ioctl_mem_query - Query properties of a GPU memory region + * @gpu_addr: A GPU address contained within the region + * @query: The type of query + * @value: The result of the query + * + * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_query { + struct { + __u64 gpu_addr; + __u64 query; + } in; + struct { + __u64 value; + } out; +}; + +#define KBASE_IOCTL_MEM_QUERY \ + _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) + +#define KBASE_MEM_QUERY_COMMIT_SIZE 1 +#define KBASE_MEM_QUERY_VA_SIZE 2 +#define KBASE_MEM_QUERY_FLAGS 3 + +/** + * struct kbase_ioctl_mem_free - Free a memory region + * @gpu_addr: Handle to the region to free + */ +struct kbase_ioctl_mem_free { + __u64 gpu_addr; +}; + +#define KBASE_IOCTL_MEM_FREE \ + _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) + +/** + * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader + * @buffer_count: requested number of dumping buffers + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * + * A fd is returned from the ioctl if successful, or a negative value on error + */ +struct kbase_ioctl_hwcnt_reader_setup { + __u32 buffer_count; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_READER_SETUP \ + _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) + +/** + * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection + * @dump_buffer: GPU address to write counters to + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + */ +struct kbase_ioctl_hwcnt_enable { + __u64 dump_buffer; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_ENABLE \ + _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) + +#define KBASE_IOCTL_HWCNT_DUMP \ + _IO(KBASE_IOCTL_TYPE, 10) + +#define KBASE_IOCTL_HWCNT_CLEAR \ + _IO(KBASE_IOCTL_TYPE, 11) + +/** + * struct kbase_ioctl_disjoint_query - Query the disjoint counter + * @counter: A counter of disjoint events in the kernel + */ +struct kbase_ioctl_disjoint_query { + __u32 counter; +}; + +#define KBASE_IOCTL_DISJOINT_QUERY \ + _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) + +/** + * struct kbase_ioctl_get_ddk_version - Query the kernel version + * @version_buffer: Buffer to receive the kernel version string + * @size: Size of the buffer + * + * The ioctl will return the number of bytes written into version_buffer + * (which includes a NULL byte) or a negative error code + */ +struct kbase_ioctl_get_ddk_version { + union kbase_pointer version_buffer; + __u32 size; +}; + +#define KBASE_IOCTL_GET_DDK_VERSION \ + _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) + +/** + * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator + * + * @va_pages: Number of VA pages to reserve for JIT + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + */ +struct kbase_ioctl_mem_jit_init { + __u64 va_pages; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) + +/** + * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * + * @handle: GPU memory handle (GPU VA) + * @user_addr: The address where it is mapped in user space + * @size: The number of bytes to synchronise + * @type: The direction to synchronise: 0 is sync to memory (clean), + * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_mem_sync { + __u64 handle; + __u64 user_addr; + __u64 size; + __u8 type; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_MEM_SYNC \ + _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) + +/** + * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer + * + * @gpu_addr: The GPU address of the memory region + * @cpu_addr: The CPU address to locate + * @size: A size in bytes to validate is contained within the region + * @offset: The offset from the start of the memory region to @cpu_addr + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_find_cpu_offset { + struct { + __u64 gpu_addr; + __u64 cpu_addr; + __u64 size; + } in; + struct { + __u64 offset; + } out; +}; + +#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ + _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) + +/** + * struct kbase_ioctl_get_context_id - Get the kernel context ID + * + * @id: The kernel context ID + */ +struct kbase_ioctl_get_context_id { + int id; /* This should really be __u32, but see GPUCORE-10048 */ +}; + +#define KBASE_IOCTL_GET_CONTEXT_ID \ + _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) + +/** + * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd + * + * @flags: Flags + * + * The ioctl returns a file descriptor when successful + */ +struct kbase_ioctl_tlstream_acquire { + __u32 flags; +}; + +#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ + _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) + +#define KBASE_IOCTL_TLSTREAM_FLUSH \ + _IO(KBASE_IOCTL_TYPE, 19) + +/** + * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region + * + * @gpu_addr: The memory region to modify + * @pages: The number of physical pages that should be present + * + * The ioctl may return on the following error codes or 0 for success: + * -ENOMEM: Out of memory + * -EINVAL: Invalid arguments + */ +struct kbase_ioctl_mem_commit { + __u64 gpu_addr; + __u64 pages; +}; + +#define KBASE_IOCTL_MEM_COMMIT \ + _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) + +/** + * union kbase_ioctl_mem_alias - Create an alias of memory regions + * @flags: Flags, see BASE_MEM_xxx + * @stride: Bytes between start of each memory region + * @nents: The number of regions to pack together into the alias + * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alias { + struct { + __u64 flags; + __u64 stride; + __u64 nents; + union kbase_pointer aliasing_info; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_ALIAS \ + _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) + +/** + * union kbase_ioctl_mem_import - Import memory for use by the GPU + * @flags: Flags, see BASE_MEM_xxx + * @phandle: Handle to the external memory + * @type: Type of external memory, see base_mem_import_type + * @padding: Amount of extra VA pages to append to the imported buffer + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_import { + struct { + __u64 flags; + union kbase_pointer phandle; + __u32 type; + __u32 padding; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_IMPORT \ + _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) + +/** + * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region + * @gpu_va: The GPU region to modify + * @flags: The new flags to set + * @mask: Mask of the flags to modify + */ +struct kbase_ioctl_mem_flags_change { + __u64 gpu_va; + __u64 flags; + __u64 mask; +}; + +#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ + _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) + +/** + * struct kbase_ioctl_stream_create - Create a synchronisation stream + * @name: A name to identify this stream. Must be NULL-terminated. + * + * Note that this is also called a "timeline", but is named stream to avoid + * confusion with other uses of the word. + * + * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. + * + * The ioctl returns a file descriptor. + */ +struct kbase_ioctl_stream_create { + char name[32]; +}; + +#define KBASE_IOCTL_STREAM_CREATE \ + _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) + +/** + * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence + * @fd: The file descriptor to validate + */ +struct kbase_ioctl_fence_validate { + int fd; +}; + +#define KBASE_IOCTL_FENCE_VALIDATE \ + _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) + +/** + * struct kbase_ioctl_get_profiling_controls - Get the profiling controls + * @count: The size of @buffer in u32 words + * @buffer: The buffer to receive the profiling controls + */ +struct kbase_ioctl_get_profiling_controls { + union kbase_pointer buffer; + __u32 count; +}; + +#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ + _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) + +/** + * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel + * @buffer: Pointer to the information + * @len: Length + * @padding: Padding + * + * The data provided is accessible through a debugfs file + */ +struct kbase_ioctl_mem_profile_add { + union kbase_pointer buffer; + __u32 len; + __u32 padding; +}; + +#define KBASE_IOCTL_MEM_PROFILE_ADD \ + _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) + +/** + * struct kbase_ioctl_soft_event_update - Update the status of a soft-event + * @event: GPU address of the event which has been updated + * @new_status: The new status to set + * @flags: Flags for future expansion + */ +struct kbase_ioctl_soft_event_update { + __u64 event; + __u32 new_status; + __u32 flags; +}; + +#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ + _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) + +/*************** + * test ioctls * + ***************/ +#if MALI_UNIT_TEST +/* These ioctls are purely for test purposes and are not used in the production + * driver, they therefore may change without notice + */ + +#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) + +/** + * struct kbase_ioctl_tlstream_test - Start a timeline stream test + * + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + */ +struct kbase_ioctl_tlstream_test { + __u32 tpw_count; + __u32 msg_delay; + __u32 msg_count; + __u32 aux_msg; +}; + +#define KBASE_IOCTL_TLSTREAM_TEST \ + _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) + +/** + * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + */ +struct kbase_ioctl_tlstream_stats { + __u32 bytes_collected; + __u32 bytes_generated; +}; + +#define KBASE_IOCTL_TLSTREAM_STATS \ + _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) + +#endif + +/********************************** + * Definitions for GPU properties * + **********************************/ +#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) +#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) +#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) +#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) + +#define KBASE_GPUPROP_PRODUCT_ID 1 +#define KBASE_GPUPROP_VERSION_STATUS 2 +#define KBASE_GPUPROP_MINOR_REVISION 3 +#define KBASE_GPUPROP_MAJOR_REVISION 4 +#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 +#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 +#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 +#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 +#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 +#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 + +#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 +#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 +#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 + +#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 +#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 + +#define KBASE_GPUPROP_MAX_THREADS 18 +#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 +#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 +#define KBASE_GPUPROP_MAX_REGISTERS 21 +#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 +#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 +#define KBASE_GPUPROP_IMPL_TECH 24 + +#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 +#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 +#define KBASE_GPUPROP_RAW_L2_PRESENT 27 +#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 +#define KBASE_GPUPROP_RAW_L2_FEATURES 29 +#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 +#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 +#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 +#define KBASE_GPUPROP_RAW_AS_PRESENT 33 +#define KBASE_GPUPROP_RAW_JS_PRESENT 34 +#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 +#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 +#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 +#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 +#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 +#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 +#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 +#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 +#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 +#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 +#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 +#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 +#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 +#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 +#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 +#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 +#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 +#define KBASE_GPUPROP_RAW_GPU_ID 55 +#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 +#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 +#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 +#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 +#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 + +#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 +#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 +#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 +#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 +#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 +#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 +#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 +#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 +#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 +#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 +#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 +#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 +#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 +#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 +#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 +#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 +#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 +#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 +#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/mali_kbase/mali_kbase_jd.c b/mali_kbase/mali_kbase_jd.c index c2a5571..f39f1b0 100644 --- a/mali_kbase/mali_kbase_jd.c +++ b/mali_kbase/mali_kbase_jd.c @@ -596,7 +596,7 @@ static inline void jd_resolve_dep(struct list_head *out_list, #ifdef CONFIG_MALI_DMA_FENCE int dep_count; - dep_count = atomic_read(&dep_atom->dma_fence.dep_count); + dep_count = kbase_fence_dep_count_read(dep_atom); if (likely(dep_count == -1)) { dep_satisfied = true; } else { @@ -735,8 +735,8 @@ static void jd_try_submitting_deps(struct list_head *out_list, #ifdef CONFIG_MALI_DMA_FENCE int dep_count; - dep_count = atomic_read( - &dep_atom->dma_fence.dep_count); + dep_count = kbase_fence_dep_count_read( + dep_atom); if (likely(dep_count == -1)) { dep_satisfied = true; } else { @@ -975,7 +975,11 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ jctx->job_nr++; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) katom->start_timestamp.tv64 = 0; +#else + katom->start_timestamp = 0; +#endif katom->udata = user_atom->udata; katom->kctx = kctx; katom->nr_extres = user_atom->nr_extres; @@ -1007,7 +1011,7 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us katom->kds_rset = NULL; #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE - atomic_set(&katom->dma_fence.dep_count, -1); + kbase_fence_dep_count_set(katom, -1); #endif /* Don't do anything if there is a mess up with dependencies. @@ -1233,7 +1237,7 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us #ifdef CONFIG_MALI_DMA_FENCE - if (atomic_read(&katom->dma_fence.dep_count) != -1) { + if (kbase_fence_dep_count_read(katom) != -1) { ret = false; goto out; } @@ -1270,26 +1274,20 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us return ret; } -#ifdef BASE_LEGACY_UK6_SUPPORT -int kbase_jd_submit(struct kbase_context *kctx, - const struct kbase_uk_job_submit *submit_data, - int uk6_atom) -#else int kbase_jd_submit(struct kbase_context *kctx, - const struct kbase_uk_job_submit *submit_data) -#endif /* BASE_LEGACY_UK6_SUPPORT */ + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom) { struct kbase_jd_context *jctx = &kctx->jctx; int err = 0; int i; bool need_to_try_schedule_context = false; struct kbase_device *kbdev; - void __user *user_addr; u32 latest_flush; /* - * kbase_jd_submit isn't expected to fail and so all errors with the jobs - * are reported by immediately falling them (through event system) + * kbase_jd_submit isn't expected to fail and so all errors with the + * jobs are reported by immediately failing them (through event system) */ kbdev = kctx->kbdev; @@ -1300,29 +1298,25 @@ int kbase_jd_submit(struct kbase_context *kctx, return -EINVAL; } -#ifdef BASE_LEGACY_UK6_SUPPORT - if ((uk6_atom && submit_data->stride != - sizeof(struct base_jd_atom_v2_uk6)) || - submit_data->stride != sizeof(base_jd_atom_v2)) { -#else - if (submit_data->stride != sizeof(base_jd_atom_v2)) { -#endif /* BASE_LEGACY_UK6_SUPPORT */ + if (stride != sizeof(base_jd_atom_v2)) { dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); return -EINVAL; } - user_addr = get_compat_pointer(kctx, &submit_data->addr); - - KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(submit_data->nr_atoms, &kctx->timeline.jd_atoms_in_flight)); + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, + &kctx->timeline.jd_atoms_in_flight)); /* All atoms submitted in this call have the same flush ID */ latest_flush = kbase_backend_get_current_flush_id(kbdev); - for (i = 0; i < submit_data->nr_atoms; i++) { + for (i = 0; i < nr_atoms; i++) { struct base_jd_atom_v2 user_atom; struct kbase_jd_atom *katom; #ifdef BASE_LEGACY_UK6_SUPPORT + BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) != + sizeof(base_jd_atom_v2)); + if (uk6_atom) { struct base_jd_atom_v2_uk6 user_atom_v6; base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA}; @@ -1332,7 +1326,7 @@ int kbase_jd_submit(struct kbase_context *kctx, err = -EINVAL; KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return( - submit_data->nr_atoms - i, + nr_atoms - i, &kctx->timeline.jd_atoms_in_flight)); break; } @@ -1364,14 +1358,17 @@ int kbase_jd_submit(struct kbase_context *kctx, user_atom.device_nr = user_atom_v6.device_nr; } else { #endif /* BASE_LEGACY_UK6_SUPPORT */ - if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) { - err = -EINVAL; - KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(submit_data->nr_atoms - i, &kctx->timeline.jd_atoms_in_flight)); - break; - } + if (copy_from_user(&user_atom, user_addr, + sizeof(user_atom)) != 0) { + err = -EINVAL; + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, + atomic_sub_return(nr_atoms - i, + &kctx->timeline.jd_atoms_in_flight)); + break; + } #ifdef BASE_LEGACY_UK6_SUPPORT } -#endif /* BASE_LEGACY_UK6_SUPPORT */ +#endif #ifdef BASE_LEGACY_UK10_2_SUPPORT if (KBASE_API_VERSION(10, 3) > kctx->api_version) @@ -1379,7 +1376,7 @@ int kbase_jd_submit(struct kbase_context *kctx, & 0x7fff); #endif /* BASE_LEGACY_UK10_2_SUPPORT */ - user_addr = (void __user *)((uintptr_t) user_addr + submit_data->stride); + user_addr = (void __user *)((uintptr_t) user_addr + stride); mutex_lock(&jctx->lock); #ifndef compiletime_assert @@ -1846,8 +1843,9 @@ int kbase_jd_init(struct kbase_context *kctx) kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; -#ifdef CONFIG_MALI_DMA_FENCE - kctx->jctx.atoms[i].dma_fence.context = fence_context_alloc(1); +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + kctx->jctx.atoms[i].dma_fence.context = + dma_fence_context_alloc(1); atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); #endif diff --git a/mali_kbase/mali_kbase_jd_debugfs.c b/mali_kbase/mali_kbase_jd_debugfs.c index 47ea426..c8b37c4 100644 --- a/mali_kbase/mali_kbase_jd_debugfs.c +++ b/mali_kbase/mali_kbase_jd_debugfs.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -15,63 +15,58 @@ -#include <linux/seq_file.h> +#ifdef CONFIG_DEBUG_FS +#include <linux/seq_file.h> #include <mali_kbase.h> - #include <mali_kbase_jd_debugfs.h> #include <mali_kbase_dma_fence.h> - -#ifdef CONFIG_DEBUG_FS +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include <mali_kbase_sync.h> +#endif struct kbase_jd_debugfs_depinfo { u8 id; char type; }; -#ifdef CONFIG_SYNC -#include <mali_kbase_sync.h> -#include "sync.h" -#endif /* CONFIG_SYNC */ - - static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, struct seq_file *sfile) { -#ifdef CONFIG_SYNC - struct sync_fence *fence = atom->fence; - int status; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_sync_fence_info info; + int res; switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - if (!fence) + res = kbase_sync_fence_out_info_get(atom, &info); + if (0 == res) { + seq_printf(sfile, "Sa([%p]%d) ", + info.fence, info.status); break; - - status = kbase_fence_get_status(fence); - - seq_printf(sfile, "Sa([%p]%d) ", - fence, status); - break; + } case BASE_JD_REQ_SOFT_FENCE_WAIT: - if (!fence) + res = kbase_sync_fence_in_info_get(atom, &info); + if (0 == res) { + seq_printf(sfile, "Wa([%p]%d) ", + info.fence, info.status); break; - - status = kbase_fence_get_status(fence); - - seq_printf(sfile, "Wa([%p]%d) ", - fence, status); - break; + } default: break; } -#endif /* CONFIG_SYNC */ +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ #ifdef CONFIG_MALI_DMA_FENCE if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { - struct kbase_dma_fence_cb *cb; + struct kbase_fence_cb *cb; if (atom->dma_fence.fence) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) struct fence *fence = atom->dma_fence.fence; +#else + struct dma_fence *fence = atom->dma_fence.fence; +#endif seq_printf(sfile, #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) @@ -81,13 +76,17 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, #endif fence->context, fence->seqno, - fence_is_signaled(fence) ? + dma_fence_is_signaled(fence) ? "signaled" : "active"); } list_for_each_entry(cb, &atom->dma_fence.callbacks, node) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) struct fence *fence = cb->fence; +#else + struct dma_fence *fence = cb->fence; +#endif seq_printf(sfile, #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) @@ -97,7 +96,7 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, #endif fence->context, fence->seqno, - fence_is_signaled(fence) ? + dma_fence_is_signaled(fence) ? "signaled" : "active"); } } diff --git a/mali_kbase/mali_kbase_js.c b/mali_kbase/mali_kbase_js.c index d75dcba..b053ce1 100644 --- a/mali_kbase/mali_kbase_js.c +++ b/mali_kbase/mali_kbase_js.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,6 +27,7 @@ #endif #include <mali_kbase_tlstream.h> #include <mali_kbase_hw.h> +#include <mali_kbase_ctx_sched.h> #include <mali_kbase_defs.h> #include <mali_kbase_config_defaults.h> @@ -82,46 +83,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) { - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - int as_nr; - int refcnt = 0; - - js_devdata = &kbdev->js_data; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kctx->as_nr; - if (as_nr != KBASEP_AS_NR_INVALID) { - struct kbasep_js_per_as_data *js_per_as_data; - - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - refcnt = js_per_as_data->as_busy_refcount; - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return refcnt; -} - -static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - int as_nr; - int refcnt = 0; - - js_devdata = &kbdev->js_data; - - as_nr = kctx->as_nr; - if (as_nr != KBASEP_AS_NR_INVALID) { - struct kbasep_js_per_as_data *js_per_as_data; - - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - refcnt = js_per_as_data->as_busy_refcount; - } - - return refcnt; + return atomic_read(&kctx->refcount); } #else /* KBASE_TRACE_ENABLE */ static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, @@ -131,13 +93,6 @@ static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, CSTD_UNUSED(kctx); return 0; } -static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - CSTD_UNUSED(kbdev); - CSTD_UNUSED(kctx); - return 0; -} #endif /* KBASE_TRACE_ENABLE */ /* @@ -181,12 +136,11 @@ static void kbase_js_sync_timers(struct kbase_device *kbdev) mutex_unlock(&kbdev->js_data.runpool_mutex); } -/* Hold the hwaccess_lock for this */ +/* Hold the mmu_hw_mutex and hwaccess_lock for this */ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx) { struct kbasep_js_device_data *js_devdata; - struct kbasep_js_per_as_data *js_per_as_data; bool result = false; int as_nr; @@ -194,19 +148,15 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, KBASE_DEBUG_ASSERT(kctx != NULL); js_devdata = &kbdev->js_data; - as_nr = kctx->as_nr; - if (as_nr != KBASEP_AS_NR_INVALID) { - int new_refcnt; + lockdep_assert_held(&kbdev->hwaccess_lock); + as_nr = kctx->as_nr; + if (atomic_read(&kctx->refcount) > 0) { KBASE_DEBUG_ASSERT(as_nr >= 0); - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - KBASE_DEBUG_ASSERT(js_per_as_data->kctx != NULL); - - new_refcnt = ++(js_per_as_data->as_busy_refcount); + kbase_ctx_sched_retain_ctx_refcount(kctx); KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, - NULL, 0u, new_refcnt); + NULL, 0u, atomic_read(&kctx->refcount)); result = true; } @@ -470,25 +420,11 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) { struct kbasep_js_device_data *jsdd; int i; - u16 as_present; KBASE_DEBUG_ASSERT(kbdev != NULL); jsdd = &kbdev->js_data; - /* These two must be recalculated if nr_hw_address_spaces changes - * (e.g. for HW workarounds) */ - as_present = (1U << kbdev->nr_hw_address_spaces) - 1; - kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { - bool use_workaround; - - use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; - if (use_workaround) { - dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); - kbdev->nr_user_address_spaces = 1; - } - } #ifdef CONFIG_MALI_DEBUG /* Soft-stop will be disabled on a single context by default unless * softstop_always is set */ @@ -498,8 +434,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) jsdd->nr_user_contexts_running = 0; jsdd->nr_contexts_pullable = 0; atomic_set(&jsdd->nr_contexts_runnable, 0); - /* All ASs initially free */ - jsdd->as_free = as_present; /* No ctx allowed to submit */ jsdd->runpool_irq.submit_allowed = 0u; memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, @@ -577,10 +511,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); #endif - /* Clear the AS data, including setting NULL pointers */ - memset(&jsdd->runpool_irq.per_as_data[0], 0, - sizeof(jsdd->runpool_irq.per_as_data)); - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) jsdd->js_reqs[i] = core_reqs_from_jsn_features( kbdev->gpu_props.props.raw_props.js_features[i]); @@ -1189,7 +1119,7 @@ bool kbasep_js_add_job(struct kbase_context *kctx, enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, - kbasep_js_trace_get_refcnt_nolock(kbdev, kctx)); + kbasep_js_trace_get_refcnt(kbdev, kctx)); /* Context Attribute Refcounting */ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); @@ -1303,11 +1233,11 @@ bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, KBASE_DEBUG_ASSERT(kbdev != NULL); js_devdata = &kbdev->js_data; - /* KBASE_TRACE_ADD_REFCOUNT( kbdev, JS_RETAIN_CTX, kctx, NULL, 0, - kbasep_js_trace_get_refcnt(kbdev, kctx)); */ + mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); return result; } @@ -1318,48 +1248,23 @@ struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, unsigned long flags; struct kbasep_js_device_data *js_devdata; struct kbase_context *found_kctx = NULL; - struct kbasep_js_per_as_data *js_per_as_data; KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); js_devdata = &kbdev->js_data; - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - found_kctx = js_per_as_data->kctx; + found_kctx = kbdev->as_to_kctx[as_nr]; if (found_kctx != NULL) - ++(js_per_as_data->as_busy_refcount); + kbase_ctx_sched_retain_ctx_refcount(found_kctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return found_kctx; } -struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock( - struct kbase_device *kbdev, int as_nr) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_context *found_kctx = NULL; - struct kbasep_js_per_as_data *js_per_as_data; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - js_devdata = &kbdev->js_data; - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - found_kctx = js_per_as_data->kctx; - - if (found_kctx != NULL) - ++(js_per_as_data->as_busy_refcount); - - return found_kctx; -} - /** * kbasep_js_release_result - Try running more jobs after releasing a context * and/or atom @@ -1445,7 +1350,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( unsigned long flags; struct kbasep_js_device_data *js_devdata; struct kbasep_js_kctx_info *js_kctx_info; - struct kbasep_js_per_as_data *js_per_as_data; kbasep_js_release_result release_result = 0u; bool runpool_ctx_attr_change = false; @@ -1461,16 +1365,9 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( /* Ensure context really is scheduled in */ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - /* kctx->as_nr and js_per_as_data are only read from here. The caller's - * js_ctx_mutex provides a barrier that ensures they are up-to-date. - * - * They will not change whilst we're reading them, because the refcount - * is non-zero (and we ASSERT on that last fact). - */ kctx_as_nr = kctx->as_nr; KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); - js_per_as_data = &js_devdata->runpool_irq.per_as_data[kctx_as_nr]; - KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); /* * Transaction begins on AS and runpool_irq @@ -1479,14 +1376,14 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( */ current_as = &kbdev->as[kctx_as_nr]; mutex_lock(&kbdev->pm.lock); - mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); - KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); /* Update refcount */ - new_ref_count = --(js_per_as_data->as_busy_refcount); + kbase_ctx_sched_release_ctx(kctx); + new_ref_count = atomic_read(&kctx->refcount); /* Release the atom if it finished (i.e. wasn't soft-stopped) */ if (kbasep_js_has_atom_finished(katom_retained_state)) @@ -1496,7 +1393,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, new_ref_count); - if (new_ref_count == 1 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && + if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && !kbase_pm_is_suspending(kbdev)) { /* Context is kept scheduled into an address space even when * there are no jobs, in this case we have to handle the @@ -1509,8 +1406,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( kbasep_js_set_submit_allowed(js_devdata, kctx); } - /* Make a set of checks to see if the context should be scheduled out */ - if (new_ref_count == 0 && + /* Make a set of checks to see if the context should be scheduled out. + * Note that there'll always be at least 1 reference to the context + * which was previously acquired by kbasep_js_schedule_ctx(). */ + if (new_ref_count == 1 && (!kbasep_js_is_submit_allowed(js_devdata, kctx) || kbdev->pm.suspending)) { int num_slots = kbdev->gpu_props.num_job_slots; @@ -1518,7 +1417,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( /* Last reference, and we've been told to remove this context * from the Run Pool */ - dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d", + dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, kbasep_js_is_submit_allowed(js_devdata, kctx)); @@ -1571,7 +1470,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( kbase_backend_release_ctx_noirq(kbdev, kctx); - mutex_unlock(&kbdev->mmu_hw_mutex); mutex_unlock(&kbdev->pm.lock); /* Note: Don't reuse kctx_as_nr now */ @@ -1594,7 +1492,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( katom_retained_state, runpool_ctx_attr_change); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); mutex_unlock(&kbdev->pm.lock); } @@ -1740,8 +1637,26 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, js_kctx_info = &kctx->jctx.sched_info; /* Pick available address space for this context */ - as_nr = kbase_backend_find_free_address_space(kbdev, kctx); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + if (as_nr == KBASEP_AS_NR_INVALID) { + as_nr = kbase_backend_find_and_release_free_address_space( + kbdev, kctx); + if (as_nr != KBASEP_AS_NR_INVALID) { + /* Attempt to retain the context again, this should + * succeed */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + WARN_ON(as_nr == KBASEP_AS_NR_INVALID); + } + } if (as_nr == KBASEP_AS_NR_INVALID) return false; /* No address spaces currently available */ @@ -1752,12 +1667,16 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, */ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* Check to see if context is dying due to kbase_job_zap_context() */ if (kbase_ctx_flag(kctx, KCTX_DYING)) { /* Roll back the transaction so far and return */ - kbase_backend_release_free_address_space(kbdev, as_nr); + kbase_ctx_sched_release_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); mutex_unlock(&js_devdata->runpool_mutex); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); @@ -1770,21 +1689,17 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - /* Assign context to previously chosen address space */ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - kbase_backend_release_free_address_space(kbdev, as_nr); - + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + return false; } @@ -1851,7 +1766,8 @@ static bool kbase_js_use_ctx(struct kbase_device *kbdev, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbase_backend_use_ctx_sched(kbdev, kctx)) { + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_backend_use_ctx_sched(kbdev, kctx)) { /* Context already has ASID - mark as active */ kbdev->hwaccess.active_kctx = kctx; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -1953,14 +1869,12 @@ void kbasep_js_suspend(struct kbase_device *kbdev) /* Retain each of the contexts, so we can cause it to leave even if it * had no refcount to begin with */ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { - struct kbasep_js_per_as_data *js_per_as_data = - &js_devdata->runpool_irq.per_as_data[i]; - struct kbase_context *kctx = js_per_as_data->kctx; + struct kbase_context *kctx = kbdev->as_to_kctx[i]; retained = retained << 1; if (kctx) { - ++(js_per_as_data->as_busy_refcount); + kbase_ctx_sched_retain_ctx_refcount(kctx); retained |= 1u; /* We can only cope with up to 1 privileged context - * the instrumented context. It'll be suspended by @@ -1980,9 +1894,7 @@ void kbasep_js_suspend(struct kbase_device *kbdev) for (i = 0; i < BASE_MAX_NR_AS; ++i, retained = retained >> 1) { - struct kbasep_js_per_as_data *js_per_as_data = - &js_devdata->runpool_irq.per_as_data[i]; - struct kbase_context *kctx = js_per_as_data->kctx; + struct kbase_context *kctx = kbdev->as_to_kctx[i]; if (retained & 1u) kbasep_js_runpool_release_ctx(kbdev, kctx); @@ -2189,16 +2101,19 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) { struct kbase_jd_atom *katom; struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; int pulled; KBASE_DEBUG_ASSERT(kctx); - js_devdata = &kctx->kbdev->js_data; - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + kbdev = kctx->kbdev; + + js_devdata = &kbdev->js_data; + lockdep_assert_held(&kbdev->hwaccess_lock); if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) return NULL; - if (kbase_pm_is_suspending(kctx->kbdev)) + if (kbase_pm_is_suspending(kbdev)) return NULL; katom = jsctx_rb_peek(kctx, js); @@ -2214,7 +2129,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) * present on the same slot */ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { struct kbase_jd_atom *prev_atom = - kbase_backend_inspect_tail(kctx->kbdev, js); + kbase_backend_inspect_tail(kbdev, js); if (prev_atom && prev_atom->kctx != kctx) return NULL; @@ -2226,7 +2141,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) katom->x_pre_dep->will_fail_event_code) return NULL; if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) + kbase_backend_nr_atoms_on_slot(kbdev, js)) return NULL; } @@ -2236,13 +2151,14 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) if (pulled == 1 && !kctx->slots_pullable) { WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kctx->kbdev->js_data.nr_contexts_runnable); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); } atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; jsctx_rb_pull(kctx, katom); - kbasep_js_runpool_retain_ctx_nolock(kctx->kbdev, kctx); + kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; katom->ticks = 0; @@ -2868,22 +2784,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) static inline int trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) { - struct kbasep_js_device_data *js_devdata; - int as_nr; - int refcnt = 0; - - js_devdata = &kbdev->js_data; - - as_nr = kctx->as_nr; - if (as_nr != KBASEP_AS_NR_INVALID) { - struct kbasep_js_per_as_data *js_per_as_data; - - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - refcnt = js_per_as_data->as_busy_refcount; - } - - return refcnt; + return atomic_read(&kctx->refcount); } /** diff --git a/mali_kbase/mali_kbase_js.h b/mali_kbase/mali_kbase_js.h index d092fff..ddada8e 100644 --- a/mali_kbase/mali_kbase_js.h +++ b/mali_kbase/mali_kbase_js.h @@ -227,7 +227,8 @@ bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, * @note This function can safely be called from IRQ context. * * The following locking conditions are made on the caller: - * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be + * used internally. * * @return value != false if the retain succeeded, and the context will not be scheduled out. * @return false if the retain failed (because the context is being/has been scheduled out). @@ -241,7 +242,7 @@ bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_conte * @note This function can safely be called from IRQ context. * * The following locks must be held by the caller: - * - hwaccess_lock + * - mmu_hw_mutex, hwaccess_lock * * @return value != false if the retain succeeded, and the context will not be scheduled out. * @return false if the retain failed (because the context is being/has been scheduled out). @@ -269,28 +270,6 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbas struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); /** - * kbasep_js_runpool_lookup_ctx_nolock - Lookup a context in the Run Pool based - * upon its current address space and ensure that is stays scheduled in. - * @kbdev: Device pointer - * @as_nr: Address space to lookup - * - * The context is refcounted as being busy to prevent it from scheduling - * out. It must be released with kbasep_js_runpool_release_ctx() when it is no - * longer required to stay scheduled in. - * - * Note: This function can safely be called from IRQ context. - * - * The following locking conditions are made on the caller: - * - it must the hold the hwaccess_lock - * - * Return: a valid struct kbase_context on success, which has been refcounted as - * being busy. - * NULL on failure, indicating that no context was found in \a as_nr - */ -struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock( - struct kbase_device *kbdev, int as_nr); - -/** * @brief Handling the requeuing/killing of a context that was evicted from the * policy queue or runpool. * @@ -782,41 +761,9 @@ static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_a return (bool) (js >= 0); } -#if KBASE_DEBUG_DISABLE_ASSERTS == 0 -/** - * Debug Check the refcount of a context. Only use within ASSERTs - * - * Obtains hwaccess_lock - * - * @return negative value if the context is not scheduled in - * @return current refcount of the context if it is scheduled in. The refcount - * is not guarenteed to be kept constant. - */ -static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - int result = -1; - int as_nr; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_devdata = &kbdev->js_data; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kctx->as_nr; - if (as_nr != KBASEP_AS_NR_INVALID) - result = js_devdata->runpool_irq.per_as_data[as_nr].as_busy_refcount; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return result; -} -#endif /* KBASE_DEBUG_DISABLE_ASSERTS == 0 */ - /** * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the - * context is guarenteed to be already previously retained. + * context is guaranteed to be already previously retained. * * It is a programming error to supply the \a as_nr of a context that has not * been previously retained/has a busy refcount of zero. The only exception is @@ -825,28 +772,20 @@ static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, * The following locking conditions are made on the caller: * - it must \em not hold the hwaccess_lock, because it will be used internally. * - * @return a valid struct kbase_context on success, with a refcount that is guarenteed + * @return a valid struct kbase_context on success, with a refcount that is guaranteed * to be non-zero and unmodified by this function. * @return NULL on failure, indicating that no context was found in \a as_nr */ static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) { - unsigned long flags; - struct kbasep_js_device_data *js_devdata; struct kbase_context *found_kctx; - struct kbasep_js_per_as_data *js_per_as_data; KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); - js_devdata = &kbdev->js_data; - js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - found_kctx = js_per_as_data->kctx; - KBASE_DEBUG_ASSERT(found_kctx == NULL || js_per_as_data->as_busy_refcount > 0); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + found_kctx = kbdev->as_to_kctx[as_nr]; + KBASE_DEBUG_ASSERT(found_kctx == NULL || + atomic_read(&found_kctx->refcount) > 0); return found_kctx; } diff --git a/mali_kbase/mali_kbase_js_defs.h b/mali_kbase/mali_kbase_js_defs.h index 6f5feba..ba8b644 100644 --- a/mali_kbase/mali_kbase_js_defs.h +++ b/mali_kbase/mali_kbase_js_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -142,26 +142,6 @@ enum { typedef u32 kbasep_js_atom_done_code; /** - * Data used by the scheduler that is unique for each Address Space. - * - * This is used in IRQ context and hwaccess_lock must be held whilst accessing - * this data (inculding reads and atomic decisions based on the read). - */ -struct kbasep_js_per_as_data { - /** - * Ref count of whether this AS is busy, and must not be scheduled out - * - * When jobs are running this is always positive. However, it can still be - * positive when no jobs are running. If all you need is a heuristic to - * tell you whether jobs might be running, this should be sufficient. - */ - int as_busy_refcount; - - /** Pointer to the current context on this address space, or NULL for no context */ - struct kbase_context *kctx; -}; - -/** * @brief KBase Device Data Job Scheduler sub-structure * * This encapsulates the current context of the Job Scheduler on a particular @@ -181,10 +161,8 @@ struct kbasep_js_device_data { struct runpool_irq { /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. * When bit 'N' is set in this, it indicates whether the context bound to address space - * 'N' (per_as_data[N].kctx) is allowed to submit jobs. - * - * It is placed here because it's much more memory efficient than having a u8 in - * struct kbasep_js_per_as_data to store this flag */ + * 'N' is allowed to submit jobs. + */ u16 submit_allowed; /** Context Attributes: @@ -203,9 +181,6 @@ struct kbasep_js_device_data { * is required on updating the variable) */ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; - /** Data that is unique for each AS */ - struct kbasep_js_per_as_data per_as_data[BASE_MAX_NR_AS]; - /* * Affinity management and tracking */ @@ -256,8 +231,6 @@ struct kbasep_js_device_data { */ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; - u16 as_free; /**< Bitpattern of free Address Spaces */ - /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ s8 nr_user_contexts_running; /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c index 20a18a9..e76294d 100644 --- a/mali_kbase/mali_kbase_mem.c +++ b/mali_kbase/mali_kbase_mem.c @@ -961,6 +961,9 @@ bad_insert: KBASE_EXPORT_TEST_API(kbase_gpu_mmap); +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable); + int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) { int err; @@ -981,6 +984,20 @@ int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); } + if (reg->gpu_alloc && reg->gpu_alloc->type == + KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + struct kbase_alloc_import_user_buf *user_buf = + ®->gpu_alloc->imported.user_buf; + + if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { + user_buf->current_mapping_usage_count &= + ~PINNED_ON_IMPORT; + + kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, + (reg->flags & KBASE_REG_GPU_WR)); + } + } + if (err) return err; @@ -994,6 +1011,10 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( { struct vm_area_struct *vma; struct kbase_cpu_mapping *map; + unsigned long vm_pgoff_in_region; + unsigned long vm_off_in_region; + unsigned long map_start; + size_t map_size; lockdep_assert_held(¤t->mm->mmap_sem); @@ -1014,8 +1035,16 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( /* Not from this context! */ return NULL; - *offset = (uaddr - vma->vm_start) + - ((vma->vm_pgoff - map->region->start_pfn)<<PAGE_SHIFT); + vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; + vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; + map_start = vma->vm_start - vm_off_in_region; + map_size = map->region->nr_pages << PAGE_SHIFT; + + if ((uaddr + size) > (map_start + map_size)) + /* Not within the CPU mapping */ + return NULL; + + *offset = (uaddr - vma->vm_start) + vm_off_in_region; return map; } @@ -1092,10 +1121,9 @@ void kbase_sync_single(struct kbase_context *kctx, } static int kbase_do_syncset(struct kbase_context *kctx, - struct base_syncset *set, enum kbase_sync_type sync_fn) + struct basep_syncset *sset, enum kbase_sync_type sync_fn) { int err = 0; - struct basep_syncset *sset = &set->basep_sset; struct kbase_va_region *reg; struct kbase_cpu_mapping *map; unsigned long start; @@ -1179,23 +1207,26 @@ out_unlock: return err; } -int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset) +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) { int err = -EINVAL; - struct basep_syncset *sset; - KBASE_DEBUG_ASSERT(NULL != kctx); - KBASE_DEBUG_ASSERT(NULL != syncset); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(sset != NULL); - sset = &syncset->basep_sset; + if (sset->mem_handle.basep.handle & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, + "mem_handle: passed parameter is invalid"); + return -EINVAL; + } switch (sset->type) { case BASE_SYNCSET_OP_MSYNC: - err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_DEVICE); + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); break; case BASE_SYNCSET_OP_CSYNC: - err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_CPU); + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); break; default: @@ -1266,6 +1297,11 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) KBASE_DEBUG_ASSERT(kctx != NULL); + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); + return -EINVAL; + } + if (0 == gpu_addr) { dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); return -EINVAL; @@ -2154,12 +2190,18 @@ static int kbase_jd_user_buf_map(struct kbase_context *kctx, alloc->imported.user_buf.nr_pages, reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -#else +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) pinned_pages = get_user_pages_remote(NULL, mm, address, alloc->imported.user_buf.nr_pages, reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, pages, NULL); +#else + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); #endif if (pinned_pages <= 0) @@ -2289,28 +2331,44 @@ static int kbase_jd_umm_map(struct kbase_context *kctx, alloc->imported.umm.dma_buf->size); } - if (WARN_ONCE(count < reg->nr_pages, + if (!(reg->flags & KBASE_REG_IMPORT_PAD) && + WARN_ONCE(count < reg->nr_pages, "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", alloc->imported.umm.dma_buf->size)) { err = -EINVAL; - goto out; + goto err_unmap_attachment; } /* Update nents as we now have pages to map */ - alloc->nents = count; + alloc->nents = reg->nr_pages; err = kbase_mmu_insert_pages(kctx, reg->start_pfn, kbase_get_gpu_phy_pages(reg), - kbase_reg_current_backed_size(reg), + count, reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); - -out: - if (err) { - dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, - alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); - alloc->imported.umm.sgt = NULL; + if (err) + goto err_unmap_attachment; + + if (reg->flags & KBASE_REG_IMPORT_PAD) { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + count, + page_to_phys(kctx->aliasing_sink_page), + reg->nr_pages - count, + (reg->flags | KBASE_REG_GPU_RD) & + ~KBASE_REG_GPU_WR); + if (err) + goto err_teardown_orig_pages; } + return 0; + +err_teardown_orig_pages: + kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); +err_unmap_attachment: + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + return err; } @@ -2434,11 +2492,15 @@ void kbase_unmap_external_resource(struct kbase_context *kctx, alloc->imported.umm.current_mapping_usage_count--; if (0 == alloc->imported.umm.current_mapping_usage_count) { - if (reg && reg->gpu_alloc == alloc) - kbase_mmu_teardown_pages( + if (reg && reg->gpu_alloc == alloc) { + int err; + + err = kbase_mmu_teardown_pages( kctx, reg->start_pfn, - kbase_reg_current_backed_size(reg)); + alloc->nents); + WARN_ON(err); + } kbase_jd_umm_unmap(kctx, alloc); } diff --git a/mali_kbase/mali_kbase_mem.h b/mali_kbase/mali_kbase_mem.h index eefb4a2..e293577 100644 --- a/mali_kbase/mali_kbase_mem.h +++ b/mali_kbase/mali_kbase_mem.h @@ -151,18 +151,31 @@ struct kbase_mem_phy_alloc { } alias; /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ struct kbase_context *kctx; - struct { + struct kbase_alloc_import_user_buf { unsigned long address; unsigned long size; unsigned long nr_pages; struct page **pages; - unsigned int current_mapping_usage_count; + /* top bit (1<<31) of current_mapping_usage_count + * specifies that this import was pinned on import + * See PINNED_ON_IMPORT + */ + u32 current_mapping_usage_count; struct mm_struct *mm; dma_addr_t *dma_addrs; } user_buf; } imported; }; +/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is + * used to signify that a buffer was pinned when it was imported. Since the + * reference count is limited by the number of atoms that can be submitted at + * once there should be no danger of overflowing into this bit. + * Stealing the top bit also has the benefit that + * current_mapping_usage_count != 0 if and only if the buffer is mapped. + */ +#define PINNED_ON_IMPORT (1<<31) + static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) { KBASE_DEBUG_ASSERT(alloc); @@ -255,6 +268,9 @@ struct kbase_va_region { #define KBASE_REG_DONT_NEED (1ul << 20) +/* Imported buffer is padded? */ +#define KBASE_REG_IMPORT_PAD (1ul << 21) + #define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) /* only used with 32-bit clients */ @@ -726,7 +742,16 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); */ void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); -int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset); +/** + * kbase_sync_now - Perform cache maintenance on a memory region + * + * @kctx: The kbase context of the region + * @sset: A syncset structure describing the region and direction of the + * synchronisation required + * + * Return: 0 on success or error code + */ +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa, phys_addr_t gpu_pa, off_t offset, size_t size, enum kbase_sync_type sync_fn); diff --git a/mali_kbase/mali_kbase_mem_linux.c b/mali_kbase/mali_kbase_mem_linux.c index 0b3a7da..eea429a 100644 --- a/mali_kbase/mali_kbase_mem_linux.c +++ b/mali_kbase/mali_kbase_mem_linux.c @@ -264,6 +264,11 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * c KBASE_DEBUG_ASSERT(kctx); KBASE_DEBUG_ASSERT(out); + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + kbase_gpu_vm_lock(kctx); /* Validate the region */ @@ -723,6 +728,9 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in if (!gpu_addr) return -EINVAL; + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) + return -EINVAL; + /* nuke other bits */ flags &= mask; @@ -939,7 +947,8 @@ bad_flags: #endif /* CONFIG_UMP */ #ifdef CONFIG_DMA_SHARED_BUFFER -static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags) +static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, + int fd, u64 *va_pages, u64 *flags, u32 padding) { struct kbase_va_region *reg; struct dma_buf *dma_buf; @@ -954,7 +963,7 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in if (!dma_attachment) goto no_attachment; - *va_pages = PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT; + *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; if (!*va_pages) goto bad_size; @@ -1008,6 +1017,9 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in if (*flags & BASE_MEM_SECURE) reg->flags |= KBASE_REG_SECURE; + if (padding) + reg->flags |= KBASE_REG_IMPORT_PAD; + reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; reg->gpu_alloc->imported.umm.sgt = NULL; reg->gpu_alloc->imported.umm.dma_buf = dma_buf; @@ -1046,11 +1058,14 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( struct kbase_context *kctx, unsigned long address, unsigned long size, u64 *va_pages, u64 *flags) { + long i; struct kbase_va_region *reg; long faulted_pages; int zone = KBASE_REG_ZONE_CUSTOM_VA; bool shared_zone = false; u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); + struct kbase_alloc_import_user_buf *user_buf; + struct page **pages = NULL; if ((address & (cache_line_alignment - 1)) != 0 || (size & (cache_line_alignment - 1)) != 0) { @@ -1116,50 +1131,101 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( reg->flags &= ~KBASE_REG_FREE; reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ + reg->flags &= ~KBASE_REG_CPU_CACHED; + + user_buf = ®->gpu_alloc->imported.user_buf; + + user_buf->size = size; + user_buf->address = address; + user_buf->nr_pages = *va_pages; + user_buf->mm = current->mm; + user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), + GFP_KERNEL); + + if (!user_buf->pages) + goto no_page_array; + + /* If the region is coherent with the CPU then the memory is imported + * and mapped onto the GPU immediately. + * Otherwise get_user_pages is called as a sanity check, but with + * NULL as the pages argument which will fault the pages, but not + * pin them. The memory will then be pinned only around the jobs that + * specify the region as an external resource. + */ + if (reg->flags & KBASE_REG_SHARE_BOTH) { + pages = user_buf->pages; + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + } down_read(¤t->mm->mmap_sem); - /* A sanity check that get_user_pages will work on the memory */ - /* (so the initial import fails on weird memory regions rather than */ - /* the job failing when we try to handle the external resources). */ - /* It doesn't take a reference to the pages (because the page list is NULL). */ - /* We can't really store the page list because that would involve */ - /* keeping the pages pinned - instead we pin/unpin around the job */ - /* (as part of the external resources handling code) */ #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) faulted_pages = get_user_pages(current, current->mm, address, *va_pages, - reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL); + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); #elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) faulted_pages = get_user_pages(address, *va_pages, - reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL); + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); #else faulted_pages = get_user_pages(address, *va_pages, reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, - NULL, NULL); + pages, NULL); #endif + up_read(¤t->mm->mmap_sem); if (faulted_pages != *va_pages) goto fault_mismatch; - reg->gpu_alloc->imported.user_buf.size = size; - reg->gpu_alloc->imported.user_buf.address = address; - reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages; - reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages, - sizeof(struct page *), GFP_KERNEL); - reg->gpu_alloc->imported.user_buf.mm = current->mm; atomic_inc(¤t->mm->mm_count); - if (!reg->gpu_alloc->imported.user_buf.pages) - goto no_page_array; - reg->gpu_alloc->nents = 0; reg->extent = 0; + if (pages) { + struct device *dev = kctx->kbdev->dev; + unsigned long local_size = user_buf->size; + unsigned long offset = user_buf->address & ~PAGE_MASK; + phys_addr_t *pa = kbase_get_gpu_phy_pages(reg); + + /* Top bit signifies that this was pinned on import */ + user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; + + for (i = 0; i < faulted_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind_dma_map; + + user_buf->dma_addrs[i] = dma_addr; + pa[i] = page_to_phys(pages[i]); + + local_size -= min; + offset = 0; + } + + reg->gpu_alloc->nents = faulted_pages; + } + return reg; -no_page_array: +unwind_dma_map: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + user_buf->dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } fault_mismatch: + if (pages) { + for (i = 0; i < faulted_pages; i++) + put_page(pages[i]); + } + kfree(user_buf->pages); +no_page_array: invalid_flags: kbase_mem_phy_alloc_put(reg->cpu_alloc); kbase_mem_phy_alloc_put(reg->gpu_alloc); @@ -1362,7 +1428,7 @@ bad_flags: } int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u64 *gpu_va, u64 *va_pages, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags) { struct kbase_va_region *reg; @@ -1384,6 +1450,24 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, goto bad_flags; } + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { + dev_warn(kctx->kbdev->dev, + "padding is only supported for UMM"); + goto bad_flags; + } + switch (type) { #ifdef CONFIG_UMP case BASE_MEM_IMPORT_TYPE_UMP: { @@ -1403,7 +1487,8 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, if (get_user(fd, (int __user *)phandle)) reg = NULL; else - reg = kbase_mem_from_umm(kctx, fd, va_pages, flags); + reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, + padding); } break; #endif /* CONFIG_DMA_SHARED_BUFFER */ @@ -1535,7 +1620,7 @@ static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, return ret; } -int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason) +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) { u64 old_pages; u64 delta; @@ -1544,48 +1629,40 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en bool read_locked = false; KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(failure_reason); KBASE_DEBUG_ASSERT(gpu_addr != 0); + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + down_write(¤t->mm->mmap_sem); kbase_gpu_vm_lock(kctx); /* Validate the region */ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); - if (!reg || (reg->flags & KBASE_REG_FREE)) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; + if (!reg || (reg->flags & KBASE_REG_FREE)) goto out_unlock; - } KBASE_DEBUG_ASSERT(reg->cpu_alloc); KBASE_DEBUG_ASSERT(reg->gpu_alloc); - if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE; + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) goto out_unlock; - } - if (0 == (reg->flags & KBASE_REG_GROWABLE)) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE; + if (0 == (reg->flags & KBASE_REG_GROWABLE)) goto out_unlock; - } - if (new_pages > reg->nr_pages) { - /* Would overflow the VA region */ - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; + /* Would overflow the VA region */ + if (new_pages > reg->nr_pages) goto out_unlock; - } /* can't be mapped more than once on the GPU */ - if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE; + if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) goto out_unlock; - } /* can't grow regions which are ephemeral */ - if (reg->flags & KBASE_REG_DONT_NEED) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE; + if (reg->flags & KBASE_REG_DONT_NEED) goto out_unlock; - } if (new_pages == reg->gpu_alloc->nents) { /* no change */ @@ -1606,13 +1683,13 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en /* Allocate some more pages */ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM; + res = -ENOMEM; goto out_unlock; } if (reg->cpu_alloc != reg->gpu_alloc) { if (kbase_alloc_phy_pages_helper( reg->gpu_alloc, delta) != 0) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM; + res = -ENOMEM; kbase_free_phy_pages_helper(reg->cpu_alloc, delta); goto out_unlock; @@ -1631,7 +1708,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en if (reg->cpu_alloc != reg->gpu_alloc) kbase_free_phy_pages_helper(reg->gpu_alloc, delta); - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM; + res = -ENOMEM; goto out_unlock; } } else { @@ -1645,7 +1722,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en res = kbase_mem_shrink_gpu_mapping(kctx, reg, new_pages, old_pages); if (res) { - *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM; + res = -ENOMEM; goto out_unlock; } @@ -1735,7 +1812,11 @@ static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* insert all valid pages from the fault location */ i = rel_pgoff; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); +#else + addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); +#endif while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { int ret = vm_insert_pfn(vma, addr << PAGE_SHIFT, PFN_DOWN(map->alloc->pages[i])); diff --git a/mali_kbase/mali_kbase_mem_linux.h b/mali_kbase/mali_kbase_mem_linux.h index 97e0bec..a41d646 100644 --- a/mali_kbase/mali_kbase_mem_linux.h +++ b/mali_kbase/mali_kbase_mem_linux.h @@ -37,11 +37,22 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 *gpu_va); int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u64 *gpu_va, u64 *va_pages, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags); u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); -int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason); + +/** + * kbase_mem_commit - Change the physical backing size of a region + * + * @kctx: The kernel context + * @gpu_addr: Handle to the memory region + * @new_pages: Number of physical pages to back the region with + * + * Return: 0 on success or error code + */ +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); + int kbase_mmap(struct file *file, struct vm_area_struct *vma); /** diff --git a/mali_kbase/mali_kbase_mem_lowlevel.h b/mali_kbase/mali_kbase_mem_lowlevel.h index 441180b..9725fd3 100644 --- a/mali_kbase/mali_kbase_mem_lowlevel.h +++ b/mali_kbase/mali_kbase_mem_lowlevel.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2014, 2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mem_pool.c b/mali_kbase/mali_kbase_mem_pool.c index b5a7b8c..25c3128 100644 --- a/mali_kbase/mali_kbase_mem_pool.c +++ b/mali_kbase/mali_kbase_mem_pool.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mem_profile_debugfs.c b/mali_kbase/mali_kbase_mem_profile_debugfs.c index 092da9a..d58fd8d 100644 --- a/mali_kbase/mali_kbase_mem_profile_debugfs.c +++ b/mali_kbase/mali_kbase_mem_profile_debugfs.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -84,6 +84,8 @@ int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, kfree(kctx->mem_profile_data); kctx->mem_profile_data = data; kctx->mem_profile_size = size; + } else { + kfree(data); } dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", diff --git a/mali_kbase/mali_kbase_mmu.c b/mali_kbase/mali_kbase_mmu.c index 0329dfd..79d2d18 100644 --- a/mali_kbase/mali_kbase_mmu.c +++ b/mali_kbase/mali_kbase_mmu.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -172,18 +172,23 @@ void page_fault_worker(struct work_struct *data) dev_warn(kbdev->dev, "Access flag unexpectedly set"); goto fault_done; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: - - kbase_mmu_report_fault_and_kill(kctx, faulting_as, + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Address size fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); goto fault_done; case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: - kbase_mmu_report_fault_and_kill(kctx, faulting_as, + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Memory attributes fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); goto fault_done; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ default: kbase_mmu_report_fault_and_kill(kctx, faulting_as, @@ -1628,7 +1633,8 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) e = "TRANSLATION_FAULT"; break; case 0xC8: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 + e = "PERMISSION_FAULT"; + break; case 0xC9: case 0xCA: case 0xCB: @@ -1636,8 +1642,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) case 0xCD: case 0xCE: case 0xCF: -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ - e = "PERMISSION_FAULT"; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "PERMISSION_FAULT"; + else + e = "UNKNOWN"; break; case 0xD0: case 0xD1: @@ -1650,7 +1658,8 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) e = "TRANSTAB_BUS_FAULT"; break; case 0xD8: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 + e = "ACCESS_FLAG"; + break; case 0xD9: case 0xDA: case 0xDB: @@ -1658,10 +1667,11 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) case 0xDD: case 0xDE: case 0xDF: -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ - e = "ACCESS_FLAG"; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ACCESS_FLAG"; + else + e = "UNKNOWN"; break; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 case 0xE0: case 0xE1: case 0xE2: @@ -1670,7 +1680,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) case 0xE5: case 0xE6: case 0xE7: - e = "ADDRESS_SIZE_FAULT"; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ADDRESS_SIZE_FAULT"; + else + e = "UNKNOWN"; break; case 0xE8: case 0xE9: @@ -1680,8 +1693,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) case 0xED: case 0xEE: case 0xEF: - e = "MEMORY_ATTRIBUTES_FAULT"; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "MEMORY_ATTRIBUTES_FAULT"; + else + e = "UNKNOWN"; break; default: e = "UNKNOWN"; @@ -1696,11 +1711,10 @@ static const char *access_type_name(struct kbase_device *kbdev, { switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - return "ATOMIC"; -#else - return "UNKNOWN"; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + return "ATOMIC"; + else + return "UNKNOWN"; case AS_FAULTSTATUS_ACCESS_TYPE_READ: return "READ"; case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: @@ -1736,7 +1750,7 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, js_devdata = &kbdev->js_data; /* ASSERT that the context won't leave the runpool */ - KBASE_DEBUG_ASSERT(kbasep_js_debug_check_ctx_refcount(kbdev, kctx) > 0); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); /* decode the fault status */ exception_type = as->fault_status & 0xFF; @@ -2035,15 +2049,14 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_contex */ kbasep_js_clear_submit_allowed(js_devdata, kctx); -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - dev_warn(kbdev->dev, - "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", - as->number, as->fault_addr, - as->fault_extra_addr); -#else - dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", - as->number, as->fault_addr); -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + dev_warn(kbdev->dev, + "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", + as->number, as->fault_addr, + as->fault_extra_addr); + else + dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", + as->number, as->fault_addr); /* * We need to switch to UNMAPPED mode - but we do this in a diff --git a/mali_kbase/mali_kbase_mmu_mode.h b/mali_kbase/mali_kbase_mmu_mode.h index 2449c60..b487c00 100644 --- a/mali_kbase/mali_kbase_mmu_mode.h +++ b/mali_kbase/mali_kbase_mmu_mode.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mmu_mode_aarch64.c b/mali_kbase/mali_kbase_mmu_mode_aarch64.c index 791f3ed..60df171 100644 --- a/mali_kbase/mali_kbase_mmu_mode_aarch64.c +++ b/mali_kbase/mali_kbase_mmu_mode_aarch64.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2014, 2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2014, 2016, 2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mmu_mode_lpae.c b/mali_kbase/mali_kbase_mmu_mode_lpae.c index 683cabb..53fbbc7 100644 --- a/mali_kbase/mali_kbase_mmu_mode_lpae.c +++ b/mali_kbase/mali_kbase_mmu_mode_lpae.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -91,11 +91,7 @@ static void mmu_get_as_setup(struct kbase_context *kctx, AS_TRANSTAB_LPAE_ADRMODE_TABLE | AS_TRANSTAB_LPAE_READ_INNER; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - setup->transcfg = AS_TRANSCFG_ADRMODE_LEGACY; -#else setup->transcfg = 0; -#endif } static void mmu_update(struct kbase_context *kctx) @@ -117,10 +113,6 @@ static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - current_setup->transcfg = AS_TRANSCFG_ADRMODE_LEGACY; -#endif - /* Apply the address space setting */ kbase_mmu_hw_configure(kbdev, as, NULL); } diff --git a/mali_kbase/mali_kbase_softjobs.c b/mali_kbase/mali_kbase_softjobs.c index 8a9a47d..72809f7 100644 --- a/mali_kbase/mali_kbase_softjobs.c +++ b/mali_kbase/mali_kbase_softjobs.c @@ -23,15 +23,14 @@ #include <linux/dma-buf.h> #include <asm/cacheflush.h> #endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ -#include <linux/dma-mapping.h> -#ifdef CONFIG_SYNC -#include "sync.h" -#include <linux/syscalls.h> -#include "mali_kbase_sync.h" +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include <mali_kbase_sync.h> #endif +#include <linux/dma-mapping.h> #include <mali_base_kernel.h> #include <mali_kbase_hwaccess_time.h> #include <mali_kbase_mem_linux.h> +#include <mali_kbase_tlstream.h> #include <linux/version.h> #include <linux/ktime.h> #include <linux/pfn.h> @@ -47,7 +46,7 @@ * executed within the driver rather than being handed over to the GPU. */ -void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) +static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) { struct kbase_context *kctx = katom->kctx; unsigned long lflags; @@ -194,48 +193,11 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) return 0; } -#ifdef CONFIG_SYNC - -static enum base_jd_event_code kbase_fence_trigger(struct kbase_jd_atom *katom, int result) -{ - struct sync_pt *pt; - struct sync_timeline *timeline; - -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - if (!list_is_singular(&katom->fence->pt_list_head)) { -#else - if (katom->fence->num_fences != 1) { -#endif - /* Not exactly one item in the list - so it didn't (directly) come from us */ - return BASE_JD_EVENT_JOB_CANCELLED; - } - -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - pt = list_first_entry(&katom->fence->pt_list_head, struct sync_pt, pt_list); -#else - pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); -#endif - timeline = sync_pt_parent(pt); - - if (!kbase_sync_timeline_is_ours(timeline)) { - /* Fence has a sync_pt which isn't ours! */ - return BASE_JD_EVENT_JOB_CANCELLED; - } - - kbase_sync_signal_pt(pt, result); - - sync_timeline_signal(timeline); - - return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; -} - -static void kbase_fence_wait_worker(struct work_struct *data) +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +/* Called by the explicit fence mechanism when a fence wait has completed */ +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) { - struct kbase_jd_atom *katom; - struct kbase_context *kctx; - - katom = container_of(data, struct kbase_jd_atom, work); - kctx = katom->kctx; + struct kbase_context *kctx = katom->kctx; mutex_lock(&kctx->jctx.lock); kbasep_remove_waiting_soft_job(katom); @@ -244,89 +206,8 @@ static void kbase_fence_wait_worker(struct work_struct *data) kbase_js_sched_all(kctx->kbdev); mutex_unlock(&kctx->jctx.lock); } - -static void kbase_fence_wait_callback(struct sync_fence *fence, struct sync_fence_waiter *waiter) -{ - struct kbase_jd_atom *katom = container_of(waiter, struct kbase_jd_atom, sync_waiter); - struct kbase_context *kctx; - - KBASE_DEBUG_ASSERT(NULL != katom); - - kctx = katom->kctx; - - KBASE_DEBUG_ASSERT(NULL != kctx); - - /* Propagate the fence status to the atom. - * If negative then cancel this atom and its dependencies. - */ - if (kbase_fence_get_status(fence) < 0) - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* To prevent a potential deadlock we schedule the work onto the job_done_wq workqueue - * - * The issue is that we may signal the timeline while holding kctx->jctx.lock and - * the callbacks are run synchronously from sync_timeline_signal. So we simply defer the work. - */ - - KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); - INIT_WORK(&katom->work, kbase_fence_wait_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -static int kbase_fence_wait(struct kbase_jd_atom *katom) -{ - int ret; - - KBASE_DEBUG_ASSERT(NULL != katom); - KBASE_DEBUG_ASSERT(NULL != katom->kctx); - - sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); - - ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); - - if (ret == 1) { - /* Already signalled */ - return 0; - } - - if (ret < 0) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - /* We should cause the dependent jobs in the bag to be failed, - * to do this we schedule the work queue to complete this job */ - KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); - INIT_WORK(&katom->work, kbase_fence_wait_worker); - queue_work(katom->kctx->jctx.job_done_wq, &katom->work); - } - -#ifdef CONFIG_MALI_FENCE_DEBUG - /* The timeout code will add this job to the list of waiting soft jobs. - */ - kbasep_add_waiting_with_timeout(katom); -#else - kbasep_add_waiting_soft_job(katom); #endif - return 1; -} - -static void kbase_fence_cancel_wait(struct kbase_jd_atom *katom) -{ - if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { - /* The wait wasn't cancelled - leave the cleanup for kbase_fence_wait_callback */ - return; - } - - /* Wait was cancelled - zap the atoms */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - kbasep_remove_waiting_soft_job(katom); - kbase_finish_soft_job(katom); - - if (jd_done_nolock(katom, NULL)) - kbase_js_sched_all(katom->kctx->kbdev); -} -#endif /* CONFIG_SYNC */ - static void kbasep_soft_event_complete_job(struct work_struct *work) { struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, @@ -403,15 +284,17 @@ static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { - struct sync_fence *fence = dep->fence; - int status = kbase_fence_get_status(fence); - /* Found blocked trigger fence. */ - dev_warn(dev, - "\tVictim trigger atom %d fence [%p] %s: %s\n", - kbase_jd_atom_id(kctx, dep), - fence, fence->name, - kbase_sync_status_string(status)); + struct kbase_sync_fence_info info; + + if (!kbase_sync_fence_in_info_get(dep, &info)) { + dev_warn(dev, + "\tVictim trigger atom %d fence [%p] %s: %s\n", + kbase_jd_atom_id(kctx, dep), + info.fence, + info.name, + kbase_sync_status_string(info.status)); + } } kbase_fence_debug_check_atom(dep); @@ -423,40 +306,32 @@ static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) { struct kbase_context *kctx = katom->kctx; struct device *dev = katom->kctx->kbdev->dev; - struct sync_fence *fence; int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - int status; unsigned long lflags; + struct kbase_sync_fence_info info; spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - fence = katom->fence; - if (!fence) { + if (kbase_sync_fence_in_info_get(katom, &info)) { /* Fence must have signaled just after timeout. */ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); return; } - status = kbase_fence_get_status(fence); - dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", kctx->tgid, kctx->id, kbase_jd_atom_id(kctx, katom), - fence, timeout_ms); + info.fence, timeout_ms); dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", - fence, fence->name, - kbase_sync_status_string(status)); + info.fence, info.name, + kbase_sync_status_string(info.status)); /* Search for blocked trigger atoms */ kbase_fence_debug_check_atom(katom); spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); - /* Dump out the full state of all the Android sync fences. - * The function sync_dump() isn't exported to modules, so force - * sync_fence_wait() to time out to trigger sync_dump(). - */ - sync_fence_wait(fence, 1); + kbase_sync_fence_in_dump(katom); } struct kbase_fence_debug_work { @@ -1073,7 +948,7 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) struct base_jit_alloc_info *info; struct kbase_va_region *reg; struct kbase_vmap_struct mapping; - u64 *ptr; + u64 *ptr, new_addr; if (katom->jit_blocked) { list_del(&katom->queue); @@ -1152,7 +1027,10 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) return 0; } - *ptr = reg->start_pfn << PAGE_SHIFT; + new_addr = reg->start_pfn << PAGE_SHIFT; + *ptr = new_addr; + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( + katom, info->gpu_alloc_addr, new_addr); kbase_vunmap(kctx, &mapping); katom->event_code = BASE_JD_EVENT_DONE; @@ -1389,17 +1267,28 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom) switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: return kbase_dump_cpu_gpu_time(katom); -#ifdef CONFIG_SYNC + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - KBASE_DEBUG_ASSERT(katom->fence != NULL); - katom->event_code = kbase_fence_trigger(katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT); - /* Release the reference as we don't need it any more */ - sync_fence_put(katom->fence); - katom->fence = NULL; + katom->event_code = kbase_sync_fence_out_trigger(katom, + katom->event_code == BASE_JD_EVENT_DONE ? + 0 : -EFAULT); break; case BASE_JD_REQ_SOFT_FENCE_WAIT: - return kbase_fence_wait(katom); -#endif /* CONFIG_SYNC */ + { + int ret = kbase_sync_fence_in_wait(katom); + + if (ret == 1) { +#ifdef CONFIG_MALI_FENCE_DEBUG + kbasep_add_waiting_with_timeout(katom); +#else + kbasep_add_waiting_soft_job(katom); +#endif + } + return ret; + } +#endif + case BASE_JD_REQ_SOFT_REPLAY: return kbase_replay_process(katom); case BASE_JD_REQ_SOFT_EVENT_WAIT: @@ -1438,9 +1327,9 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom) void kbase_cancel_soft_job(struct kbase_jd_atom *katom) { switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) case BASE_JD_REQ_SOFT_FENCE_WAIT: - kbase_fence_cancel_wait(katom); + kbase_sync_fence_in_cancel_wait(katom); break; #endif case BASE_JD_REQ_SOFT_EVENT_WAIT: @@ -1461,7 +1350,7 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) return -EINVAL; } break; -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) case BASE_JD_REQ_SOFT_FENCE_TRIGGER: { struct base_fence fence; @@ -1470,21 +1359,16 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) return -EINVAL; - fd = kbase_stream_create_fence(fence.basep.stream_fd); + fd = kbase_sync_fence_out_create(katom, + fence.basep.stream_fd); if (fd < 0) return -EINVAL; - katom->fence = sync_fence_fdget(fd); - - if (katom->fence == NULL) { - /* The only way the fence can be NULL is if userspace closed it for us. - * So we don't need to clear it up */ - return -EINVAL; - } fence.basep.fd = fd; if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { - katom->fence = NULL; - sys_close(fd); + kbase_sync_fence_out_remove(katom); + kbase_sync_fence_close_fd(fd); + fence.basep.fd = -EINVAL; return -EINVAL; } } @@ -1492,14 +1376,16 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) case BASE_JD_REQ_SOFT_FENCE_WAIT: { struct base_fence fence; + int ret; if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) return -EINVAL; /* Get a reference to the fence object */ - katom->fence = sync_fence_fdget(fence.basep.fd); - if (katom->fence == NULL) - return -EINVAL; + ret = kbase_sync_fence_in_from_fd(katom, + fence.basep.fd); + if (ret < 0) + return ret; #ifdef CONFIG_MALI_DMA_FENCE /* @@ -1513,7 +1399,7 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) #endif /* CONFIG_MALI_DMA_FENCE */ } break; -#endif /* CONFIG_SYNC */ +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ case BASE_JD_REQ_SOFT_JIT_ALLOC: return kbase_jit_allocate_prepare(katom); case BASE_JD_REQ_SOFT_REPLAY: @@ -1545,23 +1431,17 @@ void kbase_finish_soft_job(struct kbase_jd_atom *katom) case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: /* Nothing to do */ break; -#ifdef CONFIG_SYNC +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - /* If fence has not yet been signalled, do it now */ - if (katom->fence) { - kbase_fence_trigger(katom, katom->event_code == - BASE_JD_EVENT_DONE ? 0 : -EFAULT); - sync_fence_put(katom->fence); - katom->fence = NULL; - } + /* If fence has not yet been signaled, do it now */ + kbase_sync_fence_out_trigger(katom, katom->event_code == + BASE_JD_EVENT_DONE ? 0 : -EFAULT); break; case BASE_JD_REQ_SOFT_FENCE_WAIT: - /* Release the reference to the fence object */ - sync_fence_put(katom->fence); - katom->fence = NULL; + /* Release katom's reference to fence object */ + kbase_sync_fence_in_remove(katom); break; -#endif /* CONFIG_SYNC */ - +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ case BASE_JD_REQ_SOFT_DEBUG_COPY: kbase_debug_copy_finish(katom); break; diff --git a/mali_kbase/mali_kbase_sync.c b/mali_kbase/mali_kbase_sync.c deleted file mode 100644 index 97e301d..0000000 --- a/mali_kbase/mali_kbase_sync.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - * - * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - * - */ - - - -#ifdef CONFIG_SYNC - -#include <linux/seq_file.h> -#include "sync.h" -#include <mali_kbase.h> -#include <mali_kbase_sync.h> - -struct mali_sync_timeline { - struct sync_timeline timeline; - atomic_t counter; - atomic_t signalled; -}; - -struct mali_sync_pt { - struct sync_pt pt; - int order; - int result; -}; - -static struct mali_sync_timeline *to_mali_sync_timeline(struct sync_timeline *timeline) -{ - return container_of(timeline, struct mali_sync_timeline, timeline); -} - -static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) -{ - return container_of(pt, struct mali_sync_pt, pt); -} - -static struct sync_pt *timeline_dup(struct sync_pt *pt) -{ - struct mali_sync_pt *mpt = to_mali_sync_pt(pt); - struct mali_sync_pt *new_mpt; - struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), sizeof(struct mali_sync_pt)); - - if (!new_pt) - return NULL; - - new_mpt = to_mali_sync_pt(new_pt); - new_mpt->order = mpt->order; - new_mpt->result = mpt->result; - - return new_pt; -} - -static int timeline_has_signaled(struct sync_pt *pt) -{ - struct mali_sync_pt *mpt = to_mali_sync_pt(pt); - struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt)); - int result = mpt->result; - - int diff = atomic_read(&mtl->signalled) - mpt->order; - - if (diff >= 0) - return (result < 0) ? result : 1; - - return 0; -} - -static int timeline_compare(struct sync_pt *a, struct sync_pt *b) -{ - struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); - struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); - - int diff = ma->order - mb->order; - - if (diff == 0) - return 0; - - return (diff < 0) ? -1 : 1; -} - -static void timeline_value_str(struct sync_timeline *timeline, char *str, - int size) -{ - struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); - - snprintf(str, size, "%d", atomic_read(&mtl->signalled)); -} - -static void pt_value_str(struct sync_pt *pt, char *str, int size) -{ - struct mali_sync_pt *mpt = to_mali_sync_pt(pt); - - snprintf(str, size, "%d(%d)", mpt->order, mpt->result); -} - -static struct sync_timeline_ops mali_timeline_ops = { - .driver_name = "Mali", - .dup = timeline_dup, - .has_signaled = timeline_has_signaled, - .compare = timeline_compare, - .timeline_value_str = timeline_value_str, - .pt_value_str = pt_value_str, -}; - -int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) -{ - return timeline->ops == &mali_timeline_ops; -} - -struct sync_timeline *kbase_sync_timeline_alloc(const char *name) -{ - struct sync_timeline *tl; - struct mali_sync_timeline *mtl; - - tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline), name); - if (!tl) - return NULL; - - /* Set the counter in our private struct */ - mtl = to_mali_sync_timeline(tl); - atomic_set(&mtl->counter, 0); - atomic_set(&mtl->signalled, 0); - - return tl; -} - -struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) -{ - struct sync_pt *pt = sync_pt_create(parent, sizeof(struct mali_sync_pt)); - struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); - struct mali_sync_pt *mpt; - - if (!pt) - return NULL; - - mpt = to_mali_sync_pt(pt); - mpt->order = atomic_inc_return(&mtl->counter); - mpt->result = 0; - - return pt; -} - -void kbase_sync_signal_pt(struct sync_pt *pt, int result) -{ - struct mali_sync_pt *mpt = to_mali_sync_pt(pt); - struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt)); - int signalled; - int diff; - - mpt->result = result; - - do { - signalled = atomic_read(&mtl->signalled); - - diff = signalled - mpt->order; - - if (diff > 0) { - /* The timeline is already at or ahead of this point. - * This should not happen unless userspace has been - * signalling fences out of order, so warn but don't - * violate the sync_pt API. - * The warning is only in debug builds to prevent - * a malicious user being able to spam dmesg. - */ -#ifdef CONFIG_MALI_DEBUG - pr_err("Fences were triggered in a different order to allocation!"); -#endif /* CONFIG_MALI_DEBUG */ - return; - } - } while (atomic_cmpxchg(&mtl->signalled, signalled, mpt->order) != signalled); -} - -const char *kbase_sync_status_string(int status) -{ - if (status == 0) - return "signaled"; - else if (status > 0) - return "active"; - else - return "error"; -} - -#endif /* CONFIG_SYNC */ diff --git a/mali_kbase/mali_kbase_sync.h b/mali_kbase/mali_kbase_sync.h index 07e5158..de72147 100644 --- a/mali_kbase/mali_kbase_sync.h +++ b/mali_kbase/mali_kbase_sync.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -15,94 +15,189 @@ - - /** * @file mali_kbase_sync.h * + * This file contains our internal "API" for explicit fences. + * It hides the implementation details of the actual explicit fence mechanism + * used (Android fences or sync file with DMA fences). */ #ifndef MALI_KBASE_SYNC_H #define MALI_KBASE_SYNC_H -#include "sync.h" - -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -/* For backwards compatiblility with kernels before 3.17. After 3.17 - * sync_pt_parent is included in the kernel. */ -static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) -{ - return pt->parent; -} +#include <linux/syscalls.h> +#ifdef CONFIG_SYNC +#include <sync.h> #endif - -static inline int kbase_fence_get_status(struct sync_fence *fence) -{ -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - return fence->status; -#else - return atomic_read(&fence->status); +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_fence_defs.h" +#include <linux/sync_file.h> #endif -} -/* - * Create a stream object. - * Built on top of timeline object. +#include "mali_kbase.h" + +/** + * struct kbase_sync_fence_info - Information about a fence + * @fence: Pointer to fence (type is void*, as underlaying struct can differ) + * @name: The name given to this fence when it was created + * @status: < 0 means error, 0 means active, 1 means signaled + * + * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() + * to get the information. + */ +struct kbase_sync_fence_info { + void *fence; + char name[32]; + int status; +}; + +/** + * kbase_sync_fence_stream_create() - Create a stream object + * @name: Name of stream (only used to ease debugging/visualization) + * @out_fd: A file descriptor representing the created stream object + * + * Can map down to a timeline implementation in some implementations. * Exposed as a file descriptor. * Life-time controlled via the file descriptor: * - dup to add a ref * - close to remove a ref + * + * return: 0 on success, < 0 on error */ -int kbase_stream_create(const char *name, int *const out_fd); +int kbase_sync_fence_stream_create(const char *name, int *const out_fd); -/* - * Create a fence in a stream object +/** + * kbase_sync_fence_out_create Create an explicit output fence to specified atom + * @katom: Atom to assign the new explicit fence to + * @stream_fd: File descriptor for stream object to create fence on + * + * return: Valid file descriptor to fence or < 0 on error */ -int kbase_stream_create_fence(int tl_fd); +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); -/* - * Validate a fd to be a valid fence - * No reference is taken. +/** + * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom + * @katom: Atom to assign the existing explicit fence to + * @fd: File descriptor to an existing fence + * + * Assigns an explicit input fence to atom. + * This can later be waited for by calling @kbase_sync_fence_in_wait + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); + +/** + * kbase_sync_fence_validate() - Validate a fd to be a valid fence + * @fd: File descriptor to check * * This function is only usable to catch unintentional user errors early, * it does not stop malicious code changing the fd after this function returns. + * + * return 0: if fd is for a valid fence, < 0 if invalid */ -int kbase_fence_validate(int fd); +int kbase_sync_fence_validate(int fd); -/* Returns true if the specified timeline is allocated by Mali */ -int kbase_sync_timeline_is_ours(struct sync_timeline *timeline); +/** + * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom + * @katom: Atom with an explicit fence to signal + * @result: < 0 means signal with error, 0 >= indicates success + * + * Signal output fence attached on katom and remove the fence from the atom. + * + * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE + */ +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); -/* Allocates a timeline for Mali +/** + * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled + * @katom: Atom with explicit fence to wait for * - * One timeline should be allocated per API context. + * If the fence is already signaled, then 0 is returned, and the caller must + * continue processing of the katom. + * + * If the fence isn't already signaled, then this kbase_sync framework will + * take responsibility to continue the processing once the fence is signaled. + * + * return: 0 if already signaled, otherwise 1 */ -struct sync_timeline *kbase_sync_timeline_alloc(const char *name); +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); -/* Allocates a sync point within the timeline. +/** + * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits + * @katom: Atom to cancel wait for * - * The timeline must be the one allocated by kbase_sync_timeline_alloc + * This function is fully responsible for continuing processing of this atom + * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) + */ +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_remove() - Remove the input fence from the katom + * @katom: Atom to remove explicit input fence for * - * Sync points must be triggered in *exactly* the same order as they are allocated. + * This will also release the corresponding reference. */ -struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent); +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); -/* Signals a particular sync point +/** + * kbase_sync_fence_out_remove() - Remove the output fence from the katom + * @katom: Atom to remove explicit output fence for * - * Sync points must be triggered in *exactly* the same order as they are allocated. + * This will also release the corresponding reference. + */ +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence + * @fd: File descriptor to close + */ +static inline void kbase_sync_fence_close_fd(int fd) +{ + sys_close(fd); +} + +/** + * kbase_sync_fence_in_info_get() - Retrieves information about input fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information * - * If they are signalled in the wrong order then a message will be printed in debug - * builds and otherwise attempts to signal order sync_pts will be ignored. + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_fence_out_info_get() - Retrieves information about output fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information * - * result can be negative to indicate error, any other value is interpreted as success. + * return: 0 on success, < 0 on error */ -void kbase_sync_signal_pt(struct sync_pt *pt, int result); +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); /** * kbase_sync_status_string() - Get string matching @status * @status: Value of fence status. * - * Return: Pointer to string describing @status. + * return: Pointer to string describing @status. */ const char *kbase_sync_status_string(int status); +/* + * Internal worker used to continue processing of atom. + */ +void kbase_sync_fence_wait_worker(struct work_struct *data); + +#ifdef CONFIG_MALI_FENCE_DEBUG +/** + * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state + * @katom: Atom to trigger fence debug dump for + */ +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); #endif + +#endif /* MALI_KBASE_SYNC_H */ diff --git a/mali_kbase/mali_kbase_sync_android.c b/mali_kbase/mali_kbase_sync_android.c new file mode 100644 index 0000000..d7349dc --- /dev/null +++ b/mali_kbase/mali_kbase_sync_android.c @@ -0,0 +1,537 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Android fences (CONFIG_SYNC) + * Known to be good for kernels 4.5 and earlier. + * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels + * (see mali_kbase_sync_file.c) + */ + +#include <linux/sched.h> +#include <linux/fdtable.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/module.h> +#include <linux/anon_inodes.h> +#include <linux/version.h> +#include "sync.h" +#include <mali_kbase.h> +#include <mali_kbase_sync.h> + +struct mali_sync_timeline { + struct sync_timeline timeline; + atomic_t counter; + atomic_t signaled; +}; + +struct mali_sync_pt { + struct sync_pt pt; + int order; + int result; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +/* For backwards compatibility with kernels before 3.17. After 3.17 + * sync_pt_parent is included in the kernel. */ +static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) +{ + return pt->parent; +} +#endif + +static struct mali_sync_timeline *to_mali_sync_timeline( + struct sync_timeline *timeline) +{ + return container_of(timeline, struct mali_sync_timeline, timeline); +} + +static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +{ + return container_of(pt, struct mali_sync_pt, pt); +} + +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_pt *new_mpt; + struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), + sizeof(struct mali_sync_pt)); + + if (!new_pt) + return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + new_mpt->order = mpt->order; + new_mpt->result = mpt->result; + + return new_pt; +} + +static int timeline_has_signaled(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int result = mpt->result; + + int diff = atomic_read(&mtl->signaled) - mpt->order; + + if (diff >= 0) + return (result < 0) ? result : 1; + + return 0; +} + +static int timeline_compare(struct sync_pt *a, struct sync_pt *b) +{ + struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); + struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); + + int diff = ma->order - mb->order; + + if (diff == 0) + return 0; + + return (diff < 0) ? -1 : 1; +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, + int size) +{ + struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); + + snprintf(str, size, "%d", atomic_read(&mtl->signaled)); +} + +static void pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + + snprintf(str, size, "%d(%d)", mpt->order, mpt->result); +} + +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .timeline_value_str = timeline_value_str, + .pt_value_str = pt_value_str, +}; + +/* Allocates a timeline for Mali + * + * One timeline should be allocated per API context. + */ +static struct sync_timeline *mali_sync_timeline_alloc(const char *name) +{ + struct sync_timeline *tl; + struct mali_sync_timeline *mtl; + + tl = sync_timeline_create(&mali_timeline_ops, + sizeof(struct mali_sync_timeline), name); + if (!tl) + return NULL; + + /* Set the counter in our private struct */ + mtl = to_mali_sync_timeline(tl); + atomic_set(&mtl->counter, 0); + atomic_set(&mtl->signaled, 0); + + return tl; +} + +static int kbase_stream_close(struct inode *inode, struct file *file) +{ + struct sync_timeline *tl; + + tl = (struct sync_timeline *)file->private_data; + sync_timeline_destroy(tl); + return 0; +} + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE, + .release = kbase_stream_close, +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + struct sync_timeline *tl; + + if (!out_fd) + return -EINVAL; + + tl = mali_sync_timeline_alloc(name); + if (!tl) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); + + if (*out_fd < 0) { + sync_timeline_destroy(tl); + return -EINVAL; + } + + return 0; +} + +/* Allocates a sync point within the timeline. + * + * The timeline must be the one allocated by kbase_sync_timeline_alloc + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + */ +static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) +{ + struct sync_pt *pt = sync_pt_create(parent, + sizeof(struct mali_sync_pt)); + struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); + struct mali_sync_pt *mpt; + + if (!pt) + return NULL; + + mpt = to_mali_sync_pt(pt); + mpt->order = atomic_inc_return(&mtl->counter); + mpt->result = 0; + + return pt; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) +{ + struct sync_timeline *tl; + struct sync_pt *pt; + struct sync_fence *fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + struct files_struct *files; + struct fdtable *fdt; +#endif + int fd; + struct file *tl_file; + + tl_file = fget(tl_fd); + if (tl_file == NULL) + return -EBADF; + + if (tl_file->f_op != &stream_fops) { + fd = -EBADF; + goto out; + } + + tl = tl_file->private_data; + + pt = kbase_sync_pt_alloc(tl); + if (!pt) { + fd = -EFAULT; + goto out; + } + + fence = sync_fence_create("mali_fence", pt); + if (!fence) { + sync_pt_free(pt); + fd = -EFAULT; + goto out; + } + + /* from here the fence owns the sync_pt */ + + /* create a fd representing the fence */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } +#else + fd = get_unused_fd(); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } + + files = current->files; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) + __set_close_on_exec(fd, fdt); +#else + FD_SET(fd, fdt->close_on_exec); +#endif + spin_unlock(&files->file_lock); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ + + /* bind fence to the new fd */ + sync_fence_install(fence, fd); + + katom->fence = sync_fence_fdget(fd); + if (katom->fence == NULL) { + /* The only way the fence can be NULL is if userspace closed it + * for us, so we don't need to clear it up */ + fd = -EINVAL; + goto out; + } + +out: + fput(tl_file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ + katom->fence = sync_fence_fdget(fd); + return katom->fence ? 0 : -ENOENT; +} + +int kbase_sync_fence_validate(int fd) +{ + struct sync_fence *fence; + + fence = sync_fence_fdget(fd); + if (!fence) + return -EINVAL; + + sync_fence_put(fence); + return 0; +} + +/* Returns true if the specified timeline is allocated by Mali */ +static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) +{ + return timeline->ops == &mali_timeline_ops; +} + +/* Signals a particular sync point + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + * + * If they are signaled in the wrong order then a message will be printed in + * debug builds and otherwise attempts to signal order sync_pts will be ignored. + * + * result can be negative to indicate error, any other value is interpreted as + * success. + */ +static void kbase_sync_signal_pt(struct sync_pt *pt, int result) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int signaled; + int diff; + + mpt->result = result; + + do { + signaled = atomic_read(&mtl->signaled); + + diff = signaled - mpt->order; + + if (diff > 0) { + /* The timeline is already at or ahead of this point. + * This should not happen unless userspace has been + * signaling fences out of order, so warn but don't + * violate the sync_pt API. + * The warning is only in debug builds to prevent + * a malicious user being able to spam dmesg. + */ +#ifdef CONFIG_MALI_DEBUG + pr_err("Fences were triggered in a different order to allocation!"); +#endif /* CONFIG_MALI_DEBUG */ + return; + } + } while (atomic_cmpxchg(&mtl->signaled, + signaled, mpt->order) != signaled); +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + struct sync_pt *pt; + struct sync_timeline *timeline; + + if (!katom->fence) + return BASE_JD_EVENT_JOB_CANCELLED; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (!list_is_singular(&katom->fence->pt_list_head)) { +#else + if (katom->fence->num_fences != 1) { +#endif + /* Not exactly one item in the list - so it didn't (directly) + * come from us */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + pt = list_first_entry(&katom->fence->pt_list_head, + struct sync_pt, pt_list); +#else + pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); +#endif + timeline = sync_pt_parent(pt); + + if (!kbase_sync_timeline_is_ours(timeline)) { + /* Fence has a sync_pt which isn't ours! */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + kbase_sync_signal_pt(pt, result); + + sync_timeline_signal(timeline); + + kbase_sync_fence_out_remove(katom); + + return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +static inline int kbase_fence_get_status(struct sync_fence *fence) +{ + if (!fence) + return -ENOENT; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return fence->status; +#else + return atomic_read(&fence->status); +#endif +} + +static void kbase_fence_wait_callback(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + struct kbase_jd_atom *katom = container_of(waiter, + struct kbase_jd_atom, sync_waiter); + struct kbase_context *kctx = katom->kctx; + + /* Propagate the fence status to the atom. + * If negative then cancel this atom and its dependencies. + */ + if (kbase_fence_get_status(fence) < 0) + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int ret; + + sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); + + ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); + + if (ret == 1) { + /* Already signaled */ + return 0; + } + + if (ret < 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { + /* The wait wasn't cancelled - leave the cleanup for + * kbase_fence_wait_callback */ + return; + } + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +#ifdef CONFIG_MALI_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Dump out the full state of all the Android sync fences. + * The function sync_dump() isn't exported to modules, so force + * sync_fence_wait() to time out to trigger sync_dump(). + */ + if (katom->fence) + sync_fence_wait(katom->fence, 1); +} +#endif diff --git a/mali_kbase/mali_kbase_sync_common.c b/mali_kbase/mali_kbase_sync_common.c new file mode 100644 index 0000000..457def2 --- /dev/null +++ b/mali_kbase/mali_kbase_sync_common.c @@ -0,0 +1,43 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * @file mali_kbase_sync_common.c + * + * Common code for our explicit fence functionality + */ + +#include <linux/workqueue.h> +#include "mali_kbase.h" + +void kbase_sync_fence_wait_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + + katom = container_of(data, struct kbase_jd_atom, work); + kbase_soft_event_wait_callback(katom); +} + +const char *kbase_sync_status_string(int status) +{ + if (status == 0) + return "signaled"; + else if (status > 0) + return "active"; + else + return "error"; +} diff --git a/mali_kbase/mali_kbase_sync_file.c b/mali_kbase/mali_kbase_sync_file.c new file mode 100644 index 0000000..4e1621c --- /dev/null +++ b/mali_kbase/mali_kbase_sync_file.c @@ -0,0 +1,339 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) + * Introduced in kernel 4.9. + * Android explicit fences (CONFIG_SYNC) can be used for older kernels + * (see mali_kbase_sync_android.c) + */ + +#include <linux/sched.h> +#include <linux/fdtable.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/module.h> +#include <linux/anon_inodes.h> +#include <linux/version.h> +#include <linux/uaccess.h> +#include <linux/sync_file.h> +#include <linux/slab.h> +#include "mali_kbase_fence_defs.h" +#include "mali_kbase_sync.h" +#include "mali_kbase_fence.h" +#include "mali_kbase.h" + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + if (!out_fd) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, NULL, + O_RDONLY | O_CLOEXEC); + if (*out_fd < 0) + return -EINVAL; + + return 0; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct sync_file *sync_file; + int fd; + + fence = kbase_fence_out_new(katom); + if (!fence) + return -ENOMEM; + + /* Take an extra reference to the fence on behalf of the katom. + * This is needed because sync_file_create() will take ownership of + * one of these refs */ + dma_fence_get(fence); + + /* create a sync_file fd representing the fence */ + sync_file = sync_file_create(fence); + if (!sync_file) { + dma_fence_put(fence); + kbase_fence_out_remove(katom); + return -ENOMEM; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + fput(sync_file->file); + kbase_fence_out_remove(katom); + return fd; + } + + fd_install(fd, sync_file->file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -ENOENT; + + kbase_fence_fence_in_set(katom, fence); + + return 0; +} + +int kbase_sync_fence_validate(int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -EINVAL; + + dma_fence_put(fence); + + return 0; /* valid */ +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + int res; + + if (!kbase_fence_out_is_ours(katom)) { + /* Not our fence */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + res = kbase_fence_out_signal(katom, result); + if (unlikely(res < 0)) { + dev_warn(katom->kctx->kbdev->dev, + "fence_signal() failed with %d\n", res); + } + + kbase_sync_fence_out_remove(katom); + + return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_fence_wait_callback(struct fence *fence, + struct fence_cb *cb) +#else +static void kbase_fence_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + struct kbase_context *kctx = katom->kctx; + + /* Cancel atom if fence is erroneous */ + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + if (kbase_fence_dep_count_dec_and_test(katom)) { + /* We take responsibility of handling this */ + kbase_fence_dep_count_set(katom, -1); + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); + } +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int err; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return 0; /* no input fence to wait for, good to go! */ + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); + + kbase_fence_put(fence); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + return 0; /* Already signaled, good to go right now */ + } + + /* Callback installed, so we just need to wait for it... */ + } else { + /* Failure */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; /* completion to be done later by callback/worker */ +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (!kbase_fence_free_callbacks(katom)) { + /* The wait wasn't cancelled - + * leave the cleanup for kbase_fence_wait_callback */ + return; + } + + /* Take responsibility of completion */ + kbase_fence_dep_count_set(katom, -1); + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_out_remove(katom); +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_free_callbacks(katom); + kbase_fence_in_remove(katom); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_sync_fence_info_get(struct fence *fence, + struct kbase_sync_fence_info *info) +#else +static void kbase_sync_fence_info_get(struct dma_fence *fence, + struct kbase_sync_fence_info *info) +#endif +{ + info->fence = fence; + + /* translate into CONFIG_SYNC status: + * < 0 : error + * 0 : active + * 1 : signaled + */ + if (dma_fence_is_signaled(fence)) { + if (fence->status < 0) + info->status = fence->status; /* signaled with error */ + else + info->status = 1; /* signaled with success */ + } else { + info->status = 0; /* still active (unsignaled) */ + } + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + scnprintf(info->name, sizeof(info->name), "%u#%u", + fence->context, fence->seqno); +#else + scnprintf(info->name, sizeof(info->name), "%llu#%u", + fence->context, fence->seqno); +#endif +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_out_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + + +#ifdef CONFIG_MALI_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Not implemented */ +} +#endif diff --git a/mali_kbase/mali_kbase_sync_user.c b/mali_kbase/mali_kbase_sync_user.c deleted file mode 100644 index b9baa91..0000000 --- a/mali_kbase/mali_kbase_sync_user.c +++ /dev/null @@ -1,156 +0,0 @@ -/* - * - * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - * - */ - - - - - -/** - * @file mali_kbase_sync_user.c - * - */ - -#ifdef CONFIG_SYNC - -#include <linux/sched.h> -#include <linux/fdtable.h> -#include <linux/file.h> -#include <linux/fs.h> -#include <linux/module.h> -#include <linux/anon_inodes.h> -#include <linux/version.h> -#include <linux/uaccess.h> -#include <mali_kbase_sync.h> - -static int kbase_stream_close(struct inode *inode, struct file *file) -{ - struct sync_timeline *tl; - - tl = (struct sync_timeline *)file->private_data; - BUG_ON(!tl); - sync_timeline_destroy(tl); - return 0; -} - -static const struct file_operations stream_fops = { - .owner = THIS_MODULE, - .release = kbase_stream_close, -}; - -int kbase_stream_create(const char *name, int *const out_fd) -{ - struct sync_timeline *tl; - - BUG_ON(!out_fd); - - tl = kbase_sync_timeline_alloc(name); - if (!tl) - return -EINVAL; - - *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY | O_CLOEXEC); - - if (*out_fd < 0) { - sync_timeline_destroy(tl); - return -EINVAL; - } - - return 0; -} - -int kbase_stream_create_fence(int tl_fd) -{ - struct sync_timeline *tl; - struct sync_pt *pt; - struct sync_fence *fence; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) - struct files_struct *files; - struct fdtable *fdt; -#endif - int fd; - struct file *tl_file; - - tl_file = fget(tl_fd); - if (tl_file == NULL) - return -EBADF; - - if (tl_file->f_op != &stream_fops) { - fd = -EBADF; - goto out; - } - - tl = tl_file->private_data; - - pt = kbase_sync_pt_alloc(tl); - if (!pt) { - fd = -EFAULT; - goto out; - } - - fence = sync_fence_create("mali_fence", pt); - if (!fence) { - sync_pt_free(pt); - fd = -EFAULT; - goto out; - } - - /* from here the fence owns the sync_pt */ - - /* create a fd representing the fence */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) - fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); - if (fd < 0) { - sync_fence_put(fence); - goto out; - } -#else - fd = get_unused_fd(); - if (fd < 0) { - sync_fence_put(fence); - goto out; - } - - files = current->files; - spin_lock(&files->file_lock); - fdt = files_fdtable(files); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) - __set_close_on_exec(fd, fdt); -#else - FD_SET(fd, fdt->close_on_exec); -#endif - spin_unlock(&files->file_lock); -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ - - /* bind fence to the new fd */ - sync_fence_install(fence, fd); - - out: - fput(tl_file); - - return fd; -} - -int kbase_fence_validate(int fd) -{ - struct sync_fence *fence; - - fence = sync_fence_fdget(fd); - if (!fence) - return -EINVAL; - - sync_fence_put(fence); - return 0; -} - -#endif /* CONFIG_SYNC */ diff --git a/mali_kbase/mali_kbase_tlstream.c b/mali_kbase/mali_kbase_tlstream.c index aa7850a..c952993 100644 --- a/mali_kbase/mali_kbase_tlstream.c +++ b/mali_kbase/mali_kbase_tlstream.c @@ -145,6 +145,7 @@ enum tl_msg_id_obj { KBASE_TL_ATTRIB_ATOM_PRIORITY, KBASE_TL_ATTRIB_ATOM_STATE, KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, + KBASE_TL_ATTRIB_ATOM_JIT, KBASE_TL_ATTRIB_AS_CONFIG, KBASE_TL_EVENT_LPU_SOFTSTOP, KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, @@ -459,6 +460,13 @@ static const struct tp_desc tp_desc_obj[] = { "atom" }, { + KBASE_TL_ATTRIB_ATOM_JIT, + __stringify(KBASE_TL_ATTRIB_ATOM_JIT), + "jit done for atom", + "@pLL", + "atom,edit_addr,new_addr" + }, + { KBASE_TL_ATTRIB_AS_CONFIG, __stringify(KBASE_TL_ATTRIB_AS_CONFIG), "address space attributes", @@ -1380,21 +1388,77 @@ void kbase_tlstream_term(void) } } -int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags) +static void kbase_create_timeline_objects(struct kbase_context *kctx) { + struct kbase_device *kbdev = kctx->kbdev; + unsigned int lpu_id; + unsigned int as_nr; + struct kbasep_kctx_list_element *element; + + /* Create LPU objects. */ + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + u32 *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); + } + + /* Create Address Space objects. */ + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); + + /* Create GPU object and make it retain all LPUs and address spaces. */ + KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( + kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores); + + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + void *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); + } + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( + &kbdev->as[as_nr], + kbdev); + + /* Create object for each known context. */ + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(element, &kbdev->kctx_list, link) { + KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( + element->kctx, + (u32)(element->kctx->id), + (u32)(element->kctx->tgid)); + } + /* Before releasing the lock, reset body stream buffers. + * This will prevent context creation message to be directed to both + * summary and body stream. + */ + kbase_tlstream_reset_body_streams(); + mutex_unlock(&kbdev->kctx_list_lock); + /* Static object are placed into summary packet that needs to be + * transmitted first. Flush all streams to make it available to + * user space. + */ + kbase_tlstream_flush_streams(); +} + +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) +{ + int ret; u32 tlstream_enabled = TLSTREAM_ENABLED | flags; if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { int rcode; - *fd = anon_inode_getfd( + ret = anon_inode_getfd( "[mali_tlstream]", &kbasep_tlstream_fops, kctx, O_RDONLY | O_CLOEXEC); - if (0 > *fd) { + if (ret < 0) { atomic_set(&kbase_tlstream_enabled, 0); - return *fd; + return ret; } /* Reset and initialize header streams. */ @@ -1430,11 +1494,17 @@ int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags) 1800000); } + /* Summary stream was cleared during acquire. + * Create static timeline objects that will be + * read by client. + */ + kbase_create_timeline_objects(kctx); + } else { - *fd = -EBUSY; + ret = -EBUSY; } - return 0; + return ret; } void kbase_tlstream_flush_streams(void) @@ -2161,6 +2231,35 @@ void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); } +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(edit_addr) + sizeof(new_addr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &edit_addr, sizeof(edit_addr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &new_addr, sizeof(new_addr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + void __kbase_tlstream_tl_attrib_as_config( void *as, u64 transtab, u64 memattr, u64 transcfg) { diff --git a/mali_kbase/mali_kbase_tlstream.h b/mali_kbase/mali_kbase_tlstream.h index 69e188e..c0a1117 100644 --- a/mali_kbase/mali_kbase_tlstream.h +++ b/mali_kbase/mali_kbase_tlstream.h @@ -38,7 +38,6 @@ void kbase_tlstream_term(void); /** * kbase_tlstream_acquire - acquire timeline stream file descriptor * @kctx: kernel common context - * @fd: timeline stream file descriptor * @flags: timeline stream flags * * This descriptor is meant to be used by userspace timeline to gain access to @@ -46,12 +45,11 @@ void kbase_tlstream_term(void); * timeline client. * Only one entity can own the descriptor at any given time. Descriptor shall be * closed if unused. If descriptor cannot be obtained (i.e. when it is already - * being used) argument fd will contain negative value. + * being used) return will be a negative value. * - * Return: zero on success (this does not necessarily mean that stream - * descriptor could be returned), negative number on error + * Return: file descriptor on success, negative number on error */ -int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags); +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); /** * kbase_tlstream_flush_streams - flush timeline streams. @@ -137,6 +135,8 @@ void __kbase_tlstream_tl_attrib_atom_config( void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr); void __kbase_tlstream_tl_attrib_as_config( void *as, u64 transtab, u64 memattr, u64 transcfg); void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); @@ -170,6 +170,13 @@ extern atomic_t kbase_tlstream_enabled; __kbase_tlstream_##trace_name(__VA_ARGS__); \ } while (0) +#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + /*****************************************************************************/ /** @@ -482,6 +489,15 @@ extern atomic_t kbase_tlstream_enabled; __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) /** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom + * @atom: atom identifier + * @edit_addr: address edited by jit + * @new_addr: address placed into the edited location + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ + __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) + +/** * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes * @as: assigned address space * @transtab: configuration of the TRANSTAB register diff --git a/mali_kbase/mali_kbase_uku.h b/mali_kbase/mali_kbase_uku.h index 3f2cb15..f3fcde4 100644 --- a/mali_kbase/mali_kbase_uku.h +++ b/mali_kbase/mali_kbase_uku.h @@ -66,6 +66,8 @@ #define BASE_UK_VERSION_MAJOR 10 #define BASE_UK_VERSION_MINOR 6 +#define LINUX_UK_BASE_MAGIC 0x80 + struct kbase_uk_mem_alloc { union uk_header header; /* IN */ diff --git a/mali_kbase/mali_kbase_vinstr.c b/mali_kbase/mali_kbase_vinstr.c index 7f7efc8..b2d46ed 100644 --- a/mali_kbase/mali_kbase_vinstr.c +++ b/mali_kbase/mali_kbase_vinstr.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -1160,6 +1160,28 @@ static enum hrtimer_restart kbasep_vinstr_wake_up_callback( return HRTIMER_NORESTART; } +#ifdef CONFIG_DEBUG_OBJECT_TIMERS +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +/** + * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(), + * rewritten + * + * @timer: high resolution timer + * + * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for + * earlier kernel versions it is not possible to call it explicitly. + * Since this function must accompany hrtimer_init_on_stack(), which + * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS + * is defined in order to avoid the warning about object on stack not being + * annotated, we rewrite it here to be used for earlier kernel versions. + */ +static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer) +{ + debug_object_free(timer, &hrtimer_debug_descr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */ +#endif /* CONFIG_DEBUG_OBJECT_TIMERS */ + /** * kbasep_vinstr_service_task - HWC dumping service thread * @@ -1174,7 +1196,8 @@ static int kbasep_vinstr_service_task(void *data) KBASE_DEBUG_ASSERT(vinstr_ctx); - hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + timer.hrtimer.function = kbasep_vinstr_wake_up_callback; timer.vinstr_ctx = vinstr_ctx; @@ -1277,6 +1300,14 @@ static int kbasep_vinstr_service_task(void *data) mutex_unlock(&vinstr_ctx->lock); } +#ifdef CONFIG_DEBUG_OBJECTS_TIMERS +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) + kbase_destroy_hrtimer_on_stack(&timer.hrtimer); +#else + destroy_hrtimer_on_stack(&timer.hrtimer); +#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */ +#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */ + return 0; } diff --git a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h index 34f6d57..49e107f 100644 --- a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h +++ b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -71,10 +71,3 @@ #define PLATFORM_FUNCS (NULL) extern struct kbase_pm_callback_conf pm_callbacks; - -/** - * Protected mode switch - * - * Attached value: pointer to @ref kbase_protected_ops - */ -#define PROTECTED_CALLBACKS (NULL) diff --git a/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c b/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c deleted file mode 100644 index d237a38..0000000 --- a/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - * - */ - - - -#include <linux/ioport.h> -#include <linux/thermal.h> -#include <mali_kbase.h> -#include <mali_kbase_defs.h> -#include <mali_kbase_config.h> -#include <mali_kbase_smc.h> - -/* Versatile Express (VE) Juno Development Platform */ - -#define HARD_RESET_AT_POWER_OFF 0 - -#ifndef CONFIG_OF -static struct kbase_io_resources io_resources = { - .job_irq_number = 65, - .mmu_irq_number = 66, - .gpu_irq_number = 64, - .io_memory_region = { - .start = 0x2D000000, - .end = 0x2D000000 + (4096 * 4) - 1} -}; -#endif - -static int pm_callback_power_on(struct kbase_device *kbdev) -{ - /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ - return 1; -} - -static void pm_callback_power_off(struct kbase_device *kbdev) -{ -#if HARD_RESET_AT_POWER_OFF - /* Cause a GPU hard reset to test whether we have actually idled the GPU - * and that we properly reconfigure the GPU on power up. - * Usually this would be dangerous, but if the GPU is working correctly it should - * be completely safe as the GPU should not be active at this point. - * However this is disabled normally because it will most likely interfere with - * bus logging etc. - */ - KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); - kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); -#endif -} - -struct kbase_pm_callback_conf pm_callbacks = { - .power_on_callback = pm_callback_power_on, - .power_off_callback = pm_callback_power_off, - .power_suspend_callback = NULL, - .power_resume_callback = NULL -}; - -/* - * Juno Protected Mode integration - */ - -/* SMC Function Numbers */ -#define JUNO_SMC_PROTECTED_ENTER_FUNC 0xff06 -#define JUNO_SMC_PROTECTED_RESET_FUNC 0xff07 - -static int juno_protected_mode_enter(struct kbase_device *kbdev) -{ - /* T62X in SoC detected */ - u64 ret = kbase_invoke_smc(SMC_OEN_SIP, - JUNO_SMC_PROTECTED_ENTER_FUNC, false, - 0, 0, 0); - return ret; -} - -/* TODO: Remove these externs, reset should should be done by the firmware */ -extern void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, - struct kbase_context *kctx); - -extern u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, - struct kbase_context *kctx); - -static int juno_protected_mode_reset(struct kbase_device *kbdev) -{ - - /* T62X in SoC detected */ - u64 ret = kbase_invoke_smc(SMC_OEN_SIP, - JUNO_SMC_PROTECTED_RESET_FUNC, false, - 0, 0, 0); - - /* The GPU should now be reset */ - - return ret; -} - -static bool juno_protected_mode_supported(struct kbase_device *kbdev) -{ - u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; - - /* - * Protected mode is only supported for the built in GPU - * _and_ only if the right firmware is running. - * - * Given that at init time the GPU is not powered up the - * juno_protected_mode_reset function can't be used as - * is needs to access GPU registers. - * However, although we don't want the GPU to boot into - * protected mode we know a GPU reset will be done after - * this function is called so although we set the GPU to - * protected mode it will exit protected mode before the - * driver is ready to run work. - */ - if (gpu_id == GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0) && - (kbdev->reg_start == 0x2d000000)) - return juno_protected_mode_enter(kbdev) == 0; - - return false; -} - -struct kbase_protected_ops juno_protected_ops = { - .protected_mode_enter = juno_protected_mode_enter, - .protected_mode_reset = juno_protected_mode_reset, - .protected_mode_supported = juno_protected_mode_supported, -}; - -static struct kbase_platform_config versatile_platform_config = { -#ifndef CONFIG_OF - .io_resources = &io_resources -#endif -}; - -struct kbase_platform_config *kbase_get_platform_config(void) -{ - return &versatile_platform_config; -} - -int kbase_platform_early_init(void) -{ - /* Nothing needed at this stage */ - return 0; -} diff --git a/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h b/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h deleted file mode 100644 index ab29e9d..0000000 --- a/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - * - */ - - - -/** - * Maximum frequency GPU will be clocked at. Given in kHz. - * This must be specified as there is no default value. - * - * Attached value: number in kHz - * Default value: NA - */ -#define GPU_FREQ_KHZ_MAX 600000 -/** - * Minimum frequency GPU will be clocked at. Given in kHz. - * This must be specified as there is no default value. - * - * Attached value: number in kHz - * Default value: NA - */ -#define GPU_FREQ_KHZ_MIN 600000 - -/** - * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock - * - * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func - * for the function prototype. - * - * Attached value: A kbase_cpu_clk_speed_func. - * Default Value: NA - */ -#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) - -/** - * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock - * - * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func - * for the function prototype. - * - * Attached value: A kbase_gpu_clk_speed_func. - * Default Value: NA - */ -#define GPU_SPEED_FUNC (NULL) - -/** - * Power management configuration - * - * Attached value: pointer to @ref kbase_pm_callback_conf - * Default value: See @ref kbase_pm_callback_conf - */ -#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) - -/** - * Platform specific configuration functions - * - * Attached value: pointer to @ref kbase_platform_funcs_conf - * Default value: See @ref kbase_platform_funcs_conf - */ -#define PLATFORM_FUNCS (NULL) - -/** - * Protected mode switch - * - * Attached value: pointer to @ref kbase_protected_ops - */ -#define PROTECTED_CALLBACKS (&juno_protected_ops) - -extern struct kbase_pm_callback_conf pm_callbacks; -#ifdef CONFIG_DEVFREQ_THERMAL -extern struct devfreq_cooling_ops juno_model_ops; -#endif -extern struct kbase_protected_ops juno_protected_ops; diff --git a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h index dc4471b..02835f1 100644 --- a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -72,11 +72,4 @@ */ #define PLATFORM_FUNCS (NULL) -/** - * Protected mode switch - * - * Attached value: pointer to @ref kbase_protected_ops - */ -#define PROTECTED_CALLBACKS (NULL) - extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h index b0490ca..0efbf39 100644 --- a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -70,11 +70,4 @@ */ #define PLATFORM_FUNCS (NULL) -/** - * Protected mode switch - * - * Attached value: pointer to @ref kbase_protected_ops - */ -#define PROTECTED_CALLBACKS (NULL) - extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h index 22ffccb..dbdf21e 100644 --- a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -72,11 +72,4 @@ */ #define PLATFORM_FUNCS (NULL) -/** - * Protected mode switch - * - * Attached value: pointer to @ref kbase_protected_ops - */ -#define PROTECTED_CALLBACKS (NULL) - extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/mali_kbase/protected_mode_switcher.h b/mali_kbase/protected_mode_switcher.h new file mode 100644 index 0000000..5dc2f3b --- /dev/null +++ b/mali_kbase/protected_mode_switcher.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _PROTECTED_MODE_SWITCH_H_ +#define _PROTECTED_MODE_SWITCH_H_ + +struct protected_mode_device; + +/** + * struct protected_mode_ops - Callbacks for protected mode switch operations + * + * @protected_mode_enable: Callback to enable protected mode for device + * @protected_mode_disable: Callback to disable protected mode for device + */ +struct protected_mode_ops { + /** + * protected_mode_enable() - Enable protected mode on device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_enable)( + struct protected_mode_device *protected_dev); + + /** + * protected_mode_disable() - Disable protected mode on device, and + * reset device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_disable)( + struct protected_mode_device *protected_dev); +}; + +/** + * struct protected_mode_device - Device structure for protected mode devices + * + * @ops - Callbacks associated with this device + * @data - Pointer to device private data + * + * This structure should be registered with the platform device using + * platform_set_drvdata(). + */ +struct protected_mode_device { + struct protected_mode_ops ops; + void *data; +}; + +#endif /* _PROTECTED_MODE_SWITCH_H_ */ diff --git a/mali_kbase/sconscript b/mali_kbase/sconscript index 137d532..ff23d7a 100644 --- a/mali_kbase/sconscript +++ b/mali_kbase/sconscript @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -16,14 +16,13 @@ import sys Import('env') -if Glob('tests/sconscript'): - SConscript( 'tests/sconscript' ) +SConscript( 'tests/sconscript' ) mock_test = 0 # Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data. # For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0. -if env['platform_config']=='devicetree': +if env['platform_config']=='devicetree' or env['platform_config']=='juno_soc': fake_platform_device = 0 else: fake_platform_device = 1 @@ -33,10 +32,14 @@ kbase_src = [ Glob('*.c'), Glob('backend/*/*.c'), Glob('internal/*/*.c'), - Glob('ipa/*.c'), - Glob('platform/%s/*.c' % env['platform_config']), + Glob('ipa/*.c') ] +if env['platform_config']=='juno_soc': + kbase_src += [Glob('platform/devicetree/*.c')] +else: + kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] + if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] mock_test = 1 @@ -57,14 +60,14 @@ make_args = env.kernel_get_config_defines(ret_list = True, fake = fake_platform_device) + [ 'PLATFORM=%s' % env['platform'], 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], - 'MALI_KERNEL_TEST_API=%s' % env['unit'], + 'MALI_KERNEL_TEST_API=%s' % env['debug'], 'MALI_UNIT_TEST=%s' % env['unit'], 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], 'MALI_MOCK_TEST=%s' % mock_test, 'MALI_CUSTOMER_RELEASE=%s' % env['release'], 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], 'MALI_COVERAGE=%s' % env['coverage'], - 'MALI_BUS_LOG=%s' % env['buslog'], + 'MALI_BUS_LOG=%s' % env['buslog'] ] kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, @@ -81,6 +84,9 @@ if env['os'] != 'android': if int(env['ump']) == 1: env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') +if 'smc_protected_mode_switcher' in env: + env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') + env.KernelObjTarget('kbase', kbase) env.AppendUnique(BASE=['cutils_linked_list']) diff --git a/mali_kbase/platform/juno_soc/Kbuild b/mali_kbase/tests/Kbuild index 5e1ce63..b4bed04 100644 --- a/mali_kbase/platform/juno_soc/Kbuild +++ b/mali_kbase/tests/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -13,4 +13,5 @@ # -mali_kbase-y += $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_juno_soc.o +obj-$(CONFIG_MALI_KUTF) += kutf/ +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ diff --git a/mali_kbase/tests/Kconfig b/mali_kbase/tests/Kconfig new file mode 100644 index 0000000..da0515c --- /dev/null +++ b/mali_kbase/tests/Kconfig @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" +source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" diff --git a/mali_kbase/tests/include/kutf/kutf_mem.h b/mali_kbase/tests/include/kutf/kutf_mem.h new file mode 100644 index 0000000..0d145e4 --- /dev/null +++ b/mali_kbase/tests/include/kutf/kutf_mem.h @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_MEM_H_ +#define _KERNEL_UTF_MEM_H_ + +/* kutf_mem.h + * Functions for management of memory pools in the kernel. + * + * This module implements a memory pool allocator, allowing a test + * implementation to allocate linked allocations which can then be freed by a + * single free which releases all of the resources held by the entire pool. + * + * Note that it is not possible to free single resources within the pool once + * allocated. + */ + +#include <linux/list.h> + +/** + * struct kutf_mempool - the memory pool context management structure + * @head: list head on which the allocations in this context are added to + * + */ +struct kutf_mempool { + struct list_head head; +}; + +/** + * kutf_mempool_init() - Initialize a memory pool. + * @pool: Memory pool structure to initialize, provided by the user + * + * Return: zero on success + */ +int kutf_mempool_init(struct kutf_mempool *pool); + +/** + * kutf_mempool_alloc() - Allocate memory from a pool + * @pool: Memory pool to allocate from + * @size: Size of memory wanted in number of bytes + * + * Return: Pointer to memory on success, NULL on failure. + */ +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); + +/** + * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. + * @pool: The memory pool to free + */ +void kutf_mempool_destroy(struct kutf_mempool *pool); +#endif /* _KERNEL_UTF_MEM_H_ */ diff --git a/mali_kbase/tests/include/kutf/kutf_resultset.h b/mali_kbase/tests/include/kutf/kutf_resultset.h new file mode 100644 index 0000000..1cc85f1 --- /dev/null +++ b/mali_kbase/tests/include/kutf/kutf_resultset.h @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_RESULTSET_H_ +#define _KERNEL_UTF_RESULTSET_H_ + +/* kutf_resultset.h + * Functions and structures for handling test results and result sets. + * + * This section of the kernel UTF contains structures and functions used for the + * management of Results and Result Sets. + */ + +/** + * enum kutf_result_status - Status values for a single Test error. + * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark + * results. + * @KUTF_RESULT_SKIP: The test was skipped. + * @KUTF_RESULT_UNKNOWN: The test has an unknown result. + * @KUTF_RESULT_PASS: The test result passed. + * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug + * message. + * @KUTF_RESULT_INFO: The test result passed, but raised + * an informative message. + * @KUTF_RESULT_WARN: The test result passed, but raised a warning + * message. + * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. + * @KUTF_RESULT_FATAL: The test result failed with a fatal error. + * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF + * assertion failure. + * @KUTF_RESULT_COUNT: The current number of possible status messages. + */ +enum kutf_result_status { + KUTF_RESULT_BENCHMARK = -3, + KUTF_RESULT_SKIP = -2, + KUTF_RESULT_UNKNOWN = -1, + + KUTF_RESULT_PASS = 0, + KUTF_RESULT_DEBUG = 1, + KUTF_RESULT_INFO = 2, + KUTF_RESULT_WARN = 3, + KUTF_RESULT_FAIL = 4, + KUTF_RESULT_FATAL = 5, + KUTF_RESULT_ABORT = 6, + + KUTF_RESULT_COUNT +}; + +/* The maximum size of a kutf_result_status result when + * converted to a string + */ +#define KUTF_ERROR_MAX_NAME_SIZE 21 + +#ifdef __KERNEL__ + +#include <kutf/kutf_mem.h> + +/** + * struct kutf_result - Represents a single test result. + * @node: Next result in the list of results. + * @status: The status summary (pass / warn / fail / etc). + * @message: A more verbose status message. + */ +struct kutf_result { + struct list_head node; + enum kutf_result_status status; + const char *message; +}; + +/** + * kutf_create_result_set() - Create a new result set + * to which results can be added. + * + * Return: The created resultset. + */ +struct kutf_result_set *kutf_create_result_set(void); + +/** + * kutf_add_result() - Add a result to the end of an existing resultset. + * + * @mempool: The memory pool to allocate the result storage from. + * @set: The resultset to add the result to. + * @status: The result status to add. + * @message: The result message to add. + */ +void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, + enum kutf_result_status status, const char *message); + +/** + * kutf_remove_result() - Remove a result from the head of a resultset. + * @set: The resultset. + * + * Return: result or NULL if there are no further results in the resultset. + */ +struct kutf_result *kutf_remove_result( + struct kutf_result_set *set); + +/** + * kutf_destroy_result_set() - Free a previously created resultset. + * + * @results: The result set whose resources to free. + */ +void kutf_destroy_result_set(struct kutf_result_set *results); + +#endif /* __KERNEL__ */ + +#endif /* _KERNEL_UTF_RESULTSET_H_ */ diff --git a/mali_kbase/tests/include/kutf/kutf_suite.h b/mali_kbase/tests/include/kutf/kutf_suite.h new file mode 100644 index 0000000..754c3ad --- /dev/null +++ b/mali_kbase/tests/include/kutf/kutf_suite.h @@ -0,0 +1,508 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_SUITE_H_ +#define _KERNEL_UTF_SUITE_H_ + +/* kutf_suite.h + * Functions for management of test suites. + * + * This collection of data structures, macros, and functions are used to + * create Test Suites, Tests within those Test Suites, and Fixture variants + * of each test. + */ + +#include <kutf/kutf_mem.h> +#include <kutf/kutf_resultset.h> + +/** + * Pseudo-flag indicating an absence of any specified test class. Note that + * tests should not be annotated with this constant as it is simply a zero + * value; tests without a more specific class must be marked with the flag + * KUTF_F_TEST_GENERIC. + */ +#define KUTF_F_TEST_NONE ((unsigned int)(0)) + +/** + * Class indicating this test is a smoke test. + * A given set of smoke tests should be quick to run, enabling rapid turn-around + * of "regress-on-commit" test runs. + */ +#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) + +/** + * Class indicating this test is a performance test. + * These tests typically produce a performance metric, such as "time to run" or + * "frames per second", + */ +#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) + +/** + * Class indicating that this test is a deprecated test. + * These tests have typically been replaced by an alternative test which is + * more efficient, or has better coverage. + */ +#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) + +/** + * Class indicating that this test is a known failure. + * These tests have typically been run and failed, but marking them as a known + * failure means it is easier to triage results. + * + * It is typically more convenient to triage known failures using the + * results database and web UI, as this means there is no need to modify the + * test code. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) + +/** + * Class indicating that this test is a generic test, which is not a member of + * a more specific test class. Tests which are not created with a specific set + * of filter flags by the user are assigned this test class by default. + */ +#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) + +/** + * Class indicating this test is a resource allocation failure test. + * A resource allocation failure test will test that an error code is + * correctly propagated when an allocation fails. + */ +#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) + +/** + * Additional flag indicating that this test is an expected failure when + * run in resource failure mode. These tests are never run when running + * the low resource mode. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) + +/** + * Flag reserved for user-defined filter zero. + */ +#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) + +/** + * Flag reserved for user-defined filter one. + */ +#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) + +/** + * Flag reserved for user-defined filter two. + */ +#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) + +/** + * Flag reserved for user-defined filter three. + */ +#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) + +/** + * Flag reserved for user-defined filter four. + */ +#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) + +/** + * Flag reserved for user-defined filter five. + */ +#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) + +/** + * Flag reserved for user-defined filter six. + */ +#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) + +/** + * Flag reserved for user-defined filter seven. + */ +#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) + +/** + * Pseudo-flag indicating that all test classes should be executed. + */ +#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) + +/** + * union kutf_callback_data - Union used to store test callback data + * @ptr_value: pointer to the location where test callback data + * are stored + * @u32_value: a number which represents test callback data + */ +union kutf_callback_data { + void *ptr_value; + u32 u32_value; +}; + +/** + * struct kutf_context - Structure representing a kernel test context + * @suite: Convenience pointer to the suite this context + * is running + * @test_fix: The fixture that is being run in this context + * @fixture_pool: The memory pool used for the duration of + * the fixture/text context. + * @fixture: The user provided fixture structure. + * @fixture_index: The index (id) of the current fixture. + * @fixture_name: The name of the current fixture (or NULL if unnamed). + * @test_data: Any user private data associated with this test + * @result_set: All the results logged by this test context + * @status: The status of the currently running fixture. + * @expected_status: The expected status on exist of the currently + * running fixture. + */ +struct kutf_context { + struct kutf_suite *suite; + struct kutf_test_fixture *test_fix; + struct kutf_mempool fixture_pool; + void *fixture; + unsigned int fixture_index; + const char *fixture_name; + union kutf_callback_data test_data; + struct kutf_result_set *result_set; + enum kutf_result_status status; + enum kutf_result_status expected_status; +}; + +/** + * struct kutf_suite - Structure representing a kernel test suite + * @app: The application this suite belongs to. + * @name: The name of this suite. + * @suite_data: Any user private data associated with this + * suite. + * @create_fixture: Function used to create a new fixture instance + * @remove_fixture: Function used to destroy a new fixture instance + * @fixture_variants: The number of variants (must be at least 1). + * @suite_default_flags: Suite global filter flags which are set on + * all tests. + * @node: List node for suite_list + * @dir: The debugfs directory for this suite + * @test_list: List head to store all the tests which are + * part of this suite + */ +struct kutf_suite { + struct kutf_application *app; + const char *name; + union kutf_callback_data suite_data; + void *(*create_fixture)(struct kutf_context *context); + void (*remove_fixture)(struct kutf_context *context); + unsigned int fixture_variants; + unsigned int suite_default_flags; + struct list_head node; + struct dentry *dir; + struct list_head test_list; +}; + +/* ============================================================================ + Application functions +============================================================================ */ + +/** + * kutf_create_application() - Create an in kernel test application. + * @name: The name of the test application. + * + * Return: pointer to the kutf_application on success or NULL + * on failure + */ +struct kutf_application *kutf_create_application(const char *name); + +/** + * kutf_destroy_application() - Destroy an in kernel test application. + * + * @app: The test application to destroy. + */ +void kutf_destroy_application(struct kutf_application *app); + +/* ============================================================================ + Suite functions +============================================================================ */ + +/** + * kutf_create_suite() - Create a kernel test suite. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)); + +/** + * kutf_create_suite_with_filters() - Create a kernel test suite with user + * defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL on failure + */ +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with + * user defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * @suite_data: Suite specific callback data, provided during the + * running of the test in the kutf_context + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data); + +/** + * kutf_add_test() - Add a test to a kernel test suite. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * + * Note: As no filters are provided the test will use the suite filters instead + */ +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)); + +/** + * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + */ +void kutf_add_test_with_filters(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite + * with filters. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + * @test_data: Test specific callback data, provoided during the + * running of the test in the kutf_context + */ +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data); + +/* ============================================================================ + Test functions +============================================================================ */ +/** + * kutf_test_log_result_external() - Log a result which has been created + * externally into a in a standard form + * recognized by the log parser. + * @context: The test context the test is running in + * @message: The message for this result + * @new_status: The result status of this log message + */ +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status); + +/** + * kutf_test_expect_abort() - Tell the kernel that you expect the current + * fixture to produce an abort. + * @context: The test context this test is running in. + */ +void kutf_test_expect_abort(struct kutf_context *context); + +/** + * kutf_test_expect_fatal() - Tell the kernel that you expect the current + * fixture to produce a fatal error. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fatal(struct kutf_context *context); + +/** + * kutf_test_expect_fail() - Tell the kernel that you expect the current + * fixture to fail. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fail(struct kutf_context *context); + +/** + * kutf_test_expect_warn() - Tell the kernel that you expect the current + * fixture to produce a warning. + * @context: The test context this test is running in. + */ +void kutf_test_expect_warn(struct kutf_context *context); + +/** + * kutf_test_expect_pass() - Tell the kernel that you expect the current + * fixture to pass. + * @context: The test context this test is running in. + */ +void kutf_test_expect_pass(struct kutf_context *context); + +/** + * kutf_test_skip() - Tell the kernel that the test should be skipped. + * @context: The test context this test is running in. + */ +void kutf_test_skip(struct kutf_context *context); + +/** + * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, + * supplying a reason string. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the skip. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a prebaked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_skip_msg(struct kutf_context *context, const char *message); + +/** + * kutf_test_pass() - Tell the kernel that this test has passed. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the pass. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_pass(struct kutf_context *context, char const *message); + +/** + * kutf_test_debug() - Send a debug message + * @context: The test context this test is running in. + * @message: A message string containing the debug information. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_debug(struct kutf_context *context, char const *message); + +/** + * kutf_test_info() - Send an information message + * @context: The test context this test is running in. + * @message: A message string containing the information message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_info(struct kutf_context *context, char const *message); + +/** + * kutf_test_warn() - Send a warning message + * @context: The test context this test is running in. + * @message: A message string containing the warning message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_warn(struct kutf_context *context, char const *message); + +/** + * kutf_test_fail() - Tell the kernel that a test has failed + * @context: The test context this test is running in. + * @message: A message string containing the failure message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fail(struct kutf_context *context, char const *message); + +/** + * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error + * @context: The test context this test is running in. + * @message: A message string containing the fatal error message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fatal(struct kutf_context *context, char const *message); + +/** + * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test + * + * @context: The test context this test is running in. + */ +void kutf_test_abort(struct kutf_context *context); + +#endif /* _KERNEL_UTF_SUITE_H_ */ diff --git a/mali_kbase/tests/include/kutf/kutf_utils.h b/mali_kbase/tests/include/kutf/kutf_utils.h new file mode 100644 index 0000000..c458c1f --- /dev/null +++ b/mali_kbase/tests/include/kutf/kutf_utils.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_UTILS_H_ +#define _KERNEL_UTF_UTILS_H_ + +/* kutf_utils.h + * Utilities for the kernel UTF test infrastructure. + * + * This collection of library functions are provided for use by kernel UTF + * and users of kernel UTF which don't directly fit within the other + * code modules. + */ + +#include <kutf/kutf_mem.h> + +/** + * Maximum size of the message strings within kernel UTF, messages longer then + * this will be truncated. + */ +#define KUTF_MAX_DSPRINTF_LEN 1024 + +/** + * kutf_dsprintf() - dynamic sprintf + * @pool: memory pool to allocate from + * @fmt: The format string describing the string to document. + * @... The parameters to feed in to the format string. + * + * This function implements sprintf which dynamically allocates memory to store + * the string. The library will free the memory containing the string when the + * result set is cleared or destroyed. + * + * Note The returned string may be truncated to fit an internal temporary + * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. + * + * Return: Returns pointer to allocated string, or NULL on error. + */ +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...); + +#endif /* _KERNEL_UTF_UTILS_H_ */ diff --git a/mali_kbase/tests/kutf/Kbuild b/mali_kbase/tests/kutf/Kbuild new file mode 100644 index 0000000..6b840c2 --- /dev/null +++ b/mali_kbase/tests/kutf/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include + +obj-$(CONFIG_MALI_KUTF) += kutf.o + +kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o diff --git a/mali_kbase/tests/kutf/Kconfig b/mali_kbase/tests/kutf/Kconfig new file mode 100644 index 0000000..6a87bdb --- /dev/null +++ b/mali_kbase/tests/kutf/Kconfig @@ -0,0 +1,22 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +config MALI_KUTF + tristate "Mali Kernel Unit Test Framework" + default m + help + Enables MALI testing framework. To compile it as a module, + choose M here - this will generate a single module called kutf. diff --git a/mali_kbase/tests/kutf/Makefile b/mali_kbase/tests/kutf/Makefile new file mode 100644 index 0000000..010c92c --- /dev/null +++ b/mali_kbase/tests/kutf/Makefile @@ -0,0 +1,29 @@ +# +# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/mali_kbase/tests/kutf/kutf_mem.c b/mali_kbase/tests/kutf/kutf_mem.c new file mode 100644 index 0000000..5408e57 --- /dev/null +++ b/mali_kbase/tests/kutf/kutf_mem.c @@ -0,0 +1,94 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF memory management functions */ + +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/module.h> + +#include <kutf/kutf_mem.h> + + +/** + * struct kutf_alloc_entry - Structure representing an allocation. + * @node: List node for use with kutf_mempool. + * @data: Data area of the allocation + */ +struct kutf_alloc_entry { + struct list_head node; + u8 data[0]; +}; + +int kutf_mempool_init(struct kutf_mempool *pool) +{ + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return -1; + } + + INIT_LIST_HEAD(&pool->head); + + return 0; +} +EXPORT_SYMBOL(kutf_mempool_init); + +void kutf_mempool_destroy(struct kutf_mempool *pool) +{ + struct list_head *remove; + struct list_head *tmp; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return; + } + + list_for_each_safe(remove, tmp, &pool->head) { + struct kutf_alloc_entry *remove_alloc; + + remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); + list_del(&remove_alloc->node); + kfree(remove_alloc); + } +} +EXPORT_SYMBOL(kutf_mempool_destroy); + +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) +{ + struct kutf_alloc_entry *ret; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + goto fail_pool; + } + + ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); + if (!ret) { + pr_err("Failed to allocate memory\n"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&ret->node); + list_add(&ret->node, &pool->head); + + return &ret->data[0]; + +fail_alloc: +fail_pool: + return NULL; +} +EXPORT_SYMBOL(kutf_mempool_alloc); diff --git a/mali_kbase/tests/kutf/kutf_resultset.c b/mali_kbase/tests/kutf/kutf_resultset.c new file mode 100644 index 0000000..5bd0496 --- /dev/null +++ b/mali_kbase/tests/kutf/kutf_resultset.c @@ -0,0 +1,95 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF result management functions */ + +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/printk.h> + +#include <kutf/kutf_resultset.h> + +/** + * struct kutf_result_set - Represents a set of results. + * @results: Pointer to the linked list where the results are stored. + */ +struct kutf_result_set { + struct list_head results; +}; + +struct kutf_result_set *kutf_create_result_set(void) +{ + struct kutf_result_set *set; + + set = kmalloc(sizeof(*set), GFP_KERNEL); + if (!set) { + pr_err("Failed to allocate resultset"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&set->results); + + return set; + +fail_alloc: + return NULL; +} + +void kutf_add_result(struct kutf_mempool *mempool, + struct kutf_result_set *set, + enum kutf_result_status status, + const char *message) +{ + /* Create the new result */ + struct kutf_result *new_result; + + BUG_ON(set == NULL); + + new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); + if (!new_result) { + pr_err("Result allocation failed\n"); + return; + } + + INIT_LIST_HEAD(&new_result->node); + new_result->status = status; + new_result->message = message; + + list_add_tail(&new_result->node, &set->results); +} + +void kutf_destroy_result_set(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) + pr_err("kutf_destroy_result_set: Unread results from test\n"); + + kfree(set); +} + +struct kutf_result *kutf_remove_result(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) { + struct kutf_result *ret; + + ret = list_first_entry(&set->results, struct kutf_result, node); + list_del(&ret->node); + return ret; + } + + return NULL; +} + diff --git a/mali_kbase/tests/kutf/kutf_suite.c b/mali_kbase/tests/kutf/kutf_suite.c new file mode 100644 index 0000000..a7cfd3b --- /dev/null +++ b/mali_kbase/tests/kutf/kutf_suite.c @@ -0,0 +1,1041 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF suite, test and fixture management including user to kernel + * interaction */ + +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/debugfs.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/uaccess.h> +#include <linux/fs.h> +#include <linux/version.h> + +#include <generated/autoconf.h> + +#include <kutf/kutf_suite.h> +#include <kutf/kutf_resultset.h> +#include <kutf/kutf_utils.h> + +#if defined(CONFIG_DEBUG_FS) + +/** + * struct kutf_application - Structure which represents kutf application + * @name: The name of this test application. + * @dir: The debugfs directory for this test + * @suite_list: List head to store all the suites which are part of this + * application + */ +struct kutf_application { + const char *name; + struct dentry *dir; + struct list_head suite_list; +}; + +/** + * struct kutf_test_function - Structure which represents kutf test function + * @suite: Back reference to the suite this test function + * belongs to + * @filters: Filters that apply to this test function + * @test_id: Test ID + * @execute: Function to run for this test + * @test_data: Static data for this test + * @node: List node for test_list + * @variant_list: List head to store all the variants which can run on + * this function + * @dir: debugfs directory for this test function + */ +struct kutf_test_function { + struct kutf_suite *suite; + unsigned int filters; + unsigned int test_id; + void (*execute)(struct kutf_context *context); + union kutf_callback_data test_data; + struct list_head node; + struct list_head variant_list; + struct dentry *dir; +}; + +/** + * struct kutf_test_fixture - Structure which holds information on the kutf + * test fixture + * @test_func: Test function this fixture belongs to + * @fixture_index: Index of this fixture + * @node: List node for variant_list + * @dir: debugfs directory for this test fixture + */ +struct kutf_test_fixture { + struct kutf_test_function *test_func; + unsigned int fixture_index; + struct list_head node; + struct dentry *dir; +}; + +struct dentry *base_dir; + +/** + * struct kutf_convert_table - Structure which keeps test results + * @result_name: Status of the test result + * @result: Status value for a single test + */ +struct kutf_convert_table { + char result_name[50]; + enum kutf_result_status result; +}; + +struct kutf_convert_table kutf_convert[] = { +#define ADD_UTF_RESULT(_name) \ +{ \ + #_name, \ + _name, \ +}, +ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) +ADD_UTF_RESULT(KUTF_RESULT_SKIP) +ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) +ADD_UTF_RESULT(KUTF_RESULT_PASS) +ADD_UTF_RESULT(KUTF_RESULT_DEBUG) +ADD_UTF_RESULT(KUTF_RESULT_INFO) +ADD_UTF_RESULT(KUTF_RESULT_WARN) +ADD_UTF_RESULT(KUTF_RESULT_FAIL) +ADD_UTF_RESULT(KUTF_RESULT_FATAL) +ADD_UTF_RESULT(KUTF_RESULT_ABORT) +}; + +#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) + +/** + * kutf_create_context() - Create a test context in which a specific fixture + * of an application will be run and its results + * reported back to the user + * @test_fix: Test fixture to be run. + * + * Return: Returns the created test context on success or NULL on failure + */ +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix); + +/** + * kutf_destroy_context() - Destroy a previously created test context + * @context: Test context to destroy + */ +static void kutf_destroy_context(struct kutf_context *context); + +/** + * kutf_set_result() - Set the test result against the specified test context + * @context: Test context + * @status: Result status + */ +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status); + +/** + * kutf_set_expected_result() - Set the expected test result for the specified + * test context + * @context: Test context + * @expected_status: Expected result status + */ +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) +/* Pre 3.4.0 kernels don't have the simple_open helper */ + +/** + * simple_open() - Helper for file opening which stores the inode private data + * into the file private data + * @inode: File entry representation + * @file: A specific opening of the file + * + * Return: always 0; if inode private data do not exist, the file will not + * be assigned private data + */ +static int simple_open(struct inode *inode, struct file *file) +{ + if (inode->i_private) + file->private_data = inode->i_private; + return 0; +} +#endif + +/** + * kutf_result_to_string() - Converts a KUTF result into a string + * @result_str: Output result string + * @result: Result status to convert + * + * Return: 1 if test result was successfully converted to string, 0 otherwise + */ +static int kutf_result_to_string(char **result_str, + enum kutf_result_status result) +{ + int i; + int ret = 0; + + for (i = 0; i < UTF_CONVERT_SIZE; i++) { + if (result == kutf_convert[i].result) { + *result_str = kutf_convert[i].result_name; + ret = 1; + } + } + return ret; +} + +/** + * kutf_debugfs_const_string_read() - Simple debugfs read callback which + * returns a constant string + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * Return: On success, the number of bytes read and offset @ppos advanced by + * this number; on error, negative value + */ +static ssize_t kutf_debugfs_const_string_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + char *str = file->private_data; + + return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); +} + +static const struct file_operations kutf_debugfs_const_string_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = kutf_debugfs_const_string_read, + .llseek = default_llseek, +}; + +/** + * kutf_add_explicit_result() - Check if an explicit result needs to be added + * @context: KUTF test context + */ +static void kutf_add_explicit_result(struct kutf_context *context) +{ + switch (context->expected_status) { + case KUTF_RESULT_UNKNOWN: + if (context->status == KUTF_RESULT_UNKNOWN) + kutf_test_pass(context, "(implicit pass)"); + break; + + case KUTF_RESULT_WARN: + if (context->status == KUTF_RESULT_WARN) + kutf_test_pass(context, + "Pass (expected warn occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected warn missing)"); + break; + + case KUTF_RESULT_FAIL: + if (context->status == KUTF_RESULT_FAIL) + kutf_test_pass(context, + "Pass (expected fail occurred)"); + else if (context->status != KUTF_RESULT_SKIP) { + /* Force the expected status so the fail gets logged */ + context->expected_status = KUTF_RESULT_PASS; + kutf_test_fail(context, + "Fail (expected fail missing)"); + } + break; + + case KUTF_RESULT_FATAL: + if (context->status == KUTF_RESULT_FATAL) + kutf_test_pass(context, + "Pass (expected fatal occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected fatal missing)"); + break; + + case KUTF_RESULT_ABORT: + if (context->status == KUTF_RESULT_ABORT) + kutf_test_pass(context, + "Pass (expected abort occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected abort missing)"); + break; + default: + break; + } +} + +/** + * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. + * @inode: inode of the opened file + * @file: Opened file to read from + * + * This function retrieves the test fixture data that is associated with the + * opened file and works back to get the test, suite and application so + * it can then run the test that is associated with the file entry. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_open(struct inode *inode, struct file *file) +{ + struct kutf_test_fixture *test_fix = inode->i_private; + struct kutf_test_function *test_func = test_fix->test_func; + struct kutf_suite *suite = test_func->suite; + struct kutf_context *test_context; + + test_context = kutf_create_context(test_fix); + if (!test_context) + return -ENODEV; + + file->private_data = test_context; + + /* + * Call the create fixture function if required before the + * fixture is run + */ + if (suite->create_fixture) + test_context->fixture = suite->create_fixture(test_context); + + /* Only run the test if the fixture was created (if required) */ + if ((suite->create_fixture && test_context->fixture) || + (!suite->create_fixture)) { + /* Run this fixture */ + test_func->execute(test_context); + + if (suite->remove_fixture) + suite->remove_fixture(test_context); + + kutf_add_explicit_result(test_context); + } + return 0; +} + +/** + * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * This function emits the results which where logged during the opening of + * the file kutf_debugfs_run_open. + * Results will be emitted one at a time, once all the results have been read + * 0 will be returned to indicate there is no more data. + * + * Return: Number of bytes read. + */ +static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_result *res; + unsigned long bytes_not_copied; + ssize_t bytes_copied = 0; + + /* Note: This code assumes a result is read completely */ + res = kutf_remove_result(test_context->result_set); + if (res) { + char *kutf_str_ptr = NULL; + unsigned int kutf_str_len = 0; + unsigned int message_len = 0; + char separator = ':'; + char terminator = '\n'; + + kutf_result_to_string(&kutf_str_ptr, res->status); + if (kutf_str_ptr) + kutf_str_len = strlen(kutf_str_ptr); + + if (res->message) + message_len = strlen(res->message); + + if ((kutf_str_len + 1 + message_len + 1) > len) { + pr_err("Not enough space in user buffer for a single result"); + return 0; + } + + /* First copy the result string */ + if (kutf_str_ptr) { + bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, + kutf_str_len); + bytes_copied += kutf_str_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Then the separator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &separator, 1); + bytes_copied += 1 - bytes_not_copied; + if (bytes_not_copied) + goto exit; + + /* Finally Next copy the result string */ + if (res->message) { + bytes_not_copied = copy_to_user(&buf[bytes_copied], + res->message, message_len); + bytes_copied += message_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Finally the terminator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &terminator, 1); + bytes_copied += 1 - bytes_not_copied; + } +exit: + return bytes_copied; +} + +/** + * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. + * @inode: File entry representation + * @file: A specific opening of the file + * + * Release any resources that where created during the opening of the file + * + * Return: 0 on success + */ +static int kutf_debugfs_run_release(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = file->private_data; + + kutf_destroy_context(test_context); + return 0; +} + +static const struct file_operations kutf_debugfs_run_ops = { + .owner = THIS_MODULE, + .open = kutf_debugfs_run_open, + .read = kutf_debugfs_run_read, + .release = kutf_debugfs_run_release, + .llseek = default_llseek, +}; + +/** + * create_fixture_variant() - Creates a fixture variant for the specified + * test function and index and the debugfs entries + * that represent it. + * @test_func: Test function + * @fixture_index: Fixture index + * + * Return: 0 on success, negative value corresponding to error code in failure + */ +static int create_fixture_variant(struct kutf_test_function *test_func, + unsigned int fixture_index) +{ + struct kutf_test_fixture *test_fix; + char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ + struct dentry *tmp; + int err; + + test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); + if (!test_fix) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + err = -ENOMEM; + goto fail_alloc; + } + + test_fix->test_func = test_func; + test_fix->fixture_index = fixture_index; + + snprintf(name, sizeof(name), "%d", fixture_index); + test_fix->dir = debugfs_create_dir(name, test_func->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, + &kutf_debugfs_run_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + list_add(&test_fix->node, &test_func->variant_list); + return 0; + +fail_file: + debugfs_remove_recursive(test_fix->dir); +fail_dir: + kfree(test_fix); +fail_alloc: + return err; +} + +/** + * kutf_remove_test_variant() - Destroy a previously created fixture variant. + * @test_fix: Test fixture + */ +static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) +{ + debugfs_remove_recursive(test_fix->dir); + kfree(test_fix); +} + +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data) +{ + struct kutf_test_function *test_func; + struct dentry *tmp; + unsigned int i; + + test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); + if (!test_func) { + pr_err("Failed to allocate memory when adding test %s\n", name); + goto fail_alloc; + } + + INIT_LIST_HEAD(&test_func->variant_list); + + test_func->dir = debugfs_create_dir(name, suite->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + test_func->filters = filters; + tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, + &test_func->filters); + if (!tmp) { + pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); + goto fail_file; + } + + test_func->test_id = id; + tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, + &test_func->test_id); + if (!tmp) { + pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); + goto fail_file; + } + + for (i = 0; i < suite->fixture_variants; i++) { + if (create_fixture_variant(test_func, i)) { + pr_err("Failed to create fixture %d when adding test %s\n", i, name); + goto fail_file; + } + } + + test_func->suite = suite; + test_func->execute = execute; + test_func->test_data = test_data; + + list_add(&test_func->node, &suite->test_list); + return; + +fail_file: + debugfs_remove_recursive(test_func->dir); +fail_dir: + kfree(test_func); +fail_alloc: + return; +} +EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); + +void kutf_add_test_with_filters( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test_with_filters); + +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test); + +/** + * kutf_remove_test(): Remove a previously added test function. + * @test_func: Test function + */ +static void kutf_remove_test(struct kutf_test_function *test_func) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &test_func->variant_list) { + struct kutf_test_fixture *test_fix; + + test_fix = list_entry(pos, struct kutf_test_fixture, node); + kutf_remove_test_variant(test_fix); + } + + list_del(&test_func->node); + debugfs_remove_recursive(test_func->dir); + kfree(test_func); +} + +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data) +{ + struct kutf_suite *suite; + struct dentry *tmp; + + suite = kmalloc(sizeof(*suite), GFP_KERNEL); + if (!suite) { + pr_err("Failed to allocate memory when creating suite %s\n", name); + goto fail_kmalloc; + } + + suite->dir = debugfs_create_dir(name, app->dir); + if (!suite->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&suite->test_list); + suite->app = app; + suite->name = name; + suite->fixture_variants = fixture_count; + suite->create_fixture = create_fixture; + suite->remove_fixture = remove_fixture; + suite->suite_default_flags = filters; + suite->suite_data = suite_data; + + list_add(&suite->node, &app->suite_list); + + return suite; + +fail_file: + debugfs_remove_recursive(suite->dir); +fail_debugfs: + kfree(suite); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); + +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + filters, + data); +} +EXPORT_SYMBOL(kutf_create_suite_with_filters); + +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + KUTF_F_TEST_GENERIC, + data); +} +EXPORT_SYMBOL(kutf_create_suite); + +/** + * kutf_destroy_suite() - Destroy a previously added test suite. + * @suite: Test suite + */ +static void kutf_destroy_suite(struct kutf_suite *suite) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &suite->test_list) { + struct kutf_test_function *test_func; + + test_func = list_entry(pos, struct kutf_test_function, node); + kutf_remove_test(test_func); + } + + list_del(&suite->node); + debugfs_remove_recursive(suite->dir); + kfree(suite); +} + +struct kutf_application *kutf_create_application(const char *name) +{ + struct kutf_application *app; + struct dentry *tmp; + + app = kmalloc(sizeof(*app), GFP_KERNEL); + if (!app) { + pr_err("Failed to create allocate memory when creating application %s\n", name); + goto fail_kmalloc; + } + + app->dir = debugfs_create_dir(name, base_dir); + if (!app->dir) { + pr_err("Failed to create debugfs direcotry when creating application %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&app->suite_list); + app->name = name; + + return app; + +fail_file: + debugfs_remove_recursive(app->dir); +fail_debugfs: + kfree(app); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_application); + +void kutf_destroy_application(struct kutf_application *app) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &app->suite_list) { + struct kutf_suite *suite; + + suite = list_entry(pos, struct kutf_suite, node); + kutf_destroy_suite(suite); + } + + debugfs_remove_recursive(app->dir); + kfree(app); +} +EXPORT_SYMBOL(kutf_destroy_application); + +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix) +{ + struct kutf_context *new_context; + + new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); + if (!new_context) { + pr_err("Failed to allocate test context"); + goto fail_alloc; + } + + new_context->result_set = kutf_create_result_set(); + if (!new_context->result_set) { + pr_err("Failed to create resultset"); + goto fail_result_set; + } + + new_context->test_fix = test_fix; + /* Save the pointer to the suite as the callbacks will require it */ + new_context->suite = test_fix->test_func->suite; + new_context->status = KUTF_RESULT_UNKNOWN; + new_context->expected_status = KUTF_RESULT_UNKNOWN; + + kutf_mempool_init(&new_context->fixture_pool); + new_context->fixture = NULL; + new_context->fixture_index = test_fix->fixture_index; + new_context->fixture_name = NULL; + new_context->test_data = test_fix->test_func->test_data; + + return new_context; + +fail_result_set: + kfree(new_context); +fail_alloc: + return NULL; +} + +static void kutf_destroy_context(struct kutf_context *context) +{ + kutf_destroy_result_set(context->result_set); + kutf_mempool_destroy(&context->fixture_pool); + kfree(context); +} + +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status) +{ + context->status = status; +} + +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status) +{ + context->expected_status = expected_status; +} + +/** + * kutf_test_log_result() - Log a result for the specified test context + * @context: Test context + * @message: Result string + * @new_status: Result status + */ +static void kutf_test_log_result( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + if (context->status < new_status) + context->status = new_status; + + if (context->expected_status != new_status) + kutf_add_result(&context->fixture_pool, context->result_set, + new_status, message); +} + +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + kutf_test_log_result(context, message, new_status); +} +EXPORT_SYMBOL(kutf_test_log_result_external); + +void kutf_test_expect_abort(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_expect_abort); + +void kutf_test_expect_fatal(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_expect_fatal); + +void kutf_test_expect_fail(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_expect_fail); + +void kutf_test_expect_warn(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_expect_warn); + +void kutf_test_expect_pass(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_expect_pass); + +void kutf_test_skip(struct kutf_context *context) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip); + +void kutf_test_skip_msg(struct kutf_context *context, const char *message) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, + "Test skipped: %s", message), KUTF_RESULT_SKIP); + kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip_msg); + +void kutf_test_debug(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); +} +EXPORT_SYMBOL(kutf_test_debug); + +void kutf_test_pass(struct kutf_context *context, char const *message) +{ + static const char explicit_message[] = "(explicit pass)"; + + if (!message) + message = explicit_message; + + kutf_test_log_result(context, message, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_pass); + +void kutf_test_info(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_INFO); +} +EXPORT_SYMBOL(kutf_test_info); + +void kutf_test_warn(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_warn); + +void kutf_test_fail(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_fail); + +void kutf_test_fatal(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_fatal); + +void kutf_test_abort(struct kutf_context *context) +{ + kutf_test_log_result(context, "", KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_abort); + +/** + * init_kutf_core() - Module entry point. + * + * Create the base entry point in debugfs. + */ +static int __init init_kutf_core(void) +{ + int ret; + + base_dir = debugfs_create_dir("kutf_tests", NULL); + if (!base_dir) { + ret = -ENODEV; + goto exit_dir; + } + + return 0; + +exit_dir: + return ret; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Remove the base entry point in debugfs. + */ +static void __exit exit_kutf_core(void) +{ + debugfs_remove_recursive(base_dir); +} + +#else /* defined(CONFIG_DEBUG_FS) */ + +/** + * init_kutf_core() - Module entry point. + * + * Stub for when build against a kernel without debugfs support + */ +static int __init init_kutf_core(void) +{ + pr_debug("KUTF requires a kernel with debug fs support"); + + return -ENODEV; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Stub for when build against a kernel without debugfs support + */ +static void __exit exit_kutf_core(void) +{ +} +#endif /* defined(CONFIG_DEBUG_FS) */ + +MODULE_LICENSE("GPL"); + +module_init(init_kutf_core); +module_exit(exit_kutf_core); diff --git a/mali_kbase/tests/kutf/kutf_utils.c b/mali_kbase/tests/kutf/kutf_utils.c new file mode 100644 index 0000000..a429a2d --- /dev/null +++ b/mali_kbase/tests/kutf/kutf_utils.c @@ -0,0 +1,71 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF utility functions */ + +#include <linux/mutex.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/printk.h> + +#include <kutf/kutf_utils.h> +#include <kutf/kutf_mem.h> + +static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; + +DEFINE_MUTEX(buffer_lock); + +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...) +{ + va_list args; + int len; + int size; + void *buffer; + + mutex_lock(&buffer_lock); + va_start(args, fmt); + len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); + va_end(args); + + if (len < 0) { + pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); + goto fail_format; + } + + if (len >= sizeof(tmp_buffer)) { + pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); + size = sizeof(tmp_buffer); + } else { + size = len + 1; + } + + buffer = kutf_mempool_alloc(pool, size); + if (!buffer) + goto fail_alloc; + + memcpy(buffer, tmp_buffer, size); + mutex_unlock(&buffer_lock); + + return buffer; + +fail_alloc: +fail_format: + mutex_unlock(&buffer_lock); + return NULL; +} +EXPORT_SYMBOL(kutf_dsprintf); diff --git a/mali_kbase/tests/kutf/sconscript b/mali_kbase/tests/kutf/sconscript new file mode 100644 index 0000000..d7f1124 --- /dev/null +++ b/mali_kbase/tests/kutf/sconscript @@ -0,0 +1,21 @@ +# +# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import('kutf_env') + +make_args = kutf_env.kernel_get_config_defines(ret_list = True) + +mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) +kutf_env.KernelObjTarget('kutf', mod) diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kbuild b/mali_kbase/tests/mali_kutf_irq_test/Kbuild new file mode 100644 index 0000000..0cd9ceb --- /dev/null +++ b/mali_kbase/tests/mali_kutf_irq_test/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android + +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o + +mali_kutf_irq_test-y := mali_kutf_irq_test_main.o diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kconfig b/mali_kbase/tests/mali_kutf_irq_test/Kconfig new file mode 100644 index 0000000..f4553d3 --- /dev/null +++ b/mali_kbase/tests/mali_kutf_irq_test/Kconfig @@ -0,0 +1,23 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +config MALI_IRQ_LATENCY + tristate "Mali GPU IRQ latency measurement" + depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF + default m + help + This option will build a test module mali_kutf_irq_test that + can determine the latency of the Mali GPU IRQ on your system. + Choosing M here will generate a single module called mali_kutf_irq_test. diff --git a/mali_kbase/tests/mali_kutf_irq_test/Makefile b/mali_kbase/tests/mali_kutf_irq_test/Makefile new file mode 100644 index 0000000..4e94876 --- /dev/null +++ b/mali_kbase/tests/mali_kutf_irq_test/Makefile @@ -0,0 +1,51 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +TEST_CCFLAGS := \ + -DMALI_DEBUG=$(MALI_DEBUG) \ + -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ + -DMALI_MODEL=$(MALI_MODEL) \ + -DMALI_NO_MALI=$(MALI_NO_MALI) \ + -DMALI_BASE_QA_LEAK=$(MALI_BASE_QA_LEAK) \ + -DMALI_BASE_QA_RESFAIL=$(MALI_BASE_QA_RESFAIL) \ + -DMALI_BASE_QA_USE_AFTER_FREE=$(MALI_BASE_QA_USE_AFTER_FREE) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_USE_UMP=$(MALI_USE_UMP) \ + -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + $(SCONS_CFLAGS) \ + -I$(CURDIR)/../include \ + -I$(CURDIR)/../../../../../../include \ + -I$(CURDIR)/../../../ \ + -I$(CURDIR)/../../ \ + -I$(CURDIR)/../../backend/gpu \ + -I$(CURDIR)/ \ + -I$(srctree)/drivers/staging/android \ + -I$(srctree)/include/linux + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c new file mode 100644 index 0000000..ffd802f --- /dev/null +++ b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -0,0 +1,257 @@ +/* + * + * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/interrupt.h> + +#include "mali_kbase.h" +#include <midgard/backend/gpu/mali_kbase_device_internal.h> + +#include <kutf/kutf_suite.h> +#include <kutf/kutf_utils.h> + +/* + * This file contains the code which is used for measuring interrupt latency + * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is + * used with this purpose and it is called within KUTF framework - a kernel + * unit test framework. The measured latency provided by this test should + * be representative for the latency of the Mali JOB/MMU IRQs as well. + */ + +/* KUTF test application pointer for this test */ +struct kutf_application *irq_app; + +/** + * struct kutf_irq_fixture data - test fixture used by the test functions. + * @kbdev: kbase device for the GPU. + * + */ +struct kutf_irq_fixture_data { + struct kbase_device *kbdev; +}; + +#define SEC_TO_NANO(s) ((s)*1000000000LL) + +/* ID for the GPU IRQ */ +#define GPU_IRQ_HANDLER 2 + +#define NR_TEST_IRQS 1000000 + +/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not + * expect to see this in normal use (e.g., when Android is running). */ +#define TEST_IRQ MULTIPLE_GPU_FAULTS + +#define IRQ_TIMEOUT HZ + +/* Kernel API for setting irq throttle hook callback and irq time in us*/ +extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type); +extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); + +static DECLARE_WAIT_QUEUE_HEAD(wait); +static bool triggered; +static u64 irq_time; + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +/** + * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler + * @irq: IRQ number + * @data: Data associated with this IRQ + * + * Return: state of the IRQ + */ +static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) +{ + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); + if (val & TEST_IRQ) { + struct timespec tval; + + getnstimeofday(&tval); + irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, + NULL); + + triggered = true; + wake_up(&wait); + + return IRQ_HANDLED; + } + + /* Trigger main irq handler */ + return kbase_gpu_irq_handler(irq, data); +} + +/** + * mali_kutf_irq_default_create_fixture() - Creates the fixture data required + * for all the tests in the irq suite. + * @context: KUTF context. + * + * Return: Fixture data created on success or NULL on failure + */ +static void *mali_kutf_irq_default_create_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data; + + data = kutf_mempool_alloc(&context->fixture_pool, + sizeof(struct kutf_irq_fixture_data)); + + if (!data) + goto fail; + + /* Acquire the kbase device */ + data->kbdev = kbase_find_device(-1); + if (data->kbdev == NULL) { + kutf_test_fail(context, "Failed to find kbase device"); + goto fail; + } + + return data; + +fail: + return NULL; +} + +/** + * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously + * created by mali_kutf_irq_default_create_fixture. + * + * @context: KUTF context. + */ +static void mali_kutf_irq_default_remove_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + + kbase_release_device(kbdev); +} + +/** + * mali_kutf_irq_latency() - measure GPU IRQ latency + * @context: kutf context within which to perform the test + * + * The test triggers IRQs manually, and measures the + * time between triggering the IRQ and the IRQ handler being executed. + * + * This is not a traditional test, in that the pass/fail status has little + * meaning (other than indicating that the IRQ handler executed at all). Instead + * the results are in the latencies provided with the test result. There is no + * meaningful pass/fail result that can be obtained here, instead the latencies + * are provided for manual analysis only. + */ +static void mali_kutf_irq_latency(struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + u64 min_time = U64_MAX, max_time = 0, average_time = 0; + int i; + bool test_failed = false; + + /* Force GPU to be powered */ + kbase_pm_context_active(kbdev); + + kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, + GPU_IRQ_HANDLER); + + for (i = 0; i < NR_TEST_IRQS; i++) { + struct timespec tval; + u64 start_time; + int ret; + + triggered = false; + getnstimeofday(&tval); + start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + /* Trigger fake IRQ */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + TEST_IRQ, NULL); + + ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); + + if (ret == 0) { + kutf_test_fail(context, "Timed out waiting for IRQ\n"); + test_failed = true; + break; + } + + if ((irq_time - start_time) < min_time) + min_time = irq_time - start_time; + if ((irq_time - start_time) > max_time) + max_time = irq_time - start_time; + average_time += irq_time - start_time; + + udelay(10); + } + + /* Go back to default handler */ + kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); + + kbase_pm_context_idle(kbdev); + + if (!test_failed) { + const char *results; + + do_div(average_time, NR_TEST_IRQS); + results = kutf_dsprintf(&context->fixture_pool, + "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", + min_time, max_time, average_time); + kutf_test_pass(context, results); + } +} + +/** + * Module entry point for this test. + */ +int mali_kutf_irq_test_main_init(void) +{ + struct kutf_suite *suite; + + irq_app = kutf_create_application("irq"); + suite = kutf_create_suite(irq_app, "irq_default", + 1, mali_kutf_irq_default_create_fixture, + mali_kutf_irq_default_remove_fixture); + + kutf_add_test(suite, 0x0, "irq_latency", + mali_kutf_irq_latency); + return 0; +} + +/** + * Module exit point for this test. + */ +void mali_kutf_irq_test_main_exit(void) +{ + kutf_destroy_application(irq_app); +} + +module_init(mali_kutf_irq_test_main_init); +module_exit(mali_kutf_irq_test_main_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION("1.0"); diff --git a/mali_kbase/tests/mali_kutf_irq_test/sconscript b/mali_kbase/tests/mali_kutf_irq_test/sconscript new file mode 100644 index 0000000..ec837f1 --- /dev/null +++ b/mali_kbase/tests/mali_kutf_irq_test/sconscript @@ -0,0 +1,30 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +import os +Import('env') + +src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] + +if env.GetOption('clean') : + env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) + env.KernelObjTarget('mali_kutf_irq_test', cmd) +else: + makeAction=Action("cd ${SOURCE.dir} && make MALI_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_MODEL=${mali_model} MALI_NO_MALI=${no_mali} MALI_HW_VERSION=${hwver} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % (env.base_get_qa_settings(), env.kernel_get_config_defines()), '$MAKECOMSTR') + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') + env.KernelObjTarget('mali_kutf_irq_test', cmd) diff --git a/mali_kbase/tests/sconscript b/mali_kbase/tests/sconscript new file mode 100644 index 0000000..5337e10 --- /dev/null +++ b/mali_kbase/tests/sconscript @@ -0,0 +1,37 @@ +# +# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import ('env') + +kutf_env = env.Clone() +kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') +Export('kutf_env') + +if Glob('internal/sconscript'): + SConscript('internal/sconscript') + +if kutf_env['debug'] == '1': + SConscript('kutf/sconscript') + SConscript('mali_kutf_irq_test/sconscript') + + if Glob('kutf_test/sconscript'): + SConscript('kutf_test/sconscript') + + if Glob('kutf_test_runner/sconscript'): + SConscript('kutf_test_runner/sconscript') + +if env['unit'] == '1': + SConscript('mali_kutf_ipa_test/sconscript') + SConscript('mali_kutf_vinstr_test/sconscript') |