summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSidath Senanayake <sidaths@google.com>2017-05-18 14:43:17 +0200
committerSidath Senanayake <sidaths@google.com>2017-05-18 14:43:17 +0200
commit6f5ab3baed824941f168ab133469f997d4450146 (patch)
tree95dcbadaa979f84a8d75c0919af1b85c5afd1924
parent48f3554a4abf9ce182253fb5415a1a26b0790998 (diff)
downloadgpu-6f5ab3baed824941f168ab133469f997d4450146.tar.gz
Mali Bifrost DDK r6p0 KMD
Provenance: b67d8663a (collaborate/EAC/b_r6p0) BX304L01B-BU-00000-r6p0-01rel0 BX304L06A-BU-00000-r6p0-01rel0 BX304X07X-BU-00000-r6p0-01rel0 Signed-off-by: Sidath Senanayake <sidaths@google.com> Change-Id: I6b19113374d523be9a75a252d672731600e991d6
-rw-r--r--mali_kbase/Kbuild37
-rw-r--r--mali_kbase/Kconfig3
-rw-r--r--mali_kbase/Makefile2
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_devfreq.c3
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_as.c179
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_hw.c18
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_rb.c18
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c72
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_driver.c52
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_internal.h6
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_metrics.c4
-rw-r--r--mali_kbase/ipa/Kbuild4
-rw-r--r--mali_kbase/ipa/mali_kbase_ipa.c423
-rw-r--r--mali_kbase/ipa/mali_kbase_ipa.h135
-rw-r--r--mali_kbase/ipa/mali_kbase_ipa_debugfs.c26
-rw-r--r--mali_kbase/ipa/mali_kbase_ipa_debugfs.h1
-rw-r--r--mali_kbase/ipa/mali_kbase_ipa_simple.c52
-rw-r--r--mali_kbase/mali_base_hwconfig_features.h61
-rw-r--r--mali_kbase/mali_base_hwconfig_issues.h54
-rw-r--r--mali_kbase/mali_base_kernel.h134
-rw-r--r--mali_kbase/mali_kbase.h29
-rw-r--r--mali_kbase/mali_kbase_10969_workaround.c2
-rw-r--r--mali_kbase/mali_kbase_config_defaults.h4
-rw-r--r--mali_kbase/mali_kbase_context.c12
-rw-r--r--mali_kbase/mali_kbase_core_linux.c977
-rw-r--r--mali_kbase/mali_kbase_ctx_sched.c203
-rw-r--r--mali_kbase/mali_kbase_ctx_sched.h131
-rw-r--r--mali_kbase/mali_kbase_debug_mem_view.c2
-rw-r--r--mali_kbase/mali_kbase_defs.h108
-rw-r--r--mali_kbase/mali_kbase_device.c31
-rw-r--r--mali_kbase/mali_kbase_dma_fence.c257
-rw-r--r--mali_kbase/mali_kbase_dma_fence.h23
-rw-r--r--mali_kbase/mali_kbase_fence.c196
-rw-r--r--mali_kbase/mali_kbase_fence.h266
-rw-r--r--mali_kbase/mali_kbase_fence_defs.h51
-rw-r--r--mali_kbase/mali_kbase_gator_hwcnt_names.h4
-rw-r--r--mali_kbase/mali_kbase_gpu_id.h6
-rw-r--r--mali_kbase/mali_kbase_gpuprops.c189
-rw-r--r--mali_kbase/mali_kbase_gpuprops.h22
-rw-r--r--mali_kbase/mali_kbase_gpuprops_types.h3
-rw-r--r--mali_kbase/mali_kbase_hw.c235
-rw-r--r--mali_kbase/mali_kbase_hw.h17
-rw-r--r--mali_kbase/mali_kbase_hwaccess_jm.h27
-rw-r--r--mali_kbase/mali_kbase_ioctl.h656
-rw-r--r--mali_kbase/mali_kbase_jd.c70
-rw-r--r--mali_kbase/mali_kbase_jd_debugfs.c63
-rw-r--r--mali_kbase/mali_kbase_js.c237
-rw-r--r--mali_kbase/mali_kbase_js.h77
-rw-r--r--mali_kbase/mali_kbase_js_defs.h33
-rw-r--r--mali_kbase/mali_kbase_mem.c112
-rw-r--r--mali_kbase/mali_kbase_mem.h31
-rw-r--r--mali_kbase/mali_kbase_mem_linux.c179
-rw-r--r--mali_kbase/mali_kbase_mem_linux.h15
-rw-r--r--mali_kbase/mali_kbase_mem_lowlevel.h2
-rw-r--r--mali_kbase/mali_kbase_mem_pool.c2
-rw-r--r--mali_kbase/mali_kbase_mem_profile_debugfs.c4
-rw-r--r--mali_kbase/mali_kbase_mmu.c75
-rw-r--r--mali_kbase/mali_kbase_mmu_mode.h2
-rw-r--r--mali_kbase/mali_kbase_mmu_mode_aarch64.c2
-rw-r--r--mali_kbase/mali_kbase_mmu_mode_lpae.c10
-rw-r--r--mali_kbase/mali_kbase_softjobs.c260
-rw-r--r--mali_kbase/mali_kbase_sync.c192
-rw-r--r--mali_kbase/mali_kbase_sync.h189
-rw-r--r--mali_kbase/mali_kbase_sync_android.c537
-rw-r--r--mali_kbase/mali_kbase_sync_common.c43
-rw-r--r--mali_kbase/mali_kbase_sync_file.c339
-rw-r--r--mali_kbase/mali_kbase_sync_user.c156
-rw-r--r--mali_kbase/mali_kbase_tlstream.c111
-rw-r--r--mali_kbase/mali_kbase_tlstream.h26
-rw-r--r--mali_kbase/mali_kbase_uku.h2
-rw-r--r--mali_kbase/mali_kbase_vinstr.c35
-rw-r--r--mali_kbase/platform/devicetree/mali_kbase_config_platform.h9
-rw-r--r--mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c150
-rw-r--r--mali_kbase/platform/juno_soc/mali_kbase_config_platform.h84
-rw-r--r--mali_kbase/platform/vexpress/mali_kbase_config_platform.h9
-rw-r--r--mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h9
-rw-r--r--mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h9
-rw-r--r--mali_kbase/protected_mode_switcher.h64
-rw-r--r--mali_kbase/sconscript22
-rw-r--r--mali_kbase/tests/Kbuild (renamed from mali_kbase/platform/juno_soc/Kbuild)5
-rw-r--r--mali_kbase/tests/Kconfig17
-rw-r--r--mali_kbase/tests/include/kutf/kutf_mem.h65
-rw-r--r--mali_kbase/tests/include/kutf/kutf_resultset.h121
-rw-r--r--mali_kbase/tests/include/kutf/kutf_suite.h508
-rw-r--r--mali_kbase/tests/include/kutf/kutf_utils.h55
-rw-r--r--mali_kbase/tests/kutf/Kbuild20
-rw-r--r--mali_kbase/tests/kutf/Kconfig22
-rw-r--r--mali_kbase/tests/kutf/Makefile29
-rw-r--r--mali_kbase/tests/kutf/kutf_mem.c94
-rw-r--r--mali_kbase/tests/kutf/kutf_resultset.c95
-rw-r--r--mali_kbase/tests/kutf/kutf_suite.c1041
-rw-r--r--mali_kbase/tests/kutf/kutf_utils.c71
-rw-r--r--mali_kbase/tests/kutf/sconscript21
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/Kbuild20
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/Kconfig23
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/Makefile51
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c257
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/sconscript30
-rw-r--r--mali_kbase/tests/sconscript37
99 files changed, 7921 insertions, 2556 deletions
diff --git a/mali_kbase/Kbuild b/mali_kbase/Kbuild
index 3e78928..eb01993 100644
--- a/mali_kbase/Kbuild
+++ b/mali_kbase/Kbuild
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -15,7 +15,7 @@
# Driver version string which is returned to userspace via an ioctl
-MALI_RELEASE_NAME ?= "r5p0-01rel0"
+MALI_RELEASE_NAME ?= "r6p0-01rel0"
# Paths required for build
KBASE_PATH = $(src)
@@ -77,6 +77,7 @@ SRC := \
mali_kbase_cache_policy.c \
mali_kbase_mem.c \
mali_kbase_mmu.c \
+ mali_kbase_ctx_sched.c \
mali_kbase_jd.c \
mali_kbase_jd_debugfs.c \
mali_kbase_jm.c \
@@ -97,8 +98,6 @@ SRC := \
mali_kbase_gpu_memory_debugfs.c \
mali_kbase_mem_linux.c \
mali_kbase_core_linux.c \
- mali_kbase_sync.c \
- mali_kbase_sync_user.c \
mali_kbase_replay.c \
mali_kbase_mem_profile_debugfs.c \
mali_kbase_mmu_mode_lpae.c \
@@ -143,16 +142,6 @@ ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y)
ccflags-y += -I$(src)/platform/rtsm_ve
endif
- ifeq ($(CONFIG_MALI_PLATFORM_JUNO),y)
- SRC += platform/juno/mali_kbase_config_vexpress.c
- ccflags-y += -I$(src)/platform/juno
- endif
-
- ifeq ($(CONFIG_MALI_PLATFORM_JUNO_SOC),y)
- SRC += platform/juno_soc/mali_kbase_config_juno_soc.c
- ccflags-y += -I$(src)/platform/juno_soc
- endif
-
ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_1XV7_A57),y)
SRC += platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c
ccflags-y += -I$(src)/platform/vexpress_1xv7_a57
@@ -179,11 +168,22 @@ ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y)
include $(src)/$(MALI_PLATFORM_THIRDPARTY_DIR)/Kbuild
endif
-ifeq ($(CONFIG_DEVFREQ_THERMAL),y)
- include $(src)/ipa/Kbuild
+ifeq ($(CONFIG_MALI_DEVFREQ),y)
+ ifeq ($(CONFIG_DEVFREQ_THERMAL),y)
+ include $(src)/ipa/Kbuild
+ endif
endif
-mali_kbase-$(CONFIG_MALI_DMA_FENCE) += mali_kbase_dma_fence.o
+mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \
+ mali_kbase_dma_fence.o \
+ mali_kbase_fence.o
+mali_kbase-$(CONFIG_SYNC) += \
+ mali_kbase_sync_android.o \
+ mali_kbase_sync_common.o
+mali_kbase-$(CONFIG_SYNC_FILE) += \
+ mali_kbase_sync_file.o \
+ mali_kbase_sync_common.o \
+ mali_kbase_fence.o
MALI_BACKEND_PATH ?= backend
CONFIG_MALI_BACKEND ?= gpu
@@ -213,3 +213,6 @@ mali_kbase-$(CONFIG_MALI_PLATFORM_DEVICETREE) += \
platform/devicetree/mali_kbase_runtime_pm.o \
platform/devicetree/mali_kbase_config_devicetree.o
ccflags-$(CONFIG_MALI_PLATFORM_DEVICETREE) += -I$(src)/platform/devicetree
+
+# For kutf and mali_kutf_irq_latency_test
+obj-$(CONFIG_MALI_KUTF) += tests/
diff --git a/mali_kbase/Kconfig b/mali_kbase/Kconfig
index c1acaf0..54565bb 100644
--- a/mali_kbase/Kconfig
+++ b/mali_kbase/Kconfig
@@ -169,7 +169,7 @@ config MALI_DEBUG
config MALI_FENCE_DEBUG
bool "Debug sync fence usage"
- depends on MALI_MIDGARD && MALI_EXPERT && SYNC
+ depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE)
default y if MALI_DEBUG
help
Select this option to enable additional checking and reporting on the
@@ -244,3 +244,4 @@ config MALI_PWRSOFT_765
parts of the code.
source "drivers/gpu/arm/midgard/platform/Kconfig"
+source "drivers/gpu/arm/midgard/tests/Kconfig"
diff --git a/mali_kbase/Makefile b/mali_kbase/Makefile
index e1625e6..9aa242c 100644
--- a/mali_kbase/Makefile
+++ b/mali_kbase/Makefile
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/backend/gpu/mali_kbase_devfreq.c b/mali_kbase/backend/gpu/mali_kbase_devfreq.c
index 574eb3e..e280322 100644
--- a/mali_kbase/backend/gpu/mali_kbase_devfreq.c
+++ b/mali_kbase/backend/gpu/mali_kbase_devfreq.c
@@ -40,6 +40,7 @@
#define dev_pm_opp_get_voltage opp_get_voltage
#define dev_pm_opp_get_opp_count opp_get_opp_count
#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil
+#define dev_pm_opp_find_freq_floor opp_find_freq_floor
#endif /* Linux >= 3.13 */
/**
@@ -362,7 +363,7 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
kbdev->devfreq_cooling = of_devfreq_cooling_register_power(
kbdev->dev->of_node,
kbdev->devfreq,
- &power_model_ops);
+ &kbase_ipa_power_model_ops);
if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) {
err = PTR_ERR(kbdev->devfreq_cooling);
dev_err(kbdev->dev,
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_as.c b/mali_kbase/backend/gpu/mali_kbase_jm_as.c
index 202dcfa..92358f2 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_as.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_as.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -22,6 +22,7 @@
#include <mali_kbase.h>
#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_ctx_sched.h>
/**
* assign_and_activate_kctx_addr_space - Assign an AS to a context
@@ -47,65 +48,20 @@ static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev,
struct kbase_as *current_as)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
- struct kbasep_js_per_as_data *js_per_as_data;
- int as_nr = current_as->number;
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
lockdep_assert_held(&js_devdata->runpool_mutex);
lockdep_assert_held(&kbdev->hwaccess_lock);
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
/* Attribute handling */
kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx);
- /* Assign addr space */
- kctx->as_nr = as_nr;
-
- /* If the GPU is currently powered, activate this address space on the
- * MMU */
- if (kbdev->pm.backend.gpu_powered)
- kbase_mmu_update(kctx);
- /* If the GPU was not powered then the MMU will be reprogrammed on the
- * next pm_context_active() */
-
/* Allow it to run jobs */
kbasep_js_set_submit_allowed(js_devdata, kctx);
- /* Book-keeping */
- js_per_as_data->kctx = kctx;
- js_per_as_data->as_busy_refcount = 0;
-
kbase_js_runpool_inc_context_count(kbdev, kctx);
}
-/**
- * release_addr_space - Release an address space
- * @kbdev: Kbase device
- * @kctx_as_nr: Address space of context to release
- * @kctx: Context being released
- *
- * Context: kbasep_js_device_data.runpool_mutex must be held
- *
- * Release an address space, making it available for being picked again.
- */
-static void release_addr_space(struct kbase_device *kbdev, int kctx_as_nr,
- struct kbase_context *kctx)
-{
- struct kbasep_js_device_data *js_devdata;
- u16 as_bit = (1u << kctx_as_nr);
-
- js_devdata = &kbdev->js_data;
- lockdep_assert_held(&js_devdata->runpool_mutex);
-
- /* The address space must not already be free */
- KBASE_DEBUG_ASSERT(!(js_devdata->as_free & as_bit));
-
- js_devdata->as_free |= as_bit;
-
- kbase_js_runpool_dec_context_count(kbdev, kctx);
-}
-
bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
@@ -117,10 +73,7 @@ bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
}
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- struct kbasep_js_per_as_data *js_per_as_data =
- &kbdev->js_data.runpool_irq.per_as_data[i];
-
- if (js_per_as_data->kctx == kctx) {
+ if (kbdev->as_to_kctx[i] == kctx) {
/* Context already has ASID - mark as active */
return true;
}
@@ -133,7 +86,6 @@ bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
void kbase_backend_release_ctx_irq(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
- struct kbasep_js_per_as_data *js_per_as_data;
int as_nr = kctx->as_nr;
if (as_nr == KBASEP_AS_NR_INVALID) {
@@ -143,100 +95,24 @@ void kbase_backend_release_ctx_irq(struct kbase_device *kbdev,
lockdep_assert_held(&kbdev->hwaccess_lock);
- js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr];
- if (js_per_as_data->as_busy_refcount != 0) {
+ if (atomic_read(&kctx->refcount) != 1) {
WARN(1, "Attempting to release active ASID\n");
return;
}
- /* Release context from address space */
- js_per_as_data->kctx = NULL;
-
kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx);
- /* If the GPU is currently powered, de-activate this address space on
- * the MMU */
- if (kbdev->pm.backend.gpu_powered)
- kbase_mmu_disable(kctx);
- /* If the GPU was not powered then the MMU will be reprogrammed on the
- * next pm_context_active() */
-
- release_addr_space(kbdev, as_nr, kctx);
- kctx->as_nr = KBASEP_AS_NR_INVALID;
-}
-
-void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev,
- struct kbase_context *kctx)
-{
-}
-
-void kbase_backend_release_free_address_space(struct kbase_device *kbdev,
- int as_nr)
-{
- struct kbasep_js_device_data *js_devdata;
-
- js_devdata = &kbdev->js_data;
-
- lockdep_assert_held(&js_devdata->runpool_mutex);
- js_devdata->as_free |= (1 << as_nr);
+ kbase_ctx_sched_release_ctx(kctx);
+ kbase_js_runpool_dec_context_count(kbdev, kctx);
}
-/**
- * check_is_runpool_full - check whether the runpool is full for a specified
- * context
- * @kbdev: Kbase device
- * @kctx: Kbase context
- *
- * If kctx == NULL, then this makes the least restrictive check on the
- * runpool. A specific context that is supplied immediately after could fail
- * the check, even under the same conditions.
- *
- * Therefore, once a context is obtained you \b must re-check it with this
- * function, since the return value could change to false.
- *
- * Context:
- * In all cases, the caller must hold kbasep_js_device_data.runpool_mutex.
- * When kctx != NULL the caller must hold the
- * kbasep_js_kctx_info.ctx.jsctx_mutex.
- * When kctx == NULL, then the caller need not hold any jsctx_mutex locks (but
- * it doesn't do any harm to do so).
- *
- * Return: true if the runpool is full
- */
-static bool check_is_runpool_full(struct kbase_device *kbdev,
+void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
- struct kbasep_js_device_data *js_devdata;
- bool is_runpool_full;
-
- js_devdata = &kbdev->js_data;
- lockdep_assert_held(&js_devdata->runpool_mutex);
-
- /* Regardless of whether a context is submitting or not, can't have more
- * than there are HW address spaces */
- is_runpool_full = (bool) (js_devdata->nr_all_contexts_running >=
- kbdev->nr_hw_address_spaces);
-
- if (kctx && !kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
- lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- /* Contexts that submit might use less of the address spaces
- * available, due to HW workarounds. In which case, the runpool
- * is also full when the number of submitting contexts exceeds
- * the number of submittable address spaces.
- *
- * Both checks must be made: can have nr_user_address_spaces ==
- * nr_hw_address spaces, and at the same time can have
- * nr_user_contexts_running < nr_all_contexts_running. */
- is_runpool_full |= (bool)
- (js_devdata->nr_user_contexts_running >=
- kbdev->nr_user_address_spaces);
- }
-
- return is_runpool_full;
}
-int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
- struct kbase_context *kctx)
+int kbase_backend_find_and_release_free_address_space(
+ struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
@@ -249,37 +125,23 @@ int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
- /* First try to find a free address space */
- if (check_is_runpool_full(kbdev, kctx))
- i = -1;
- else
- i = ffs(js_devdata->as_free) - 1;
-
- if (i >= 0 && i < kbdev->nr_hw_address_spaces) {
- js_devdata->as_free &= ~(1 << i);
-
- mutex_unlock(&js_devdata->runpool_mutex);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
-
- return i;
- }
-
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- /* No address space currently free, see if we can release one */
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- struct kbasep_js_per_as_data *js_per_as_data;
struct kbasep_js_kctx_info *as_js_kctx_info;
struct kbase_context *as_kctx;
- js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[i];
- as_kctx = js_per_as_data->kctx;
+ as_kctx = kbdev->as_to_kctx[i];
as_js_kctx_info = &as_kctx->jctx.sched_info;
/* Don't release privileged or active contexts, or contexts with
- * jobs running */
+ * jobs running.
+ * Note that a context will have at least 1 reference (which
+ * was previously taken by kbasep_js_schedule_ctx()) until
+ * descheduled.
+ */
if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) &&
- js_per_as_data->as_busy_refcount == 0) {
+ atomic_read(&as_kctx->refcount) == 1) {
if (!kbasep_js_runpool_retain_ctx_nolock(kbdev,
as_kctx)) {
WARN(1, "Failed to retain active context\n");
@@ -314,8 +176,6 @@ int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
as_kctx,
true);
- js_devdata->as_free &= ~(1 << i);
-
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex);
@@ -353,16 +213,15 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev,
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
- if (kbdev->hwaccess.active_kctx == kctx ||
- kctx->as_nr != KBASEP_AS_NR_INVALID ||
- as_nr == KBASEP_AS_NR_INVALID) {
- WARN(1, "Invalid parameters to use_ctx()\n");
+ if (kbdev->hwaccess.active_kctx == kctx) {
+ WARN(1, "Context is already scheduled in\n");
return false;
}
new_address_space = &kbdev->as[as_nr];
lockdep_assert_held(&js_devdata->runpool_mutex);
+ lockdep_assert_held(&kbdev->mmu_hw_mutex);
lockdep_assert_held(&kbdev->hwaccess_lock);
assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space);
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
index ef7497d..a6fb097 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
@@ -29,6 +29,7 @@
#include <mali_kbase_vinstr.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_ctx_sched.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_js_affinity.h>
@@ -1155,7 +1156,6 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
{
unsigned long flags;
struct kbase_device *kbdev;
- int i;
ktime_t end_timestamp = ktime_get();
struct kbasep_js_device_data *js_devdata;
bool try_schedule = false;
@@ -1193,6 +1193,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
KBASE_RESET_GPU_NOT_PENDING);
kbase_disjoint_state_down(kbdev);
wake_up(&kbdev->hwaccess.backend.reset_wait);
+ kbase_vinstr_resume(kbdev->vinstr_ctx);
return;
}
@@ -1265,18 +1266,9 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->mmu_hw_mutex);
- /* Reprogram the GPU's MMU */
- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-
- if (js_devdata->runpool_irq.per_as_data[i].kctx)
- kbase_mmu_update(
- js_devdata->runpool_irq.per_as_data[i].kctx);
- else
- kbase_mmu_disable_as(kbdev, i);
-
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- }
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ kbase_ctx_sched_restore_all_as(kbdev);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&kbdev->mmu_hw_mutex);
kbase_pm_enable_interrupts(kbdev);
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
index 418ae1f..1e9b9e5 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
@@ -672,10 +672,9 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev,
(katom->protected_state.enter ==
KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) {
kbase_vinstr_resume(kbdev->vinstr_ctx);
-#ifdef CONFIG_DEVFREQ_THERMAL
+
/* Go back to configured model for IPA */
kbase_ipa_model_use_configured_locked(kbdev);
-#endif
}
@@ -784,7 +783,8 @@ static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
if (kbdev->protected_ops) {
/* Switch GPU to protected mode */
- err = kbdev->protected_ops->protected_mode_enter(kbdev);
+ err = kbdev->protected_ops->protected_mode_enable(
+ kbdev->protected_dev);
if (err)
dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
@@ -806,6 +806,8 @@ static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
if (!kbdev->protected_ops)
return -EINVAL;
+ /* The protected mode disable callback will be called as part of reset
+ */
kbase_reset_gpu_silent(kbdev);
return 0;
@@ -841,10 +843,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
return -EAGAIN;
}
-#ifdef CONFIG_DEVFREQ_THERMAL
/* Use generic model for IPA in protected mode */
kbase_ipa_model_use_fallback_locked(kbdev);
-#endif
/* Once reaching this point GPU must be
* switched to protected mode or vinstr
@@ -907,10 +907,9 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
kbase_gpu_dequeue_atom(kbdev, js, NULL);
kbase_jm_return_atom_to_js(kbdev, katom[idx]);
}
-#ifdef CONFIG_DEVFREQ_THERMAL
+
/* Go back to configured model for IPA */
kbase_ipa_model_use_configured_locked(kbdev);
-#endif
return -EINVAL;
}
@@ -990,10 +989,9 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
}
kbase_vinstr_resume(kbdev->vinstr_ctx);
-#ifdef CONFIG_DEVFREQ_THERMAL
+
/* Use generic model for IPA in protected mode */
kbase_ipa_model_use_fallback_locked(kbdev);
-#endif
return -EINVAL;
}
@@ -1516,6 +1514,8 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
{
KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev);
+ kbase_vinstr_resume(kbdev->vinstr_ctx);
+
/* protected mode sanity checks */
KBASE_DEBUG_ASSERT_MSG(
kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev),
diff --git a/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c b/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c
index 4e5a74f..aa1817c 100644
--- a/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c
+++ b/mali_kbase/backend/gpu/mali_kbase_mmu_hw_direct.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -200,15 +200,15 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
KBASE_MMU_FAULT_TYPE_BUS :
KBASE_MMU_FAULT_TYPE_PAGE;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- as->fault_extra_addr = kbase_reg_read(kbdev,
- MMU_AS_REG(as_no, AS_FAULTEXTRA_HI),
- kctx);
- as->fault_extra_addr <<= 32;
- as->fault_extra_addr |= kbase_reg_read(kbdev,
- MMU_AS_REG(as_no, AS_FAULTEXTRA_LO),
- kctx);
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) {
+ as->fault_extra_addr = kbase_reg_read(kbdev,
+ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI),
+ kctx);
+ as->fault_extra_addr <<= 32;
+ as->fault_extra_addr |= kbase_reg_read(kbdev,
+ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO),
+ kctx);
+ }
if (kbase_as_has_bus_fault(as)) {
/* Mark bus fault as handled.
@@ -248,34 +248,32 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as,
struct kbase_mmu_setup *current_setup = &as->current_setup;
u32 transcfg = 0;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- transcfg = current_setup->transcfg & 0xFFFFFFFFUL;
-
- /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */
- /* Clear PTW_MEMATTR bits */
- transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK;
- /* Enable correct PTW_MEMATTR bits */
- transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK;
-
- if (kbdev->system_coherency == COHERENCY_ACE) {
- /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */
- /* Clear PTW_SH bits */
- transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK);
- /* Enable correct PTW_SH bits */
- transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS);
- }
-
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO),
- transcfg, kctx);
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI),
- (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, kctx);
-
-#else /* CONFIG_MALI_GPU_MMU_AARCH64 */
-
- if (kbdev->system_coherency == COHERENCY_ACE)
- current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER;
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) {
+ transcfg = current_setup->transcfg & 0xFFFFFFFFUL;
+
+ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */
+ /* Clear PTW_MEMATTR bits */
+ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK;
+ /* Enable correct PTW_MEMATTR bits */
+ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK;
+
+ if (kbdev->system_coherency == COHERENCY_ACE) {
+ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */
+ /* Clear PTW_SH bits */
+ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK);
+ /* Enable correct PTW_SH bits */
+ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS);
+ }
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO),
+ transcfg, kctx);
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI),
+ (current_setup->transcfg >> 32) & 0xFFFFFFFFUL,
+ kctx);
+ } else {
+ if (kbdev->system_coherency == COHERENCY_ACE)
+ current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER;
+ }
kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO),
current_setup->transtab & 0xFFFFFFFFUL, kctx);
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
index ac63aa5..ed19a8a 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
@@ -32,6 +32,7 @@
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_smc.h>
#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_ctx_sched.h>
#include <backend/gpu/mali_kbase_cache_policy_backend.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
@@ -1025,7 +1026,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
bool reset_required = is_resume;
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long flags;
- int i;
KBASE_DEBUG_ASSERT(NULL != kbdev);
lockdep_assert_held(&js_devdata->runpool_mutex);
@@ -1067,18 +1067,9 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
}
mutex_lock(&kbdev->mmu_hw_mutex);
- /* Reprogram the GPU's MMU */
- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-
- if (js_devdata->runpool_irq.per_as_data[i].kctx)
- kbase_mmu_update(
- js_devdata->runpool_irq.per_as_data[i].kctx);
- else
- kbase_mmu_disable_as(kbdev, i);
-
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- }
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ kbase_ctx_sched_restore_all_as(kbdev);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&kbdev->mmu_hw_mutex);
/* Lastly, enable the interrupts */
@@ -1359,7 +1350,7 @@ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev)
}
}
-static int kbase_pm_reset_do_normal(struct kbase_device *kbdev)
+static int kbase_pm_do_reset(struct kbase_device *kbdev)
{
struct kbasep_reset_timeout_data rtdata;
@@ -1439,14 +1430,29 @@ static int kbase_pm_reset_do_normal(struct kbase_device *kbdev)
return -EINVAL;
}
-static int kbase_pm_reset_do_protected(struct kbase_device *kbdev)
+static int kbasep_protected_mode_enable(struct protected_mode_device *pdev)
{
- KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
- KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev);
+ struct kbase_device *kbdev = pdev->data;
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_SET_PROTECTED_MODE, NULL);
+ return 0;
+}
- return kbdev->protected_ops->protected_mode_reset(kbdev);
+static int kbasep_protected_mode_disable(struct protected_mode_device *pdev)
+{
+ struct kbase_device *kbdev = pdev->data;
+
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ return kbase_pm_do_reset(kbdev);
}
+struct protected_mode_ops kbase_native_protected_ops = {
+ .protected_mode_enable = kbasep_protected_mode_enable,
+ .protected_mode_disable = kbasep_protected_mode_disable
+};
+
int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
{
unsigned long irq_flags;
@@ -1490,19 +1496,17 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
/* Soft reset the GPU */
- if (kbdev->protected_mode_support &&
- kbdev->protected_ops->protected_mode_reset)
- err = kbase_pm_reset_do_protected(kbdev);
+ if (kbdev->protected_mode_support)
+ err = kbdev->protected_ops->protected_mode_disable(
+ kbdev->protected_dev);
else
- err = kbase_pm_reset_do_normal(kbdev);
+ err = kbase_pm_do_reset(kbdev);
spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
if (kbdev->protected_mode)
resume_vinstr = true;
kbdev->protected_mode = false;
-#ifdef CONFIG_DEVFREQ_THERMAL
kbase_ipa_model_use_configured_locked(kbdev);
-#endif
spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
index 58f615d..6804f45 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -486,11 +486,11 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume);
*/
void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend);
-#ifdef CONFIG_PM_DEVFREQ
+#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS)
void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev,
unsigned long *total, unsigned long *busy);
void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev);
-#endif
+#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */
#ifdef CONFIG_MALI_MIDGARD_DVFS
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
index 7613e1d..024248c 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -155,7 +155,7 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev,
kbdev->pm.backend.metrics.time_period_start = now;
}
-#if defined(CONFIG_PM_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS)
+#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS)
/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this
* function.
*/
diff --git a/mali_kbase/ipa/Kbuild b/mali_kbase/ipa/Kbuild
index b3f8761..03dec47 100644
--- a/mali_kbase/ipa/Kbuild
+++ b/mali_kbase/ipa/Kbuild
@@ -18,3 +18,7 @@ mali_kbase-y += \
ipa/mali_kbase_ipa.o
mali_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o
+
+ifneq ($(wildcard $(src)/ipa/mali_kbase_ipa_tmix.c),)
+ mali_kbase-y += ipa/mali_kbase_ipa_tmix.o
+endif
diff --git a/mali_kbase/ipa/mali_kbase_ipa.c b/mali_kbase/ipa/mali_kbase_ipa.c
index d289ea8..ba897d7 100644
--- a/mali_kbase/ipa/mali_kbase_ipa.c
+++ b/mali_kbase/ipa/mali_kbase_ipa.c
@@ -32,6 +32,10 @@
#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model"
+static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = {
+ &kbase_simple_ipa_model_ops,
+};
+
int kbase_ipa_model_recalculate(struct kbase_ipa_model *model)
{
int err = 0;
@@ -50,57 +54,20 @@ int kbase_ipa_model_recalculate(struct kbase_ipa_model *model)
return err;
}
-int kbase_ipa_model_ops_register(struct kbase_device *kbdev,
- struct kbase_ipa_model_ops *new_model_ops)
-{
- struct kbase_ipa_model *new_model;
-
- lockdep_assert_held(&kbdev->ipa.lock);
-
- new_model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL);
- if (!new_model)
- return -ENOMEM;
-
- new_model->kbdev = kbdev;
- new_model->ops = new_model_ops;
-
- list_add(&new_model->link, &kbdev->ipa.power_models);
-
- return 0;
-}
-
-static int kbase_ipa_internal_models_append_list(struct kbase_device *kbdev)
-{
- int err;
-
- INIT_LIST_HEAD(&kbdev->ipa.power_models);
-
- /* Always have the simple IPA model */
- err = kbase_ipa_model_ops_register(kbdev, &kbase_simple_ipa_model_ops);
-
- if (err)
- return err;
-
- return err;
-}
-
-struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev,
- const char *name)
+static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev,
+ const char *name)
{
- /* Search registered power models first */
- struct list_head *it;
+ int i;
- lockdep_assert_held(&kbdev->ipa.lock);
+ for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) {
+ struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i];
- list_for_each(it, &kbdev->ipa.power_models) {
- struct kbase_ipa_model *model =
- list_entry(it,
- struct kbase_ipa_model,
- link);
- if (strcmp(model->ops->name, name) == 0)
- return model;
+ if (!strcmp(ops->name, name))
+ return ops;
}
+ dev_err(kbdev->dev, "power model \'%s\' not found\n", name);
+
return NULL;
}
@@ -109,7 +76,6 @@ void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev)
atomic_set(&kbdev->ipa_use_configured_model, false);
}
-
void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev)
{
atomic_set(&kbdev->ipa_use_configured_model, true);
@@ -142,72 +108,54 @@ static struct device_node *get_model_dt_node(struct kbase_ipa_model *model)
model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node,
NULL, compat_string);
- if (!model_dt_node) {
+ if (!model_dt_node && !model->missing_dt_node_warning) {
dev_warn(model->kbdev->dev,
"Couldn't find power_model DT node matching \'%s\'\n",
compat_string);
+ model->missing_dt_node_warning = true;
}
return model_dt_node;
}
-int kbase_ipa_model_add_param_u32_def(struct kbase_ipa_model *model,
- const char *name, u32 *addr,
- bool has_default, u32 default_value)
-{
- int err;
- struct device_node *model_dt_node = get_model_dt_node(model);
-
- err = of_property_read_u32(model_dt_node, name, addr);
-
- if (err && !has_default) {
- dev_err(model->kbdev->dev,
- "No DT entry or default found for %s.%s, err = %d\n",
- model->ops->name, name, err);
- goto exit;
- } else if (err && has_default) {
- *addr = default_value;
- dev_dbg(model->kbdev->dev, "%s.%s = %u (default)\n",
- model->ops->name, name, *addr);
- err = 0;
- } else /* !err */ {
- dev_dbg(model->kbdev->dev, "%s.%s = %u (DT)\n",
- model->ops->name, name, *addr);
- }
-
- err = kbase_ipa_model_param_add(model, name, addr, sizeof(u32),
- PARAM_TYPE_U32);
-exit:
- return err;
-}
-
-int kbase_ipa_model_add_param_s32_array(struct kbase_ipa_model *model,
- const char *name, s32 *addr,
- size_t num_elems)
+int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model,
+ const char *name, s32 *addr,
+ size_t num_elems, bool dt_required)
{
int err, i;
struct device_node *model_dt_node = get_model_dt_node(model);
+ char *origin;
err = of_property_read_u32_array(model_dt_node, name, addr, num_elems);
- if (err) {
- dev_err(model->kbdev->dev,
- "No DT entry found for %s.%s, err = %d\n",
- model->ops->name, name, err);
- goto exit;
- } else {
- for (i = 0; i < num_elems; ++i)
- dev_dbg(model->kbdev->dev, "%s.%s[%u] = %d (DT)\n",
- model->ops->name, name, i, *(addr + i));
+ if (err && dt_required) {
+ memset(addr, 0, sizeof(s32) * num_elems);
+ dev_warn(model->kbdev->dev,
+ "Error %d, no DT entry: %s.%s = %zu*[0]\n",
+ err, model->ops->name, name, num_elems);
+ origin = "zero";
+ } else if (err && !dt_required) {
+ origin = "default";
+ } else /* !err */ {
+ origin = "DT";
}
/* Create a unique debugfs entry for each element */
for (i = 0; i < num_elems; ++i) {
char elem_name[32];
- snprintf(elem_name, sizeof(elem_name), "%s.%d", name, i);
- err = kbase_ipa_model_param_add(model, elem_name, &addr[i],
- sizeof(s32), PARAM_TYPE_S32);
+ if (num_elems == 1)
+ snprintf(elem_name, sizeof(elem_name), "%s", name);
+ else
+ snprintf(elem_name, sizeof(elem_name), "%s.%d",
+ name, i);
+
+ dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n",
+ model->ops->name, elem_name, addr[i], origin);
+
+ err = kbase_ipa_model_param_add(model, elem_name,
+ &addr[i], sizeof(s32),
+ PARAM_TYPE_S32);
if (err)
goto exit;
}
@@ -217,32 +165,41 @@ exit:
int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model,
const char *name, char *addr,
- size_t len)
+ size_t size, bool dt_required)
{
int err;
struct device_node *model_dt_node = get_model_dt_node(model);
const char *string_prop_value;
+ char *origin;
err = of_property_read_string(model_dt_node, name,
&string_prop_value);
- if (err) {
- dev_err(model->kbdev->dev,
- "No DT entry found for %s.%s, err = %d\n",
- model->ops->name, name, err);
- goto exit;
- } else {
- strncpy(addr, string_prop_value, len);
- dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (DT)\n",
- model->ops->name, name, string_prop_value);
+ if (err && dt_required) {
+ strncpy(addr, "", size - 1);
+ dev_warn(model->kbdev->dev,
+ "Error %d, no DT entry: %s.%s = \'%s\'\n",
+ err, model->ops->name, name, addr);
+ err = 0;
+ origin = "zero";
+ } else if (err && !dt_required) {
+ origin = "default";
+ } else /* !err */ {
+ strncpy(addr, string_prop_value, size - 1);
+ origin = "DT";
}
- err = kbase_ipa_model_param_add(model, name, addr, len,
+ addr[size - 1] = '\0';
+
+ dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n",
+ model->ops->name, name, string_prop_value, origin);
+
+ err = kbase_ipa_model_param_add(model, name, addr, size,
PARAM_TYPE_STRING);
-exit:
+
return err;
}
-static void term_model(struct kbase_ipa_model *model)
+void kbase_ipa_term_model(struct kbase_ipa_model *model)
{
if (!model)
return;
@@ -253,42 +210,49 @@ static void term_model(struct kbase_ipa_model *model)
model->ops->term(model);
kbase_ipa_model_param_free_all(model);
+
+ kfree(model);
}
+KBASE_EXPORT_TEST_API(kbase_ipa_term_model);
-static struct kbase_ipa_model *init_model(struct kbase_device *kbdev,
- const char *model_name)
+struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev,
+ struct kbase_ipa_model_ops *ops)
{
struct kbase_ipa_model *model;
int err;
lockdep_assert_held(&kbdev->ipa.lock);
- model = kbase_ipa_get_model(kbdev, model_name);
- if (!model) {
- dev_err(kbdev->dev, "power model \'%s\' not found\n",
- model_name);
+ if (!ops || !ops->name)
return NULL;
- }
+ model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL);
+ if (!model)
+ return NULL;
+
+ model->kbdev = kbdev;
+ model->ops = ops;
INIT_LIST_HEAD(&model->params);
err = model->ops->init(model);
if (err) {
dev_err(kbdev->dev,
"init of power model \'%s\' returned error %d\n",
- model_name, err);
- term_model(model);
- return NULL;
+ ops->name, err);
+ goto term_model;
}
err = kbase_ipa_model_recalculate(model);
- if (err) {
- term_model(model);
- return NULL;
- }
+ if (err)
+ goto term_model;
return model;
+
+term_model:
+ kbase_ipa_term_model(model);
+ return NULL;
}
+KBASE_EXPORT_TEST_API(kbase_ipa_init_model);
static void kbase_ipa_term_locked(struct kbase_device *kbdev)
{
@@ -296,24 +260,18 @@ static void kbase_ipa_term_locked(struct kbase_device *kbdev)
/* Clean up the models */
if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model)
- term_model(kbdev->ipa.configured_model);
- term_model(kbdev->ipa.fallback_model);
-
- /* Clean up the list */
- if (!list_empty(&kbdev->ipa.power_models)) {
- struct kbase_ipa_model *model_p, *model_n;
+ kbase_ipa_term_model(kbdev->ipa.configured_model);
+ kbase_ipa_term_model(kbdev->ipa.fallback_model);
- list_for_each_entry_safe(model_p, model_n, &kbdev->ipa.power_models, link) {
- list_del(&model_p->link);
- kfree(model_p);
- }
- }
+ kbdev->ipa.configured_model = NULL;
+ kbdev->ipa.fallback_model = NULL;
}
int kbase_ipa_init(struct kbase_device *kbdev)
{
const char *model_name;
+ struct kbase_ipa_model_ops *ops;
struct kbase_ipa_model *default_model = NULL;
int err;
@@ -324,11 +282,18 @@ int kbase_ipa_init(struct kbase_device *kbdev)
*/
mutex_lock(&kbdev->ipa.lock);
- /* Add default ones to the list */
- err = kbase_ipa_internal_models_append_list(kbdev);
-
/* The simple IPA model must *always* be present.*/
- default_model = init_model(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME);
+ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME);
+
+ if (!ops->do_utilization_scaling_in_framework) {
+ dev_err(kbdev->dev,
+ "Fallback IPA model %s should not account for utilization\n",
+ ops->name);
+ err = -EINVAL;
+ goto end;
+ }
+
+ default_model = kbase_ipa_init_model(kbdev, ops);
if (!default_model) {
err = -EINVAL;
goto end;
@@ -344,10 +309,18 @@ int kbase_ipa_init(struct kbase_device *kbdev)
gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
model_name = kbase_ipa_model_name_from_id(gpu_id);
+ dev_dbg(kbdev->dev,
+ "Inferring model from GPU ID 0x%x: \'%s\'\n",
+ gpu_id, model_name);
+ } else {
+ dev_dbg(kbdev->dev,
+ "Using ipa-model parameter from DT: \'%s\'\n",
+ model_name);
}
if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) {
- kbdev->ipa.configured_model = init_model(kbdev, model_name);
+ ops = kbase_ipa_model_ops_find(kbdev, model_name);
+ kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops);
if (!kbdev->ipa.configured_model) {
err = -EINVAL;
goto end;
@@ -365,12 +338,13 @@ end:
else
dev_info(kbdev->dev,
"Using configured power model %s, and fallback %s\n",
- kbdev->ipa.fallback_model->ops->name,
- kbdev->ipa.configured_model->ops->name);
+ kbdev->ipa.configured_model->ops->name,
+ kbdev->ipa.fallback_model->ops->name);
mutex_unlock(&kbdev->ipa.lock);
return err;
}
+KBASE_EXPORT_TEST_API(kbase_ipa_init);
void kbase_ipa_term(struct kbase_device *kbdev)
{
@@ -378,6 +352,7 @@ void kbase_ipa_term(struct kbase_device *kbdev)
kbase_ipa_term_locked(kbdev);
mutex_unlock(&kbdev->ipa.lock);
}
+KBASE_EXPORT_TEST_API(kbase_ipa_term);
/**
* kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP
@@ -393,21 +368,20 @@ void kbase_ipa_term(struct kbase_device *kbdev)
*
* Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W)
*/
-static inline unsigned long kbase_scale_dynamic_power(const unsigned long c,
- const unsigned long freq,
- const unsigned long voltage)
+static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq,
+ const u32 voltage)
{
/* Range: 2^8 < v2 < 2^16 m(V^2) */
- const unsigned long v2 = (voltage * voltage) / 1000;
+ const u32 v2 = (voltage * voltage) / 1000;
/* Range: 2^3 < f_MHz < 2^10 MHz */
- const unsigned long f_MHz = freq / 1000000;
+ const u32 f_MHz = freq / 1000000;
/* Range: 2^11 < v2f_big < 2^26 kHz V^2 */
- const unsigned long v2f_big = v2 * f_MHz;
+ const u32 v2f_big = v2 * f_MHz;
/* Range: 2^1 < v2f < 2^16 MHz V^2 */
- const unsigned long v2f = v2f_big / 1000;
+ const u32 v2f = v2f_big / 1000;
/* Range (working backwards from next line): 0 < v2fc < 2^23 uW.
* Must be < 2^42 to avoid overflowing the return value. */
@@ -425,17 +399,16 @@ static inline unsigned long kbase_scale_dynamic_power(const unsigned long c,
*
* Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W)
*/
-unsigned long kbase_scale_static_power(const unsigned long c,
- const unsigned long voltage)
+u32 kbase_scale_static_power(const u32 c, const u32 voltage)
{
/* Range: 2^8 < v2 < 2^16 m(V^2) */
- const unsigned long v2 = (voltage * voltage) / 1000;
+ const u32 v2 = (voltage * voltage) / 1000;
/* Range: 2^17 < v3_big < 2^29 m(V^2) mV */
- const unsigned long v3_big = v2 * voltage;
+ const u32 v3_big = v2 * voltage;
/* Range: 2^7 < v3 < 2^19 m(V^3) */
- const unsigned long v3 = v3_big / 1000;
+ const u32 v3 = v3_big / 1000;
/*
* Range (working backwards from next line): 0 < v3c_big < 2^33 nW.
@@ -457,6 +430,29 @@ static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev)
return kbdev->ipa.fallback_model;
}
+static u32 get_static_power_locked(struct kbase_device *kbdev,
+ struct kbase_ipa_model *model,
+ unsigned long voltage)
+{
+ u32 power = 0;
+ int err;
+ u32 power_coeff;
+
+ lockdep_assert_held(&model->kbdev->ipa.lock);
+
+ if (!model->ops->get_static_coeff)
+ model = kbdev->ipa.fallback_model;
+
+ if (model->ops->get_static_coeff) {
+ err = model->ops->get_static_coeff(model, &power_coeff);
+ if (!err)
+ power = kbase_scale_static_power(power_coeff,
+ (u32) voltage);
+ }
+
+ return power;
+}
+
#ifdef CONFIG_MALI_PWRSOFT_765
static unsigned long kbase_get_static_power(struct devfreq *df,
unsigned long voltage)
@@ -465,7 +461,7 @@ static unsigned long kbase_get_static_power(unsigned long voltage)
#endif
{
struct kbase_ipa_model *model;
- unsigned long power_coeff = 0, power = 0;
+ u32 power = 0;
#ifdef CONFIG_MALI_PWRSOFT_765
struct kbase_device *kbdev = dev_get_drvdata(&df->dev);
#else
@@ -475,15 +471,7 @@ static unsigned long kbase_get_static_power(unsigned long voltage)
mutex_lock(&kbdev->ipa.lock);
model = get_current_model(kbdev);
-
- if (model) {
- power_coeff = model->ops->get_static_power(model);
- power = kbase_scale_static_power(power_coeff, voltage);
- } else {
- dev_err(kbdev->dev, "%s: No current IPA model set", __func__);
- }
-
- kbdev->ipa.last_static_power_coeff = power_coeff;
+ power = get_static_power_locked(kbdev, model, voltage);
mutex_unlock(&kbdev->ipa.lock);
@@ -504,7 +492,8 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq,
#endif
{
struct kbase_ipa_model *model;
- unsigned long power_coeff = 0, power = 0;
+ u32 power_coeff = 0, power = 0;
+ int err = 0;
#ifdef CONFIG_MALI_PWRSOFT_765
struct kbase_device *kbdev = dev_get_drvdata(&df->dev);
#else
@@ -513,16 +502,16 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq,
mutex_lock(&kbdev->ipa.lock);
- model = get_current_model(kbdev);
+ model = kbdev->ipa.fallback_model;
- if (model) {
- power_coeff = model->ops->get_dynamic_power(model);
- power = kbase_scale_dynamic_power(power_coeff, freq, voltage);
- } else {
- dev_err(kbdev->dev, "%s: No current IPA model set", __func__);
- }
+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq);
- kbdev->ipa.last_model_dyn_power_coeff = power_coeff;
+ if (!err)
+ power = kbase_scale_dynamic_power(power_coeff, freq, voltage);
+ else
+ dev_err_ratelimited(kbdev->dev,
+ "Model %s returned error code %d\n",
+ model->ops->name, err);
mutex_unlock(&kbdev->ipa.lock);
@@ -533,90 +522,60 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq,
return power;
}
-unsigned long kbase_power_to_state(struct devfreq *df, u32 target_power)
+int kbase_get_real_power(struct devfreq *df, u32 *power,
+ unsigned long freq,
+ unsigned long voltage)
{
+ struct kbase_ipa_model *model;
+ u32 power_coeff = 0;
+ int err = 0;
struct kbase_device *kbdev = dev_get_drvdata(&df->dev);
- struct device *dev = df->dev.parent;
- unsigned long i, state = -1;
- unsigned long dyn_coeff, static_coeff;
mutex_lock(&kbdev->ipa.lock);
- dyn_coeff = kbdev->ipa.last_model_dyn_power_coeff;
- static_coeff = kbdev->ipa.last_static_power_coeff;
+ model = get_current_model(kbdev);
- mutex_unlock(&kbdev->ipa.lock);
+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq);
- /* OPPs are sorted from highest frequency to lowest */
- for (i = 0; i < df->profile->max_state - 1; i++) {
- struct dev_pm_opp *opp;
- unsigned int freq;
- unsigned long dyn_power, static_power, voltage;
+ /* If we switch to protected model between get_current_model() and
+ * get_dynamic_coeff(), counter reading could fail. If that happens
+ * (unlikely, but possible), revert to the fallback model. */
+ if (err && model != kbdev->ipa.fallback_model) {
+ model = kbdev->ipa.fallback_model;
+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq);
+ }
- freq = df->profile->freq_table[i]; /* Hz */
+ if (err)
+ goto exit_unlock;
- rcu_read_lock();
- opp = dev_pm_opp_find_freq_exact(dev, freq, true);
- /* Allow unavailable frequencies too in case we can enable a
- * higher one. */
- if (PTR_ERR(opp) == -ERANGE)
- opp = dev_pm_opp_find_freq_exact(dev, freq, false);
+ *power = kbase_scale_dynamic_power(power_coeff, freq, voltage);
- if (IS_ERR(opp)) {
- rcu_read_unlock();
- return PTR_ERR(opp);
- }
+ if (model->ops->do_utilization_scaling_in_framework) {
+ struct devfreq_dev_status *status = &df->last_status;
+ unsigned long total_time = max(status->total_time, 1ul);
+ u64 busy_time = min(status->busy_time, total_time);
- voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */
- rcu_read_unlock();
+ *power = ((u64) *power * (u64) busy_time) / total_time;
+ }
- dyn_power = kbase_scale_dynamic_power(dyn_coeff, freq, voltage);
- static_power = kbase_scale_static_power(static_coeff, voltage);
+ *power += get_static_power_locked(kbdev, model, voltage);
- if (target_power >= dyn_power + static_power)
- break;
- }
- state = i;
+exit_unlock:
+ mutex_unlock(&kbdev->ipa.lock);
- return state;
+ return err;
}
-KBASE_EXPORT_TEST_API(kbase_power_to_state);
+KBASE_EXPORT_TEST_API(kbase_get_real_power);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
-struct devfreq_cooling_ops power_model_ops = {
+struct devfreq_cooling_ops kbase_ipa_power_model_ops = {
#else
-struct devfreq_cooling_power power_model_ops = {
+struct devfreq_cooling_power kbase_ipa_power_model_ops = {
#endif
.get_static_power = &kbase_get_static_power,
.get_dynamic_power = &kbase_get_dynamic_power,
#ifdef CONFIG_MALI_PWRSOFT_765
- .power2state = &kbase_power_to_state,
+ .get_real_power = &kbase_get_real_power,
#endif
};
-
-unsigned long kbase_ipa_dynamic_power(struct kbase_device *kbdev,
- unsigned long freq,
- unsigned long voltage)
-{
-#ifdef CONFIG_MALI_PWRSOFT_765
- struct devfreq *df = kbdev->devfreq;
-
- return kbase_get_dynamic_power(df, freq, voltage);
-#else
- return kbase_get_dynamic_power(freq, voltage);
-#endif
-}
-KBASE_EXPORT_TEST_API(kbase_ipa_dynamic_power);
-
-unsigned long kbase_ipa_static_power(struct kbase_device *kbdev,
- unsigned long voltage)
-{
-#ifdef CONFIG_MALI_PWRSOFT_765
- struct devfreq *df = kbdev->devfreq;
-
- return kbase_get_static_power(df, voltage);
-#else
- return kbase_get_static_power(voltage);
-#endif
-}
-KBASE_EXPORT_TEST_API(kbase_ipa_static_power);
+KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops);
diff --git a/mali_kbase/ipa/mali_kbase_ipa.h b/mali_kbase/ipa/mali_kbase_ipa.h
index 540d34a..b2d3db1 100644
--- a/mali_kbase/ipa/mali_kbase_ipa.h
+++ b/mali_kbase/ipa/mali_kbase_ipa.h
@@ -18,29 +18,51 @@
#ifndef _KBASE_IPA_H_
#define _KBASE_IPA_H_
+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)
+
+struct devfreq;
+
struct kbase_ipa_model {
struct list_head link;
struct kbase_device *kbdev;
void *model_data;
struct kbase_ipa_model_ops *ops;
struct list_head params;
+ bool missing_dt_node_warning;
};
-int kbase_ipa_model_add_param_u32_def(struct kbase_ipa_model *model,
- const char *name, u32 *addr,
- bool has_default, u32 default_value);
-
-int kbase_ipa_model_add_param_s32_array(struct kbase_ipa_model *model,
- const char *name, s32 *addr,
- size_t num_elems);
+/**
+ * kbase_ipa_model_add_param_s32 - Add an integer model parameter
+ * @model: pointer to IPA model
+ * @name: name of corresponding debugfs entry
+ * @addr: address where the value is stored
+ * @num_elems: number of elements (1 if not an array)
+ * @dt_required: if false, a corresponding devicetree entry is not required,
+ * and the current value will be used. If true, a warning is
+ * output and the data is zeroed
+ *
+ * Return: 0 on success, or an error code
+ */
+int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model,
+ const char *name, s32 *addr,
+ size_t num_elems, bool dt_required);
+/**
+ * kbase_ipa_model_add_param_string - Add a string model parameter
+ * @model: pointer to IPA model
+ * @name: name of corresponding debugfs entry
+ * @addr: address where the value is stored
+ * @size: size, in bytes, of the value storage (so the maximum string
+ * length is size - 1)
+ * @dt_required: if false, a corresponding devicetree entry is not required,
+ * and the current value will be used. If true, a warning is
+ * output and the data is zeroed
+ *
+ * Return: 0 on success, or an error code
+ */
int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model,
const char *name, char *addr,
- size_t len);
-
-#define kbase_ipa_model_add_param_u32(MODEL, NAME, ADDR) \
- kbase_ipa_model_add_param_u32_def((MODEL), (NAME), (ADDR), \
- false, 0)
+ size_t size, bool dt_required);
struct kbase_ipa_model_ops {
char *name;
@@ -56,64 +78,71 @@ struct kbase_ipa_model_ops {
* recalculated. */
int (*recalculate)(struct kbase_ipa_model *model);
void (*term)(struct kbase_ipa_model *model);
- /* get_dynamic_power() - return a coefficient with units pW/(Hz V^2),
- * which is scaled by the IPA framework according to the current OPP's
- * frequency and voltage. */
- unsigned long (*get_dynamic_power)(struct kbase_ipa_model *model);
- /* get_static_power() - return a coefficient with units uW/(V^3),
- * which is scaled by the IPA framework according to the current OPP's
- * voltage. */
- unsigned long (*get_static_power)(struct kbase_ipa_model *model);
+ /*
+ * get_dynamic_coeff() - calculate dynamic power coefficient
+ * @model: pointer to model
+ * @coeffp: pointer to return value location
+ * @current_freq: frequency the GPU has been running at for the
+ * previous sampling period.
+ *
+ * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which
+ * is then scaled by the IPA framework according to the current OPP's
+ * frequency and voltage.
+ *
+ * Return: 0 on success, or an error code.
+ */
+ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp,
+ u32 current_freq);
+ /*
+ * get_static_coeff() - calculate static power coefficient
+ * @model: pointer to model
+ * @coeffp: pointer to return value location
+ *
+ * Calculate a static power coefficient, with units uW/(V^3), which is
+ * scaled by the IPA framework according to the current OPP's voltage.
+ *
+ * Return: 0 on success, or an error code.
+ */
+ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp);
+ /* If false, the model's get_dynamic_coeff() method accounts for how
+ * long the GPU was active over the sample period. If true, the
+ * framework will scale the calculated power according to the
+ * utilization stats recorded by devfreq in get_real_power(). */
+ bool do_utilization_scaling_in_framework;
};
/* Models can be registered only in the platform's platform_init_func call */
int kbase_ipa_model_ops_register(struct kbase_device *kbdev,
struct kbase_ipa_model_ops *new_model_ops);
+struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev,
+ const char *name);
int kbase_ipa_init(struct kbase_device *kbdev);
void kbase_ipa_term(struct kbase_device *kbdev);
void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev);
void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev);
int kbase_ipa_model_recalculate(struct kbase_ipa_model *model);
+struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev,
+ struct kbase_ipa_model_ops *ops);
+void kbase_ipa_term_model(struct kbase_ipa_model *model);
extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops;
-/**
- * kbase_ipa_dynamic_power() - Calculate dynamic power component
- * @kbdev: Pointer to kbase device.
- * @freq: Frequency, in Hz.
- * @voltage: Voltage, in mV.
- *
- * Return: Dynamic power consumption, in mW.
- */
-unsigned long kbase_ipa_dynamic_power(struct kbase_device *kbdev,
- unsigned long freq,
- unsigned long voltage);
-
-/**
- * kbase_ipa_static_power() - Calculate static power component
- * @kbdev: Pointer to kbase device.
- * @voltage: Voltage, in mV.
- *
- * Return: Static power consumption, in mW.
- */
-unsigned long kbase_ipa_static_power(struct kbase_device *kbdev,
- unsigned long voltage);
-
-/**
- * kbase_power_to_state() - Find the OPP which consumes target_power
- * @df: Pointer to devfreq device.
- * @target_power: Maximum power consumption, in mW.
- *
- * Return: The index of the most performant OPP whose power consumption is less
- * than target_power.
- */
-unsigned long kbase_power_to_state(struct devfreq *df, u32 target_power);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
-extern struct devfreq_cooling_ops power_model_ops;
+extern struct devfreq_cooling_ops kbase_ipa_power_model_ops;
#else
-extern struct devfreq_cooling_power power_model_ops;
+extern struct devfreq_cooling_power kbase_ipa_power_model_ops;
#endif
+#else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */
+
+static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev)
+{ }
+
+static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev)
+{ }
+
+#endif /* (defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */
+
#endif
diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c
index 78bdcef..eafc140 100644
--- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c
+++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c
@@ -31,7 +31,6 @@ struct kbase_ipa_model_param {
char *name;
union {
void *voidp;
- u32 *u32p;
s32 *s32p;
char *str;
} addr;
@@ -46,10 +45,7 @@ static int param_int_get(void *data, u64 *val)
struct kbase_ipa_model_param *param = data;
mutex_lock(&param->model->kbdev->ipa.lock);
- if (param->type == PARAM_TYPE_S32)
- *(s64 *) val = *param->addr.s32p;
- else
- *val = *param->addr.u32p;
+ *(s64 *) val = *param->addr.s32p;
mutex_unlock(&param->model->kbdev->ipa.lock);
return 0;
@@ -62,28 +58,17 @@ static int param_int_set(void *data, u64 val)
s64 sval = (s64) val;
int err = 0;
- switch (param->type) {
- case PARAM_TYPE_U32:
- if (sval < 0 || val > U32_MAX)
- return -ERANGE;
- break;
- case PARAM_TYPE_S32:
- if (sval < S32_MIN || sval > S32_MAX)
- return -ERANGE;
- break;
- default:
- return -EINVAL;
- }
+ if (sval < S32_MIN || sval > S32_MAX)
+ return -ERANGE;
mutex_lock(&param->model->kbdev->ipa.lock);
- *param->addr.u32p = val;
+ *param->addr.s32p = val;
err = kbase_ipa_model_recalculate(model);
mutex_unlock(&param->model->kbdev->ipa.lock);
return err;
}
-DEFINE_DEBUGFS_ATTRIBUTE(fops_u32, param_int_get, param_int_set, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n");
static ssize_t param_string_get(struct file *file, char __user *user_buf,
@@ -206,9 +191,6 @@ static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model)
case PARAM_TYPE_S32:
fops = &fops_s32;
break;
- case PARAM_TYPE_U32:
- fops = &fops_u32;
- break;
case PARAM_TYPE_STRING:
fops = &fops_string;
break;
diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h
index d7fc908..ec06e20 100644
--- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h
+++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h
@@ -20,7 +20,6 @@
enum kbase_ipa_model_param_type {
PARAM_TYPE_S32 = 1,
- PARAM_TYPE_U32,
PARAM_TYPE_STRING,
};
diff --git a/mali_kbase/ipa/mali_kbase_ipa_simple.c b/mali_kbase/ipa/mali_kbase_ipa_simple.c
index f8d46e1..deca1bc 100644
--- a/mali_kbase/ipa/mali_kbase_ipa_simple.c
+++ b/mali_kbase/ipa/mali_kbase_ipa_simple.c
@@ -85,7 +85,7 @@ static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t)
return clamp(res_unclamped, (s64) 0, (s64) 10000000);
}
-static unsigned long model_static_power(struct kbase_ipa_model *model)
+static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)
unsigned long temp;
@@ -113,15 +113,21 @@ static unsigned long model_static_power(struct kbase_ipa_model *model)
temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts,
temp);
- return (model_data->static_coefficient * temp_scaling_factor) / 1000000;
+ *coeffp = model_data->static_coefficient * temp_scaling_factor;
+ *coeffp /= 1000000;
+
+ return 0;
}
-static unsigned long model_dynamic_power(struct kbase_ipa_model *model)
+static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp,
+ u32 current_freq)
{
struct kbase_ipa_model_simple_data *model_data =
(struct kbase_ipa_model_simple_data *) model->model_data;
- return model_data->dynamic_coefficient;
+ *coeffp = model_data->dynamic_coefficient;
+
+ return 0;
}
static int add_params(struct kbase_ipa_model *model)
@@ -130,24 +136,26 @@ static int add_params(struct kbase_ipa_model *model)
struct kbase_ipa_model_simple_data *model_data =
(struct kbase_ipa_model_simple_data *)model->model_data;
- err = kbase_ipa_model_add_param_u32(model, "static-coefficient",
- &model_data->static_coefficient);
+ err = kbase_ipa_model_add_param_s32(model, "static-coefficient",
+ &model_data->static_coefficient,
+ 1, true);
if (err)
goto end;
- err = kbase_ipa_model_add_param_u32(model, "dynamic-coefficient",
- &model_data->dynamic_coefficient);
+ err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient",
+ &model_data->dynamic_coefficient,
+ 1, true);
if (err)
goto end;
- err = kbase_ipa_model_add_param_s32_array(model, "ts",
- model_data->ts, 4);
+ err = kbase_ipa_model_add_param_s32(model, "ts",
+ model_data->ts, 4, true);
if (err)
goto end;
err = kbase_ipa_model_add_param_string(model, "thermal-zone",
model_data->tz_name,
- sizeof(model_data->tz_name));
+ sizeof(model_data->tz_name), true);
end:
return err;
@@ -175,13 +183,18 @@ static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model)
struct kbase_ipa_model_simple_data *model_data =
(struct kbase_ipa_model_simple_data *)model->model_data;
- model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name);
- if (IS_ERR(model_data->gpu_tz)) {
- pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n",
- PTR_ERR(model_data->gpu_tz),
- model_data->tz_name);
+ if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) {
model_data->gpu_tz = NULL;
- return -EPROBE_DEFER;
+ } else {
+ model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name);
+
+ if (IS_ERR(model_data->gpu_tz)) {
+ pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n",
+ PTR_ERR(model_data->gpu_tz),
+ model_data->tz_name);
+ model_data->gpu_tz = NULL;
+ return -EPROBE_DEFER;
+ }
}
return 0;
@@ -200,6 +213,7 @@ struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = {
.init = &kbase_simple_power_model_init,
.recalculate = &kbase_simple_power_model_recalculate,
.term = &kbase_simple_power_model_term,
- .get_dynamic_power = &model_dynamic_power,
- .get_static_power = &model_static_power,
+ .get_dynamic_coeff = &model_dynamic_coeff,
+ .get_static_coeff = &model_static_coeff,
+ .do_utilization_scaling_in_framework = true,
};
diff --git a/mali_kbase/mali_base_hwconfig_features.h b/mali_kbase/mali_base_hwconfig_features.h
index 8ee19fc..6be0a33 100644
--- a/mali_kbase/mali_base_hwconfig_features.h
+++ b/mali_kbase/mali_base_hwconfig_features.h
@@ -51,6 +51,7 @@ enum base_hw_feature {
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
+ BASE_HW_FEATURE_AARCH64_MMU,
BASE_HW_FEATURE_END
};
@@ -247,4 +248,64 @@ static const enum base_hw_feature base_hw_features_tSIx[] = {
};
+#ifdef MALI_INCLUDE_TKAX
+static const enum base_hw_feature base_hw_features_tKAx[] = {
+ BASE_HW_FEATURE_33BIT_VA,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
+ BASE_HW_FEATURE_XAFFINITY,
+ BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+ BASE_HW_FEATURE_BRNDOUT_CC,
+ BASE_HW_FEATURE_BRNDOUT_KILL,
+ BASE_HW_FEATURE_LD_ST_LEA_TEX,
+ BASE_HW_FEATURE_LD_ST_TILEBUFFER,
+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
+ BASE_HW_FEATURE_MRT,
+ BASE_HW_FEATURE_MSAA_16X,
+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
+ BASE_HW_FEATURE_T7XX_PAIRING_RULES,
+ BASE_HW_FEATURE_TEST4_DATUM_MODE,
+ BASE_HW_FEATURE_FLUSH_REDUCTION,
+ BASE_HW_FEATURE_PROTECTED_MODE,
+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
+ BASE_HW_FEATURE_COHERENCY_REG,
+ BASE_HW_FEATURE_END
+};
+
+#endif /* MALI_INCLUDE_TKAX */
+
+#ifdef MALI_INCLUDE_TTRX
+static const enum base_hw_feature base_hw_features_tTRx[] = {
+ BASE_HW_FEATURE_33BIT_VA,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
+ BASE_HW_FEATURE_XAFFINITY,
+ BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+ BASE_HW_FEATURE_BRNDOUT_CC,
+ BASE_HW_FEATURE_BRNDOUT_KILL,
+ BASE_HW_FEATURE_LD_ST_LEA_TEX,
+ BASE_HW_FEATURE_LD_ST_TILEBUFFER,
+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
+ BASE_HW_FEATURE_MRT,
+ BASE_HW_FEATURE_MSAA_16X,
+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
+ BASE_HW_FEATURE_T7XX_PAIRING_RULES,
+ BASE_HW_FEATURE_TEST4_DATUM_MODE,
+ BASE_HW_FEATURE_FLUSH_REDUCTION,
+ BASE_HW_FEATURE_PROTECTED_MODE,
+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
+ BASE_HW_FEATURE_COHERENCY_REG,
+ BASE_HW_FEATURE_END
+};
+
+#endif /* MALI_INCLUDE_TTRX */
+
#endif /* _BASE_HWCONFIG_FEATURES_H_ */
diff --git a/mali_kbase/mali_base_hwconfig_issues.h b/mali_kbase/mali_base_hwconfig_issues.h
index 53f3804..6d7e5c5 100644
--- a/mali_kbase/mali_base_hwconfig_issues.h
+++ b/mali_kbase/mali_base_hwconfig_issues.h
@@ -956,9 +956,11 @@ static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = {
BASE_HW_ISSUE_TMIX_8042,
BASE_HW_ISSUE_TMIX_8133,
BASE_HW_ISSUE_TMIX_8138,
+ BASE_HW_ISSUE_TMIX_8206,
BASE_HW_ISSUE_TMIX_8343,
BASE_HW_ISSUE_TMIX_8463,
BASE_HW_ISSUE_TMIX_8456,
+ BASE_HW_ISSUE_TMIX_8438,
BASE_HW_ISSUE_END
};
@@ -975,6 +977,7 @@ static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = {
BASE_HW_ISSUE_TMIX_8343,
BASE_HW_ISSUE_TMIX_8463,
BASE_HW_ISSUE_TMIX_8456,
+ BASE_HW_ISSUE_TMIX_8438,
BASE_HW_ISSUE_END
};
@@ -1001,6 +1004,15 @@ static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = {
BASE_HW_ISSUE_END
};
+static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = {
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10682,
+ BASE_HW_ISSUE_TMIX_7891,
+ BASE_HW_ISSUE_TMIX_8042,
+ BASE_HW_ISSUE_TMIX_8133,
+ BASE_HW_ISSUE_END
+};
+
static const enum base_hw_issue base_hw_issues_model_tHEx[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9435,
@@ -1041,4 +1053,46 @@ static const enum base_hw_issue base_hw_issues_model_tSIx[] = {
+#ifdef MALI_INCLUDE_TKAX
+static const enum base_hw_issue base_hw_issues_tKAx_r0p0[] = {
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TMIX_8133,
+ BASE_HW_ISSUE_TSIX_1116,
+ BASE_HW_ISSUE_END
+};
+
+#endif /* MALI_INCLUDE_TKAX */
+
+#ifdef MALI_INCLUDE_TKAX
+static const enum base_hw_issue base_hw_issues_model_tKAx[] = {
+ BASE_HW_ISSUE_5736,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TMIX_8133,
+ BASE_HW_ISSUE_TSIX_1116,
+ BASE_HW_ISSUE_END
+};
+
+#endif /* MALI_INCLUDE_TKAX */
+
+#ifdef MALI_INCLUDE_TTRX
+static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = {
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TMIX_8133,
+ BASE_HW_ISSUE_TSIX_1116,
+ BASE_HW_ISSUE_END
+};
+
+#endif /* MALI_INCLUDE_TTRX */
+
+#ifdef MALI_INCLUDE_TTRX
+static const enum base_hw_issue base_hw_issues_model_tTRx[] = {
+ BASE_HW_ISSUE_5736,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TMIX_8133,
+ BASE_HW_ISSUE_TSIX_1116,
+ BASE_HW_ISSUE_END
+};
+
+#endif /* MALI_INCLUDE_TTRX */
+
#endif /* _BASE_HWCONFIG_ISSUES_H_ */
diff --git a/mali_kbase/mali_base_kernel.h b/mali_kbase/mali_base_kernel.h
index 0c80e03..d5c8cbc 100644
--- a/mali_kbase/mali_base_kernel.h
+++ b/mali_kbase/mali_base_kernel.h
@@ -118,33 +118,44 @@ typedef union kbase_pointer {
*/
/**
- * @brief Memory allocation, access/hint flags
+ * typedef base_mem_alloc_flags - Memory allocation, access/hint flags.
*
* A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator
* in order to determine the best cache policy. Some combinations are
- * of course invalid (eg @c MEM_PROT_CPU_WR | @c MEM_HINT_CPU_RD),
- * which defines a @a write-only region on the CPU side, which is
+ * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD),
+ * which defines a write-only region on the CPU side, which is
* heavily read by the CPU...
* Other flags are only meaningful to a particular allocator.
* More flags can be added to this list, as long as they don't clash
- * (see ::BASE_MEM_FLAGS_NR_BITS for the number of the first free bit).
+ * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit).
*/
typedef u32 base_mem_alloc_flags;
-/**
- * @brief Memory allocation, access/hint flags
- *
- * See ::base_mem_alloc_flags.
+/* Memory allocation, access/hint flags.
*
+ * See base_mem_alloc_flags.
*/
-enum {
+
/* IN */
- BASE_MEM_PROT_CPU_RD = (1U << 0), /**< Read access CPU side */
- BASE_MEM_PROT_CPU_WR = (1U << 1), /**< Write access CPU side */
- BASE_MEM_PROT_GPU_RD = (1U << 2), /**< Read access GPU side */
- BASE_MEM_PROT_GPU_WR = (1U << 3), /**< Write access GPU side */
- BASE_MEM_PROT_GPU_EX = (1U << 4), /**< Execute allowed on the GPU
- side */
+/* Read access CPU side
+ */
+#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0)
+
+/* Write access CPU side
+ */
+#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1)
+
+/* Read access GPU side
+ */
+#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2)
+
+/* Write access GPU side
+ */
+#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3)
+
+/* Execute allowed on the GPU side
+ */
+#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4)
/* BASE_MEM_HINT flags have been removed, but their values are reserved
* for backwards compatibility with older user-space drivers. The values
@@ -157,54 +168,66 @@ enum {
* RESERVED: (1U << 8)
*/
- BASE_MEM_GROW_ON_GPF = (1U << 9), /**< Grow backing store on GPU
- Page Fault */
+/* Grow backing store on GPU Page Fault
+ */
+#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9)
- BASE_MEM_COHERENT_SYSTEM = (1U << 10), /**< Page coherence Outer
- shareable, if available */
- BASE_MEM_COHERENT_LOCAL = (1U << 11), /**< Page coherence Inner
- shareable */
- BASE_MEM_CACHED_CPU = (1U << 12), /**< Should be cached on the
- CPU */
+/* Page coherence Outer shareable, if available
+ */
+#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10)
+
+/* Page coherence Inner shareable
+ */
+#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11)
+
+/* Should be cached on the CPU
+ */
+#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12)
/* IN/OUT */
- BASE_MEM_SAME_VA = (1U << 13), /**< Must have same VA on both the GPU
- and the CPU */
+/* Must have same VA on both the GPU and the CPU
+ */
+#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13)
+
/* OUT */
- BASE_MEM_NEED_MMAP = (1U << 14), /**< Must call mmap to acquire a GPU
- address for the alloc */
+/* Must call mmap to acquire a GPU address for the alloc
+ */
+#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14)
+
/* IN */
- BASE_MEM_COHERENT_SYSTEM_REQUIRED = (1U << 15), /**< Page coherence
- Outer shareable, required. */
- BASE_MEM_SECURE = (1U << 16), /**< Secure memory */
- BASE_MEM_DONT_NEED = (1U << 17), /**< Not needed physical
- memory */
- BASE_MEM_IMPORT_SHARED = (1U << 18), /**< Must use shared CPU/GPU zone
- (SAME_VA zone) but doesn't
- require the addresses to
- be the same */
-};
+/* Page coherence Outer shareable, required.
+ */
+#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15)
-/**
- * @brief Number of bits used as flags for base memory management
+/* Secure memory
+ */
+#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16)
+
+/* Not needed physical memory
+ */
+#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17)
+
+/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the
+ * addresses to be the same
+ */
+#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18)
+
+/* Number of bits used as flags for base memory management
*
- * Must be kept in sync with the ::base_mem_alloc_flags flags
+ * Must be kept in sync with the base_mem_alloc_flags flags
*/
#define BASE_MEM_FLAGS_NR_BITS 19
-/**
- * A mask for all output bits, excluding IN/OUT bits.
- */
+/* A mask for all output bits, excluding IN/OUT bits.
+ */
#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP
-/**
- * A mask for all input bits, including IN/OUT bits.
- */
+/* A mask for all input bits, including IN/OUT bits.
+ */
#define BASE_MEM_FLAGS_INPUT_MASK \
(((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK)
-/**
- * A mask for all the flags which are modifiable via the base_mem_set_flags
+/* A mask for all the flags which are modifiable via the base_mem_set_flags
* interface.
*/
#define BASE_MEM_FLAGS_MODIFIABLE \
@@ -294,7 +317,6 @@ struct base_mem_import_user_buffer {
*/
typedef enum base_backing_threshold_status {
BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */
- BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE = -1, /**< Not a growable tmem object */
BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */
BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */
} base_backing_threshold_status;
@@ -1429,9 +1451,9 @@ struct mali_base_gpu_core_props {
/**
* Status of the GPU release.
- * No defined values, but starts at 0 and increases by one for each release
- * status (alpha, beta, EAC, etc.).
- * 4 bit values (0-15).
+ * No defined values, but starts at 0 and increases by one for each
+ * release status (alpha, beta, EAC, etc.).
+ * 4 bit values (0-15).
*/
u16 version_status;
@@ -1450,8 +1472,11 @@ struct mali_base_gpu_core_props {
u16 padding;
/**
- * @usecase GPU clock speed is not specified in the Midgard Architecture, but is
- * <b>necessary for OpenCL's clGetDeviceInfo() function</b>.
+ * This property is deprecated since it has not contained the real current
+ * value of GPU clock speed. It is kept here only for backwards compatibility.
+ * For the new ioctl interface, it is ignored and is treated as a padding
+ * to keep the structure of the same size and retain the placement of its
+ * members.
*/
u32 gpu_speed_mhz;
@@ -1459,6 +1484,7 @@ struct mali_base_gpu_core_props {
* @usecase GPU clock max/min speed is required for computing best/worst case
* in tasks as job scheduling ant irq_throttling. (It is not specified in the
* Midgard Architecture).
+ * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function.
*/
u32 gpu_freq_khz_max;
u32 gpu_freq_khz_min;
diff --git a/mali_kbase/mali_kbase.h b/mali_kbase/mali_kbase.h
index 2289f80..baefe8a 100644
--- a/mali_kbase/mali_kbase.h
+++ b/mali_kbase/mali_kbase.h
@@ -66,9 +66,7 @@
#include "mali_kbase_jm.h"
#include "mali_kbase_vinstr.h"
-#ifdef CONFIG_DEVFREQ_THERMAL
#include "ipa/mali_kbase_ipa.h"
-#endif
#ifdef CONFIG_GPU_TRACEPOINTS
#include <trace/events/gpu.h>
@@ -106,22 +104,27 @@ void kbase_release_device(struct kbase_device *kbdev);
void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value);
-u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control);
-
struct kbase_context *
kbase_create_context(struct kbase_device *kbdev, bool is_compat);
void kbase_destroy_context(struct kbase_context *kctx);
int kbase_jd_init(struct kbase_context *kctx);
void kbase_jd_exit(struct kbase_context *kctx);
-#ifdef BASE_LEGACY_UK6_SUPPORT
-int kbase_jd_submit(struct kbase_context *kctx,
- const struct kbase_uk_job_submit *submit_data,
- int uk6_atom);
-#else
+
+/**
+ * kbase_jd_submit - Submit atoms to the job dispatcher
+ *
+ * @kctx: The kbase context to submit to
+ * @user_addr: The address in user space of the struct base_jd_atom_v2 array
+ * @nr_atoms: The number of atoms in the array
+ * @stride: sizeof(struct base_jd_atom_v2)
+ * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6)
+ *
+ * Return: 0 on success or error code
+ */
int kbase_jd_submit(struct kbase_context *kctx,
- const struct kbase_uk_job_submit *submit_data);
-#endif
+ void __user *user_addr, u32 nr_atoms, u32 stride,
+ bool uk6_atom);
/**
* kbase_jd_done_worker - Handle a job completion
@@ -195,8 +198,10 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom);
void kbase_finish_soft_job(struct kbase_jd_atom *katom);
void kbase_cancel_soft_job(struct kbase_jd_atom *katom);
void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev);
-void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom);
void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom);
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom);
+#endif
int kbase_soft_event_update(struct kbase_context *kctx,
u64 event,
unsigned char new_status);
diff --git a/mali_kbase/mali_kbase_10969_workaround.c b/mali_kbase/mali_kbase_10969_workaround.c
index 933aa28..fde0f8f 100644
--- a/mali_kbase/mali_kbase_10969_workaround.c
+++ b/mali_kbase/mali_kbase_10969_workaround.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2015, 2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_config_defaults.h b/mali_kbase/mali_kbase_config_defaults.h
index f6eeac8..69079e7 100644
--- a/mali_kbase/mali_kbase_config_defaults.h
+++ b/mali_kbase/mali_kbase_config_defaults.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -170,7 +170,7 @@ enum {
* Default timeout for some software jobs, after which the software event wait
* jobs will be cancelled.
*/
-#define DEFAULT_JS_SOFT_JOB_TIMEOUT ((u32)3000) /* 3s */
+#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */
/*
* Default minimum number of scheduling ticks before the GPU is reset to clear a
diff --git a/mali_kbase/mali_kbase_context.c b/mali_kbase/mali_kbase_context.c
index 1401380..4e802bc 100644
--- a/mali_kbase/mali_kbase_context.c
+++ b/mali_kbase/mali_kbase_context.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -24,6 +24,8 @@
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_mem_linux.h>
+#include <mali_kbase_dma_fence.h>
+#include <mali_kbase_ctx_sched.h>
/**
* kbase_create_context() - Create a kernel base context.
@@ -53,6 +55,7 @@ kbase_create_context(struct kbase_device *kbdev, bool is_compat)
kctx->kbdev = kbdev;
kctx->as_nr = KBASEP_AS_NR_INVALID;
+ atomic_set(&kctx->refcount, 0);
if (is_compat)
kbase_ctx_flag_set(kctx, KCTX_COMPAT);
#ifdef CONFIG_MALI_TRACE_TIMELINE
@@ -210,6 +213,7 @@ void kbase_destroy_context(struct kbase_context *kctx)
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
+ unsigned long flags;
KBASE_DEBUG_ASSERT(NULL != kctx);
@@ -277,6 +281,12 @@ void kbase_destroy_context(struct kbase_context *kctx)
kbase_dma_fence_term(kctx);
+ mutex_lock(&kbdev->mmu_hw_mutex);
+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
+ kbase_ctx_sched_remove_ctx(kctx);
+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
+
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
diff --git a/mali_kbase/mali_kbase_core_linux.c b/mali_kbase/mali_kbase_core_linux.c
index e335c0d..c076300 100644
--- a/mali_kbase/mali_kbase_core_linux.c
+++ b/mali_kbase/mali_kbase_core_linux.c
@@ -22,6 +22,7 @@
#include <mali_kbase_gator.h>
#include <mali_kbase_mem_linux.h>
#ifdef CONFIG_MALI_DEVFREQ
+#include <linux/devfreq.h>
#include <backend/gpu/mali_kbase_devfreq.h>
#ifdef CONFIG_DEVFREQ_THERMAL
#include <ipa/mali_kbase_ipa_debugfs.h>
@@ -40,7 +41,9 @@
#include "mali_kbase_regs_history_debugfs.h"
#include <mali_kbase_hwaccess_backend.h>
#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_ctx_sched.h>
#include <backend/gpu/mali_kbase_device_internal.h>
+#include "mali_kbase_ioctl.h"
#ifdef CONFIG_KDS
#include <linux/kds.h>
@@ -55,6 +58,7 @@
#include <linux/errno.h>
#include <linux/of.h>
#include <linux/platform_device.h>
+#include <linux/of_platform.h>
#include <linux/miscdevice.h>
#include <linux/list.h>
#include <linux/semaphore.h>
@@ -73,12 +77,9 @@
#ifdef CONFIG_MALI_PLATFORM_FAKE
#include <platform/mali_kbase_platform_fake.h>
#endif /*CONFIG_MALI_PLATFORM_FAKE */
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
#include <mali_kbase_sync.h>
-#endif /* CONFIG_SYNC */
-#ifdef CONFIG_PM_DEVFREQ
-#include <linux/devfreq.h>
-#endif /* CONFIG_PM_DEVFREQ */
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
#include <linux/clk.h>
#include <linux/delay.h>
@@ -116,60 +117,8 @@ static inline void __compile_time_asserts(void)
CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE);
}
-static void kbase_create_timeline_objects(struct kbase_context *kctx)
-{
- struct kbase_device *kbdev = kctx->kbdev;
- unsigned int lpu_id;
- unsigned int as_nr;
- struct kbasep_kctx_list_element *element;
-
- /* Create LPU objects. */
- for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
- u32 *lpu =
- &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
- KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu);
- }
-
- /* Create Address Space objects. */
- for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++)
- KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr);
-
- /* Create GPU object and make it retain all LPUs and address spaces. */
- KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(
- kbdev,
- kbdev->gpu_props.props.raw_props.gpu_id,
- kbdev->gpu_props.num_cores);
-
- for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
- void *lpu =
- &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
- KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev);
- }
- for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++)
- KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(
- &kbdev->as[as_nr],
- kbdev);
-
- /* Create object for each known context. */
- mutex_lock(&kbdev->kctx_list_lock);
- list_for_each_entry(element, &kbdev->kctx_list, link) {
- KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(
- element->kctx,
- (u32)(element->kctx->id),
- (u32)(element->kctx->tgid));
- }
- /* Before releasing the lock, reset body stream buffers.
- * This will prevent context creation message to be directed to both
- * summary and body stream. */
- kbase_tlstream_reset_body_streams();
- mutex_unlock(&kbdev->kctx_list_lock);
- /* Static object are placed into summary packet that needs to be
- * transmitted first. Flush all streams to make it available to
- * user space. */
- kbase_tlstream_flush_streams();
-}
-
-static void kbase_api_handshake(struct uku_version_check_args *version)
+static int kbase_api_handshake(struct kbase_context *kctx,
+ struct kbase_ioctl_version_check *version)
{
switch (version->major) {
#ifdef BASE_LEGACY_UK6_SUPPORT
@@ -218,6 +167,11 @@ static void kbase_api_handshake(struct uku_version_check_args *version)
version->minor = BASE_UK_VERSION_MINOR;
break;
}
+
+ /* save the proposed version number for later use */
+ kctx->api_version = KBASE_API_VERSION(version->major, version->minor);
+
+ return 0;
}
/**
@@ -262,7 +216,9 @@ enum {
inited_registers_map = (1u << 17),
inited_io_history = (1u << 18),
inited_power_control = (1u << 19),
- inited_buslogger = (1u << 20)
+ inited_buslogger = (1u << 20),
+ inited_protected = (1u << 21),
+ inited_ctx_sched = (1u << 22)
};
@@ -281,7 +237,18 @@ void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive)
KBASE_EXPORT_TEST_API(kbase_set_driver_inactive);
#endif /* CONFIG_MALI_DEBUG */
-static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 args_size)
+/**
+ * kbase_legacy_dispatch - UKK dispatch function
+ *
+ * This is the dispatch function for the legacy UKK ioctl interface. No new
+ * ioctls should be added to this function, see kbase_ioctl instead.
+ *
+ * @kctx: The kernel context structure
+ * @args: Pointer to the data structure passed from/to user space
+ * @args_size: Size of the data structure
+ */
+static int kbase_legacy_dispatch(struct kbase_context *kctx,
+ void * const args, u32 args_size)
{
struct kbase_device *kbdev;
union uk_header *ukh = args;
@@ -301,16 +268,20 @@ static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 arg
if (UKP_FUNC_ID_CHECK_VERSION == id) {
struct uku_version_check_args *version_check;
+ struct kbase_ioctl_version_check version;
if (args_size != sizeof(struct uku_version_check_args)) {
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
return 0;
}
version_check = (struct uku_version_check_args *)args;
- kbase_api_handshake(version_check);
- /* save the proposed version number for later use */
- kctx->api_version = KBASE_API_VERSION(version_check->major,
- version_check->minor);
+ version.minor = version_check->minor;
+ version.major = version_check->major;
+
+ kbase_api_handshake(kctx, &version);
+
+ version_check->minor = version.minor;
+ version_check->major = version.major;
ukh->ret = MALI_ERROR_NONE;
return 0;
}
@@ -409,6 +380,7 @@ static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 arg
(enum base_mem_import_type)
mem_import->type,
phandle,
+ 0,
&mem_import->gpu_va,
&mem_import->va_pages,
&mem_import->flags)) {
@@ -470,21 +442,26 @@ copy_failed:
case KBASE_FUNC_MEM_COMMIT:
{
struct kbase_uk_mem_commit *commit = args;
+ int ret;
if (sizeof(*commit) != args_size)
goto bad_size;
- if (commit->gpu_addr & ~PAGE_MASK) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_COMMIT: commit->gpu_addr: passed parameter is invalid");
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
+ ret = kbase_mem_commit(kctx, commit->gpu_addr,
+ commit->pages);
- if (kbase_mem_commit(kctx, commit->gpu_addr,
- commit->pages,
- (base_backing_threshold_status *)
- &commit->result_subcode) != 0)
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ commit->result_subcode =
+ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
+
+ if (ret == 0) {
+ ukh->ret = MALI_ERROR_NONE;
+ commit->result_subcode =
+ BASE_BACKING_THRESHOLD_OK;
+ } else if (ret == -ENOMEM) {
+ commit->result_subcode =
+ BASE_BACKING_THRESHOLD_ERROR_OOM;
+ }
break;
}
@@ -496,19 +473,6 @@ copy_failed:
if (sizeof(*query) != args_size)
goto bad_size;
- if (query->gpu_addr & ~PAGE_MASK) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_QUERY: query->gpu_addr: passed parameter is invalid");
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
- if (query->query != KBASE_MEM_QUERY_COMMIT_SIZE &&
- query->query != KBASE_MEM_QUERY_VA_SIZE &&
- query->query != KBASE_MEM_QUERY_FLAGS) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_QUERY: query->query = %lld unknown", (unsigned long long)query->query);
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
-
if (kbase_mem_query(kctx, query->gpu_addr,
query->query, &query->value) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
@@ -525,12 +489,6 @@ copy_failed:
if (sizeof(*fc) != args_size)
goto bad_size;
- if ((fc->gpu_va & ~PAGE_MASK) && (fc->gpu_va >= PAGE_SIZE)) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_FLAGS_CHANGE: mem->gpu_va: passed parameter is invalid");
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
-
if (kbase_mem_flags_change(kctx, fc->gpu_va,
fc->flags, fc->mask) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
@@ -544,12 +502,6 @@ copy_failed:
if (sizeof(*mem) != args_size)
goto bad_size;
- if ((mem->gpu_addr & ~PAGE_MASK) && (mem->gpu_addr >= PAGE_SIZE)) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_MEM_FREE: mem->gpu_addr: passed parameter is invalid");
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
-
if (kbase_mem_free(kctx, mem->gpu_addr) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
@@ -562,11 +514,10 @@ copy_failed:
if (sizeof(*job) != args_size)
goto bad_size;
-#ifdef BASE_LEGACY_UK6_SUPPORT
- if (kbase_jd_submit(kctx, job, 0) != 0)
-#else
- if (kbase_jd_submit(kctx, job) != 0)
-#endif /* BASE_LEGACY_UK6_SUPPORT */
+ if (kbase_jd_submit(kctx, job->addr.value,
+ job->nr_atoms,
+ job->stride,
+ false) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
@@ -579,7 +530,10 @@ copy_failed:
if (sizeof(*job) != args_size)
goto bad_size;
- if (kbase_jd_submit(kctx, job, 1) != 0)
+ if (kbase_jd_submit(kctx, job->addr.value,
+ job->nr_atoms,
+ job->stride,
+ true) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
@@ -592,14 +546,8 @@ copy_failed:
if (sizeof(*sn) != args_size)
goto bad_size;
- if (sn->sset.basep_sset.mem_handle.basep.handle & ~PAGE_MASK) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_SYNC: sn->sset.basep_sset.mem_handle: passed parameter is invalid");
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- break;
- }
-
#ifndef CONFIG_MALI_COH_USER
- if (kbase_sync_now(kctx, &sn->sset) != 0)
+ if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
#endif
break;
@@ -693,7 +641,7 @@ copy_failed:
goto bad_size;
if (find->gpu_addr & ~PAGE_MASK) {
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid");
+ dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid");
goto out_bad;
}
@@ -728,7 +676,7 @@ copy_failed:
case KBASE_FUNC_STREAM_CREATE:
{
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args;
if (sizeof(*screate) != args_size)
@@ -740,28 +688,29 @@ copy_failed:
break;
}
- if (kbase_stream_create(screate->name, &screate->fd) != 0)
+ if (kbase_sync_fence_stream_create(screate->name,
+ &screate->fd) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
else
ukh->ret = MALI_ERROR_NONE;
-#else /* CONFIG_SYNC */
+#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
-#endif /* CONFIG_SYNC */
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
break;
}
case KBASE_FUNC_FENCE_VALIDATE:
{
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args;
if (sizeof(*fence_validate) != args_size)
goto bad_size;
- if (kbase_fence_validate(fence_validate->fd) != 0)
+ if (kbase_sync_fence_validate(fence_validate->fd) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
else
ukh->ret = MALI_ERROR_NONE;
-#endif /* CONFIG_SYNC */
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
break;
}
@@ -818,7 +767,7 @@ copy_failed:
#ifdef BASE_LEGACY_UK8_SUPPORT
case KBASE_FUNC_KEEP_GPU_POWERED:
{
- dev_warn(kbdev->dev, "kbase_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n");
+ dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n");
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
@@ -834,7 +783,8 @@ copy_failed:
goto bad_size;
for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++)
- controls->profiling_controls[i] = kbase_get_profiling_control(kbdev, i);
+ controls->profiling_controls[i] =
+ kbdev->kbase_profiling_controls[i];
break;
}
@@ -890,7 +840,6 @@ copy_failed:
if (kbasep_mem_profile_debugfs_insert(kctx, buf,
add_data->len)) {
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- kfree(buf);
goto out_bad;
}
@@ -914,20 +863,17 @@ copy_failed:
{
struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire
= args;
+ int ret;
if (sizeof(*tlstream_acquire) != args_size)
goto bad_size;
- if (0 != kbase_tlstream_acquire(
- kctx,
- &tlstream_acquire->fd, 0)) {
+ ret = kbase_tlstream_acquire(
+ kctx, 0);
+ if (ret < 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- } else if (0 <= tlstream_acquire->fd) {
- /* Summary stream was cleared during acquire.
- * Create static timeline objects that will be
- * read by client. */
- kbase_create_timeline_objects(kctx);
- }
+ else
+ tlstream_acquire->fd = ret;
break;
}
#endif /* BASE_LEGACY_UK10_4_SUPPORT */
@@ -935,6 +881,7 @@ copy_failed:
{
struct kbase_uk_tlstream_acquire *tlstream_acquire =
args;
+ int ret;
if (sizeof(*tlstream_acquire) != args_size)
goto bad_size;
@@ -942,17 +889,12 @@ copy_failed:
if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK)
goto out_bad;
- if (0 != kbase_tlstream_acquire(
- kctx,
- &tlstream_acquire->fd,
- tlstream_acquire->flags)) {
+ ret = kbase_tlstream_acquire(
+ kctx, tlstream_acquire->flags);
+ if (ret < 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- } else if (0 <= tlstream_acquire->fd) {
- /* Summary stream was cleared during acquire.
- * Create static timeline objects that will be
- * read by client. */
- kbase_create_timeline_objects(kctx);
- }
+ else
+ tlstream_acquire->fd = ret;
break;
}
case KBASE_FUNC_TLSTREAM_FLUSH:
@@ -1128,10 +1070,12 @@ void kbase_release_device(struct kbase_device *kbdev)
}
EXPORT_SYMBOL(kbase_release_device);
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \
+ !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \
+ LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))
/*
* Older versions, before v4.6, of the kernel doesn't have
- * kstrtobool_from_user().
+ * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28
*/
static int kstrtobool_from_user(const char __user *s, size_t count, bool *res)
{
@@ -1320,7 +1264,8 @@ static int kbase_release(struct inode *inode, struct file *filp)
#define CALL_MAX_SIZE 536
-static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg)
{
u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */
u32 size = _IOC_SIZE(cmd);
@@ -1334,7 +1279,7 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
return -EFAULT;
}
- if (kbase_dispatch(kctx, &msg, size) != 0)
+ if (kbase_legacy_dispatch(kctx, &msg, size) != 0)
return -EFAULT;
if (0 != copy_to_user((void __user *)arg, &msg, size)) {
@@ -1344,6 +1289,600 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
return 0;
}
+static int kbase_api_set_flags(struct kbase_context *kctx,
+ struct kbase_ioctl_set_flags *flags)
+{
+ int err;
+
+ /* setup pending, try to signal that we'll do the setup,
+ * if setup was already in progress, err this call
+ */
+ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0)
+ return -EINVAL;
+
+ err = kbase_context_set_create_flags(kctx, flags->create_flags);
+ /* if bad flags, will stay stuck in setup mode */
+ if (err)
+ return err;
+
+ atomic_set(&kctx->setup_complete, 1);
+ return 0;
+}
+
+static int kbase_api_job_submit(struct kbase_context *kctx,
+ struct kbase_ioctl_job_submit *submit)
+{
+ return kbase_jd_submit(kctx, submit->addr.value, submit->nr_atoms,
+ submit->stride, false);
+}
+
+static int kbase_api_get_gpuprops(struct kbase_context *kctx,
+ struct kbase_ioctl_get_gpuprops *get_props)
+{
+ struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props;
+ int err;
+
+ if (get_props->flags != 0) {
+ dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops");
+ return -EINVAL;
+ }
+
+ if (get_props->size == 0)
+ return kprops->prop_buffer_size;
+ if (get_props->size < kprops->prop_buffer_size)
+ return -EINVAL;
+
+ err = copy_to_user(get_props->buffer.value, kprops->prop_buffer,
+ kprops->prop_buffer_size);
+ if (err)
+ return err;
+ return kprops->prop_buffer_size;
+}
+
+static int kbase_api_post_term(struct kbase_context *kctx)
+{
+ kbase_event_close(kctx);
+ return 0;
+}
+
+static int kbase_api_mem_alloc(struct kbase_context *kctx,
+ union kbase_ioctl_mem_alloc *alloc)
+{
+ struct kbase_va_region *reg;
+ u64 flags = alloc->in.flags;
+ u64 gpu_va;
+
+#if defined(CONFIG_64BIT)
+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
+ /* force SAME_VA if a 64-bit client */
+ flags |= BASE_MEM_SAME_VA;
+ }
+#endif
+
+ reg = kbase_mem_alloc(kctx, alloc->in.va_pages,
+ alloc->in.commit_pages,
+ alloc->in.extent,
+ &flags, &gpu_va);
+
+ if (!reg)
+ return -ENOMEM;
+
+ alloc->out.flags = flags;
+ alloc->out.gpu_va = gpu_va;
+
+ return 0;
+}
+
+static int kbase_api_mem_query(struct kbase_context *kctx,
+ union kbase_ioctl_mem_query *query)
+{
+ return kbase_mem_query(kctx, query->in.gpu_addr,
+ query->in.query, &query->out.value);
+}
+
+static int kbase_api_mem_free(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_free *free)
+{
+ return kbase_mem_free(kctx, free->gpu_addr);
+}
+
+static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx,
+ struct kbase_ioctl_hwcnt_reader_setup *setup)
+{
+ int ret;
+ struct kbase_uk_hwcnt_reader_setup args = {
+ .buffer_count = setup->buffer_count,
+ .jm_bm = setup->jm_bm,
+ .shader_bm = setup->shader_bm,
+ .tiler_bm = setup->tiler_bm,
+ .mmu_l2_bm = setup->mmu_l2_bm
+ };
+
+ mutex_lock(&kctx->vinstr_cli_lock);
+ ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args);
+ mutex_unlock(&kctx->vinstr_cli_lock);
+
+ if (ret)
+ return ret;
+ return args.fd;
+}
+
+static int kbase_api_hwcnt_enable(struct kbase_context *kctx,
+ struct kbase_ioctl_hwcnt_enable *enable)
+{
+ int ret;
+ struct kbase_uk_hwcnt_setup args = {
+ .dump_buffer = enable->dump_buffer,
+ .jm_bm = enable->jm_bm,
+ .shader_bm = enable->shader_bm,
+ .tiler_bm = enable->tiler_bm,
+ .mmu_l2_bm = enable->mmu_l2_bm
+ };
+
+ mutex_lock(&kctx->vinstr_cli_lock);
+ ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx,
+ &kctx->vinstr_cli, &args);
+ mutex_unlock(&kctx->vinstr_cli_lock);
+
+ return ret;
+}
+
+static int kbase_api_hwcnt_dump(struct kbase_context *kctx)
+{
+ int ret;
+
+ mutex_lock(&kctx->vinstr_cli_lock);
+ ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli,
+ BASE_HWCNT_READER_EVENT_MANUAL);
+ mutex_unlock(&kctx->vinstr_cli_lock);
+
+ return ret;
+}
+
+static int kbase_api_hwcnt_clear(struct kbase_context *kctx)
+{
+ int ret;
+
+ mutex_lock(&kctx->vinstr_cli_lock);
+ ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli);
+ mutex_unlock(&kctx->vinstr_cli_lock);
+
+ return ret;
+}
+
+static int kbase_api_disjoint_query(struct kbase_context *kctx,
+ struct kbase_ioctl_disjoint_query *query)
+{
+ query->counter = kbase_disjoint_event_get(kctx->kbdev);
+
+ return 0;
+}
+
+static int kbase_api_get_ddk_version(struct kbase_context *kctx,
+ struct kbase_ioctl_get_ddk_version *version)
+{
+ int ret;
+ int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING);
+
+ if (version->version_buffer.value == NULL)
+ return len;
+
+ if (version->size < len)
+ return -EOVERFLOW;
+
+ ret = copy_to_user(version->version_buffer.value,
+ KERNEL_SIDE_DDK_VERSION_STRING,
+ sizeof(KERNEL_SIDE_DDK_VERSION_STRING));
+
+ if (ret)
+ return ret;
+
+ return len;
+}
+
+static int kbase_api_mem_jit_init(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_jit_init *jit_init)
+{
+ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages);
+}
+
+static int kbase_api_mem_sync(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_sync *sync)
+{
+#ifdef CONFIG_MALI_COH_USER
+ return 0;
+#endif
+ struct basep_syncset sset = {
+ .mem_handle.basep.handle = sync->handle,
+ .user_addr = sync->user_addr,
+ .size = sync->size,
+ .type = sync->type
+ };
+
+ return kbase_sync_now(kctx, &sset);
+}
+
+static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx,
+ union kbase_ioctl_mem_find_cpu_offset *find)
+{
+ return kbasep_find_enclosing_cpu_mapping_offset(
+ kctx,
+ find->in.cpu_addr,
+ find->in.size,
+ &find->out.offset);
+}
+
+static int kbase_api_get_context_id(struct kbase_context *kctx,
+ struct kbase_ioctl_get_context_id *info)
+{
+ info->id = kctx->id;
+
+ return 0;
+}
+
+static int kbase_api_tlstream_acquire(struct kbase_context *kctx,
+ struct kbase_ioctl_tlstream_acquire *acquire)
+{
+ return kbase_tlstream_acquire(kctx, acquire->flags);
+}
+
+static int kbase_api_tlstream_flush(struct kbase_context *kctx)
+{
+ kbase_tlstream_flush_streams();
+
+ return 0;
+}
+
+static int kbase_api_mem_commit(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_commit *commit)
+{
+ return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages);
+}
+
+static int kbase_api_mem_alias(struct kbase_context *kctx,
+ union kbase_ioctl_mem_alias *alias)
+{
+ struct base_mem_aliasing_info *ai;
+ u64 flags;
+ int err;
+
+ if (alias->in.nents == 0 || alias->in.nents > 2048)
+ return -EINVAL;
+
+ ai = vmalloc(sizeof(*ai) * alias->in.nents);
+ if (!ai)
+ return -ENOMEM;
+
+ err = copy_from_user(ai, alias->in.aliasing_info.value,
+ sizeof(*ai) * alias->in.nents);
+ if (err) {
+ vfree(ai);
+ return err;
+ }
+
+ flags = alias->in.flags;
+
+ alias->out.gpu_va = kbase_mem_alias(kctx, &flags,
+ alias->in.stride, alias->in.nents,
+ ai, &alias->out.va_pages);
+
+ alias->out.flags = flags;
+
+ vfree(ai);
+
+ if (alias->out.gpu_va == 0)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static int kbase_api_mem_import(struct kbase_context *kctx,
+ union kbase_ioctl_mem_import *import)
+{
+ int ret;
+ u64 flags = import->in.flags;
+
+ ret = kbase_mem_import(kctx,
+ import->in.type,
+ import->in.phandle.value,
+ import->in.padding,
+ &import->out.gpu_va,
+ &import->out.va_pages,
+ &flags);
+
+ import->out.flags = flags;
+
+ return ret;
+}
+
+static int kbase_api_mem_flags_change(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_flags_change *change)
+{
+ return kbase_mem_flags_change(kctx, change->gpu_va,
+ change->flags, change->mask);
+}
+
+static int kbase_api_stream_create(struct kbase_context *kctx,
+ struct kbase_ioctl_stream_create *stream)
+{
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+ int fd, ret;
+
+ /* Name must be NULL-terminated and padded with NULLs, so check last
+ * character is NULL
+ */
+ if (stream->name[sizeof(stream->name)-1] != 0)
+ return -EINVAL;
+
+ ret = kbase_sync_fence_stream_create(stream->name, &fd);
+
+ if (ret)
+ return ret;
+ return fd;
+#else
+ return -ENOENT;
+#endif
+}
+
+static int kbase_api_fence_validate(struct kbase_context *kctx,
+ struct kbase_ioctl_fence_validate *validate)
+{
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+ return kbase_sync_fence_validate(validate->fd);
+#else
+ return -ENOENT;
+#endif
+}
+
+static int kbase_api_get_profiling_controls(struct kbase_context *kctx,
+ struct kbase_ioctl_get_profiling_controls *controls)
+{
+ if (controls->count > FBDUMP_CONTROL_MAX)
+ return -EINVAL;
+
+ return copy_to_user(controls->buffer.value,
+ &kctx->kbdev->kbase_profiling_controls[
+ FBDUMP_CONTROL_MIN],
+ controls->count * sizeof(u32));
+}
+
+static int kbase_api_mem_profile_add(struct kbase_context *kctx,
+ struct kbase_ioctl_mem_profile_add *data)
+{
+ char *buf;
+ int err;
+
+ if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) {
+ dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n");
+ return -EINVAL;
+ }
+
+ buf = kmalloc(data->len, GFP_KERNEL);
+ if (ZERO_OR_NULL_PTR(buf))
+ return -ENOMEM;
+
+ err = copy_from_user(buf, data->buffer.value, data->len);
+ if (err) {
+ kfree(buf);
+ return err;
+ }
+
+ return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len);
+}
+
+static int kbase_api_soft_event_update(struct kbase_context *kctx,
+ struct kbase_ioctl_soft_event_update *update)
+{
+ if (update->flags != 0)
+ return -EINVAL;
+
+ return kbase_soft_event_update(kctx, update->event, update->new_status);
+}
+
+#if MALI_UNIT_TEST
+static int kbase_api_tlstream_test(struct kbase_context *kctx,
+ struct kbase_ioctl_tlstream_test *test)
+{
+ kbase_tlstream_test(
+ test->tpw_count,
+ test->msg_delay,
+ test->msg_count,
+ test->aux_msg);
+
+ return 0;
+}
+
+static int kbase_api_tlstream_stats(struct kbase_context *kctx,
+ struct kbase_ioctl_tlstream_stats *stats)
+{
+ kbase_tlstream_stats(
+ &stats->bytes_collected,
+ &stats->bytes_generated);
+
+ return 0;
+}
+#endif /* MALI_UNIT_TEST */
+
+#define KBASE_HANDLE_IOCTL(cmd, function) \
+ case cmd: \
+ do { \
+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \
+ return function(kctx); \
+ } while (0)
+
+#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \
+ case cmd: \
+ do { \
+ type param; \
+ int err; \
+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \
+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \
+ err = copy_from_user(&param, uarg, sizeof(param)); \
+ if (err) \
+ return -EFAULT; \
+ return function(kctx, &param); \
+ } while (0)
+
+#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \
+ case cmd: \
+ do { \
+ type param; \
+ int ret, err; \
+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \
+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \
+ ret = function(kctx, &param); \
+ err = copy_to_user(uarg, &param, sizeof(param)); \
+ if (err) \
+ return -EFAULT; \
+ return ret; \
+ } while (0)
+
+#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \
+ case cmd: \
+ do { \
+ type param; \
+ int ret, err; \
+ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \
+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \
+ err = copy_from_user(&param, uarg, sizeof(param)); \
+ if (err) \
+ return -EFAULT; \
+ ret = function(kctx, &param); \
+ err = copy_to_user(uarg, &param, sizeof(param)); \
+ if (err) \
+ return -EFAULT; \
+ return ret; \
+ } while (0)
+
+static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct kbase_context *kctx = filp->private_data;
+ struct kbase_device *kbdev = kctx->kbdev;
+ void __user *uarg = (void __user *)arg;
+
+ /* The UK ioctl values overflow the cmd field causing the type to be
+ * incremented
+ */
+ if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2)
+ return kbase_legacy_ioctl(filp, cmd, arg);
+
+ /* The UK version check IOCTL doesn't overflow the cmd field, so is
+ * handled separately here
+ */
+ if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC,
+ UKP_FUNC_ID_CHECK_VERSION,
+ sizeof(struct uku_version_check_args)))
+ return kbase_legacy_ioctl(filp, cmd, arg);
+
+ /* Only these ioctls are available until setup is complete */
+ switch (cmd) {
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK,
+ kbase_api_handshake,
+ struct kbase_ioctl_version_check);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS,
+ kbase_api_set_flags,
+ struct kbase_ioctl_set_flags);
+ }
+
+ /* Block call until version handshake and setup is complete */
+ if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete))
+ return -EINVAL;
+
+ /* Normal ioctls */
+ switch (cmd) {
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT,
+ kbase_api_job_submit,
+ struct kbase_ioctl_job_submit);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS,
+ kbase_api_get_gpuprops,
+ struct kbase_ioctl_get_gpuprops);
+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM,
+ kbase_api_post_term);
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC,
+ kbase_api_mem_alloc,
+ union kbase_ioctl_mem_alloc);
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY,
+ kbase_api_mem_query,
+ union kbase_ioctl_mem_query);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE,
+ kbase_api_mem_free,
+ struct kbase_ioctl_mem_free);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP,
+ kbase_api_hwcnt_reader_setup,
+ struct kbase_ioctl_hwcnt_reader_setup);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE,
+ kbase_api_hwcnt_enable,
+ struct kbase_ioctl_hwcnt_enable);
+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP,
+ kbase_api_hwcnt_dump);
+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR,
+ kbase_api_hwcnt_clear);
+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY,
+ kbase_api_disjoint_query,
+ struct kbase_ioctl_disjoint_query);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION,
+ kbase_api_get_ddk_version,
+ struct kbase_ioctl_get_ddk_version);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT,
+ kbase_api_mem_jit_init,
+ struct kbase_ioctl_mem_jit_init);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC,
+ kbase_api_mem_sync,
+ struct kbase_ioctl_mem_sync);
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET,
+ kbase_api_mem_find_cpu_offset,
+ union kbase_ioctl_mem_find_cpu_offset);
+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID,
+ kbase_api_get_context_id,
+ struct kbase_ioctl_get_context_id);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE,
+ kbase_api_tlstream_acquire,
+ struct kbase_ioctl_tlstream_acquire);
+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH,
+ kbase_api_tlstream_flush);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT,
+ kbase_api_mem_commit,
+ struct kbase_ioctl_mem_commit);
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS,
+ kbase_api_mem_alias,
+ union kbase_ioctl_mem_alias);
+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT,
+ kbase_api_mem_import,
+ union kbase_ioctl_mem_import);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE,
+ kbase_api_mem_flags_change,
+ struct kbase_ioctl_mem_flags_change);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE,
+ kbase_api_stream_create,
+ struct kbase_ioctl_stream_create);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE,
+ kbase_api_fence_validate,
+ struct kbase_ioctl_fence_validate);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS,
+ kbase_api_get_profiling_controls,
+ struct kbase_ioctl_get_profiling_controls);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD,
+ kbase_api_mem_profile_add,
+ struct kbase_ioctl_mem_profile_add);
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE,
+ kbase_api_soft_event_update,
+ struct kbase_ioctl_soft_event_update);
+
+#if MALI_UNIT_TEST
+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST,
+ kbase_api_tlstream_test,
+ struct kbase_ioctl_tlstream_test);
+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS,
+ kbase_api_tlstream_stats,
+ struct kbase_ioctl_tlstream_stats);
+#endif
+ }
+
+ dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd));
+
+ return -ENOIOCTLCMD;
+}
+
static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos)
{
struct kbase_context *kctx = filp->private_data;
@@ -1576,6 +2115,7 @@ static unsigned long kbase_get_unmapped_area(struct file *filp,
int gpu_pc_bits =
kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
bool is_shader_code = false;
+ unsigned long ret;
/* err on fixed address */
if ((flags & MAP_FIXED) || addr)
@@ -1634,7 +2174,19 @@ static unsigned long kbase_get_unmapped_area(struct file *filp,
info.align_offset = align_offset;
info.align_mask = align_mask;
- return kbase_unmapped_area_topdown(&info, is_shader_code);
+ ret = kbase_unmapped_area_topdown(&info, is_shader_code);
+
+ if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base &&
+ high_limit < (kctx->same_va_end << PAGE_SHIFT)) {
+ /* Retry above mmap_base */
+ info.low_limit = mm->mmap_base;
+ info.high_limit = min_t(u64, TASK_SIZE,
+ (kctx->same_va_end << PAGE_SHIFT));
+
+ ret = kbase_unmapped_area_topdown(&info, is_shader_code);
+ }
+
+ return ret;
}
static const struct file_operations kbase_fops = {
@@ -3132,42 +3684,74 @@ static const struct file_operations kbasep_serialize_jobs_debugfs_fops = {
#endif /* CONFIG_DEBUG_FS */
-static int kbasep_protected_mode_enter(struct kbase_device *kbdev)
+static int kbasep_protected_mode_init(struct kbase_device *kbdev)
{
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
- GPU_COMMAND_SET_PROTECTED_MODE, NULL);
- return 0;
-}
+#ifdef CONFIG_OF
+ struct device_node *protected_node;
+ struct platform_device *pdev;
+ struct protected_mode_device *protected_dev;
+#endif
-static bool kbasep_protected_mode_supported(struct kbase_device *kbdev)
-{
- return true;
-}
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) {
+ /* Use native protected ops */
+ kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev),
+ GFP_KERNEL);
+ if (!kbdev->protected_dev)
+ return -ENOMEM;
+ kbdev->protected_dev->data = kbdev;
+ kbdev->protected_ops = &kbase_native_protected_ops;
+ kbdev->protected_mode_support = true;
+ return 0;
+ }
-static struct kbase_protected_ops kbasep_protected_ops = {
- .protected_mode_enter = kbasep_protected_mode_enter,
- .protected_mode_reset = NULL,
- .protected_mode_supported = kbasep_protected_mode_supported,
-};
+ kbdev->protected_mode_support = false;
-static void kbasep_protected_mode_init(struct kbase_device *kbdev)
-{
- kbdev->protected_ops = NULL;
+#ifdef CONFIG_OF
+ protected_node = of_parse_phandle(kbdev->dev->of_node,
+ "protected-mode-switcher", 0);
- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) {
- /* Use native protected ops */
- kbdev->protected_ops = &kbasep_protected_ops;
+ if (!protected_node)
+ protected_node = of_parse_phandle(kbdev->dev->of_node,
+ "secure-mode-switcher", 0);
+
+ if (!protected_node) {
+ /* If protected_node cannot be looked up then we assume
+ * protected mode is not supported on this platform. */
+ dev_info(kbdev->dev, "Protected mode not available\n");
+ return 0;
+ }
+
+ pdev = of_find_device_by_node(protected_node);
+ if (!pdev)
+ return -EINVAL;
+
+ protected_dev = platform_get_drvdata(pdev);
+ if (!protected_dev)
+ return -EPROBE_DEFER;
+
+ kbdev->protected_ops = &protected_dev->ops;
+ kbdev->protected_dev = protected_dev;
+
+ if (kbdev->protected_ops) {
+ int err;
+
+ /* Make sure protected mode is disabled on startup */
+ mutex_lock(&kbdev->pm.lock);
+ err = kbdev->protected_ops->protected_mode_disable(
+ kbdev->protected_dev);
+ mutex_unlock(&kbdev->pm.lock);
+
+ /* protected_mode_disable() returns -EINVAL if not supported */
+ kbdev->protected_mode_support = (err != -EINVAL);
}
-#ifdef PROTECTED_CALLBACKS
- else
- kbdev->protected_ops = PROTECTED_CALLBACKS;
#endif
+ return 0;
+}
- if (kbdev->protected_ops)
- kbdev->protected_mode_support =
- kbdev->protected_ops->protected_mode_supported(kbdev);
- else
- kbdev->protected_mode_support = false;
+static void kbasep_protected_mode_term(struct kbase_device *kbdev)
+{
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE))
+ kfree(kbdev->protected_dev);
}
#ifdef CONFIG_MALI_NO_MALI
@@ -3649,6 +4233,8 @@ static int kbase_platform_device_remove(struct platform_device *pdev)
if (!kbdev)
return -ENODEV;
+ kfree(kbdev->gpu_props.prop_buffer);
+
#ifdef CONFIG_MALI_FPGA_BUS_LOGGER
if (kbdev->inited_subsys & inited_buslogger) {
bl_core_client_unregister(kbdev->buslogger);
@@ -3718,6 +4304,11 @@ static int kbase_platform_device_remove(struct platform_device *pdev)
if (kbdev->inited_subsys & inited_mem)
kbase_mem_halt(kbdev);
+ if (kbdev->inited_subsys & inited_protected) {
+ kbasep_protected_mode_term(kbdev);
+ kbdev->inited_subsys &= ~inited_protected;
+ }
+
if (kbdev->inited_subsys & inited_js) {
kbasep_js_devdata_term(kbdev);
kbdev->inited_subsys &= ~inited_js;
@@ -3733,6 +4324,11 @@ static int kbase_platform_device_remove(struct platform_device *pdev)
kbdev->inited_subsys &= ~inited_pm_runtime_init;
}
+ if (kbdev->inited_subsys & inited_ctx_sched) {
+ kbase_ctx_sched_term(kbdev);
+ kbdev->inited_subsys &= ~inited_ctx_sched;
+ }
+
if (kbdev->inited_subsys & inited_device) {
kbase_device_term(kbdev);
kbdev->inited_subsys &= ~inited_device;
@@ -3805,10 +4401,6 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
kbdev->dev = &pdev->dev;
dev_set_drvdata(kbdev->dev, kbdev);
-#ifdef CONFIG_DEVFREQ_THERMAL
- INIT_LIST_HEAD(&kbdev->ipa.power_models);
-#endif
-
#ifdef CONFIG_MALI_NO_MALI
err = gpu_device_create(kbdev);
if (err) {
@@ -3877,6 +4469,15 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
}
kbdev->inited_subsys |= inited_device;
+ err = kbase_ctx_sched_init(kbdev);
+ if (err) {
+ dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n",
+ err);
+ kbase_platform_device_remove(pdev);
+ return err;
+ }
+ kbdev->inited_subsys |= inited_ctx_sched;
+
if (kbdev->pm.callback_power_runtime_init) {
err = kbdev->pm.callback_power_runtime_init(kbdev);
if (err) {
@@ -3902,7 +4503,13 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
kbase_device_coherency_init(kbdev, prod_id);
- kbasep_protected_mode_init(kbdev);
+ err = kbasep_protected_mode_init(kbdev);
+ if (err) {
+ dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n");
+ kbase_platform_device_remove(pdev);
+ return err;
+ }
+ kbdev->inited_subsys |= inited_protected;
dev_list = kbase_dev_list_get();
list_add(&kbdev->entry, &kbase_dev_list);
@@ -4020,6 +4627,13 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
}
#endif
+ err = kbase_gpuprops_populate_user_buffer(kbdev);
+ if (err) {
+ dev_err(&pdev->dev, "GPU property population failed");
+ kbase_platform_device_remove(pdev);
+ return err;
+ }
+
dev_info(kbdev->dev,
"Probed as %s\n", dev_name(kbdev->mdev.this_device));
@@ -4030,7 +4644,6 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE
-
/**
* kbase_device_suspend - Suspend callback from the OS.
*
diff --git a/mali_kbase/mali_kbase_ctx_sched.c b/mali_kbase/mali_kbase_ctx_sched.c
new file mode 100644
index 0000000..e2f7baa
--- /dev/null
+++ b/mali_kbase/mali_kbase_ctx_sched.c
@@ -0,0 +1,203 @@
+/*
+ *
+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <mali_kbase.h>
+#include <mali_kbase_config_defaults.h>
+
+#include "mali_kbase_ctx_sched.h"
+
+int kbase_ctx_sched_init(struct kbase_device *kbdev)
+{
+ int as_present = (1U << kbdev->nr_hw_address_spaces) - 1;
+
+ /* These two must be recalculated if nr_hw_address_spaces changes
+ * (e.g. for HW workarounds) */
+ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces;
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
+ bool use_workaround;
+
+ use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
+ if (use_workaround) {
+ dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only");
+ kbdev->nr_user_address_spaces = 1;
+ }
+ }
+
+ kbdev->as_free = as_present; /* All ASs initially free */
+
+ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx));
+
+ return 0;
+}
+
+void kbase_ctx_sched_term(struct kbase_device *kbdev)
+{
+ s8 i;
+
+ /* Sanity checks */
+ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) {
+ WARN_ON(kbdev->as_to_kctx[i] != NULL);
+ WARN_ON(!(kbdev->as_free & (1u << i)));
+ }
+}
+
+/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space
+ *
+ * @kbdev: The context for which to find a free address space
+ *
+ * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID
+ *
+ * This function returns an address space available for use. It would prefer
+ * returning an AS that has been previously assigned to the context to
+ * avoid having to reprogram the MMU.
+ */
+static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx)
+{
+ struct kbase_device *const kbdev = kctx->kbdev;
+ int free_as;
+
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+
+ /* First check if the previously assigned AS is available */
+ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) &&
+ (kbdev->as_free & (1u << kctx->as_nr)))
+ return kctx->as_nr;
+
+ /* The previously assigned AS was taken, we'll be returning any free
+ * AS at this point.
+ */
+ free_as = ffs(kbdev->as_free) - 1;
+ if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces)
+ return free_as;
+
+ return KBASEP_AS_NR_INVALID;
+}
+
+int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx)
+{
+ struct kbase_device *const kbdev = kctx->kbdev;
+
+ lockdep_assert_held(&kbdev->mmu_hw_mutex);
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+
+ WARN_ON(!kbdev->pm.backend.gpu_powered);
+
+ if (atomic_inc_return(&kctx->refcount) == 1) {
+ int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx);
+
+ if (free_as != KBASEP_AS_NR_INVALID) {
+ kbdev->as_free &= ~(1u << free_as);
+ /* Only program the MMU if the context has not been
+ * assigned the same address space before.
+ */
+ if (free_as != kctx->as_nr) {
+ struct kbase_context *const prev_kctx =
+ kbdev->as_to_kctx[free_as];
+
+ if (prev_kctx) {
+ WARN_ON(atomic_read(&prev_kctx->refcount) != 0);
+ kbase_mmu_disable(prev_kctx);
+ prev_kctx->as_nr = KBASEP_AS_NR_INVALID;
+ }
+
+ kctx->as_nr = free_as;
+ kbdev->as_to_kctx[free_as] = kctx;
+ kbase_mmu_update(kctx);
+ }
+ } else {
+ atomic_dec(&kctx->refcount);
+
+ /* Failed to find an available address space, we must
+ * be returning an error at this point.
+ */
+ WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID);
+ }
+ }
+
+ return kctx->as_nr;
+}
+
+void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx)
+{
+ struct kbase_device *const kbdev = kctx->kbdev;
+
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+ WARN_ON(atomic_read(&kctx->refcount) == 0);
+ WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID);
+ WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx);
+
+ atomic_inc(&kctx->refcount);
+}
+
+void kbase_ctx_sched_release_ctx(struct kbase_context *kctx)
+{
+ struct kbase_device *const kbdev = kctx->kbdev;
+
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+
+ if (atomic_dec_return(&kctx->refcount) == 0)
+ kbdev->as_free |= (1u << kctx->as_nr);
+}
+
+void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx)
+{
+ struct kbase_device *const kbdev = kctx->kbdev;
+
+ lockdep_assert_held(&kbdev->mmu_hw_mutex);
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+
+ WARN_ON(atomic_read(&kctx->refcount) != 0);
+
+ if (kctx->as_nr != KBASEP_AS_NR_INVALID) {
+ if (kbdev->pm.backend.gpu_powered)
+ kbase_mmu_disable(kctx);
+
+ kbdev->as_to_kctx[kctx->as_nr] = NULL;
+ kctx->as_nr = KBASEP_AS_NR_INVALID;
+ }
+}
+
+void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev)
+{
+ s8 i;
+
+ lockdep_assert_held(&kbdev->mmu_hw_mutex);
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+
+ WARN_ON(!kbdev->pm.backend.gpu_powered);
+
+ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) {
+ struct kbase_context *kctx;
+
+ kctx = kbdev->as_to_kctx[i];
+ if (kctx) {
+ if (atomic_read(&kctx->refcount)) {
+ WARN_ON(kctx->as_nr != i);
+
+ kbase_mmu_update(kctx);
+ } else {
+ /* This context might have been assigned an
+ * AS before, clear it.
+ */
+ kbdev->as_to_kctx[kctx->as_nr] = NULL;
+ kctx->as_nr = KBASEP_AS_NR_INVALID;
+ }
+ } else {
+ kbase_mmu_disable_as(kbdev, i);
+ }
+ }
+}
diff --git a/mali_kbase/mali_kbase_ctx_sched.h b/mali_kbase/mali_kbase_ctx_sched.h
new file mode 100644
index 0000000..e551525
--- /dev/null
+++ b/mali_kbase/mali_kbase_ctx_sched.h
@@ -0,0 +1,131 @@
+/*
+ *
+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_CTX_SCHED_H_
+#define _KBASE_CTX_SCHED_H_
+
+#include <mali_kbase.h>
+
+/* The Context Scheduler manages address space assignment and reference
+ * counting to kbase_context. The interface has been designed to minimise
+ * interactions between the Job Scheduler and Power Management/MMU to support
+ * both the existing Job Scheduler and Command Stream Frontend interface.
+ *
+ * The initial implementation of the Context Scheduler does not schedule
+ * contexts. Instead it relies on the Job Scheduler/CSF to make decisions of
+ * when to schedule/evict contexts if address spaces are starved. In the
+ * future, once an interface between the CS and JS/CSF have been devised to
+ * provide enough information about how each context is consuming GPU resources,
+ * those decisions can be made in the CS itself, thereby reducing duplicated
+ * code.
+ */
+
+/* base_ctx_sched_init - Initialise the context scheduler
+ *
+ * @kbdev: The device for which the context scheduler needs to be
+ * initialised
+ *
+ * Return: 0 for success, otherwise failure
+ *
+ * This must be called during device initilisation. The number of hardware
+ * address spaces must already be established before calling this function.
+ */
+int kbase_ctx_sched_init(struct kbase_device *kbdev);
+
+/* base_ctx_sched_term - Terminate the context scheduler
+ *
+ * @kbdev: The device for which the context scheduler needs to be
+ * terminated
+ *
+ * This must be called during device termination after all contexts have been
+ * destroyed.
+ */
+void kbase_ctx_sched_term(struct kbase_device *kbdev);
+
+/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context
+ *
+ * @kctx: The context to which to retain a reference
+ *
+ * Return: The address space that the context has been assigned to or
+ * KBASEP_AS_NR_INVALID if no address space was available.
+ *
+ * This function should be called whenever an address space should be assigned
+ * to a context and programmed onto the MMU. It should typically be called
+ * when jobs are ready to be submitted to the GPU.
+ *
+ * It can be called as many times as necessary. The address space will be
+ * assigned to the context for as long as there is a reference to said context.
+ *
+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be
+ * held whilst calling this function.
+ */
+int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx);
+
+/* kbase_ctx_sched_retain_ctx_refcount
+ *
+ * @kctx: The context to which to retain a reference
+ *
+ * This function only retains a reference to the context. It must be called
+ * only when the context already has a reference.
+ *
+ * This is typically called inside an atomic session where we know the context
+ * is already scheduled in but want to take an extra reference to ensure that
+ * it doesn't get descheduled.
+ *
+ * The kbase_device::hwaccess_lock must be held whilst calling this function
+ */
+void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx);
+
+/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context
+ *
+ * @kctx: The context from which to release a reference
+ *
+ * This function should be called whenever an address space could be unassigned
+ * from a context. When there are no more references to said context, the
+ * address space previously assigned to this context shall be reassigned to
+ * other contexts as needed.
+ *
+ * The kbase_device::hwaccess_lock must be held whilst calling this function
+ */
+void kbase_ctx_sched_release_ctx(struct kbase_context *kctx);
+
+/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space
+ *
+ * @kctx: The context to be removed
+ *
+ * This function should be called when a context is being destroyed. The
+ * context must no longer have any reference. If it has been assigned an
+ * address space before then the AS will be unprogrammed.
+ *
+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be
+ * held whilst calling this function.
+ */
+void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx);
+
+/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces
+ *
+ * @kbdev: The device for which address spaces to be reprogrammed
+ *
+ * This function shall reprogram all address spaces previously assigned to
+ * contexts. It can be used after the GPU is reset.
+ *
+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be
+ * held whilst calling this function.
+ */
+void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev);
+
+#endif /* _KBASE_CTX_SCHED_H_ */
diff --git a/mali_kbase/mali_kbase_debug_mem_view.c b/mali_kbase/mali_kbase_debug_mem_view.c
index dc75c86..6f2cbdf 100644
--- a/mali_kbase/mali_kbase_debug_mem_view.c
+++ b/mali_kbase/mali_kbase_debug_mem_view.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_defs.h b/mali_kbase/mali_kbase_defs.h
index 73f849c..8dabf7c 100644
--- a/mali_kbase/mali_kbase_defs.h
+++ b/mali_kbase/mali_kbase_defs.h
@@ -35,6 +35,7 @@
#include <mali_kbase_mmu_mode.h>
#include <mali_kbase_instr_defs.h>
#include <mali_kbase_pm.h>
+#include <protected_mode_switcher.h>
#include <linux/atomic.h>
#include <linux/mempool.h>
@@ -50,19 +51,19 @@
#include <linux/kds.h>
#endif /* CONFIG_KDS */
-#ifdef CONFIG_SYNC
-#include "sync.h"
-#endif /* CONFIG_SYNC */
-
-#include "mali_kbase_dma_fence.h"
+#if defined(CONFIG_SYNC)
+#include <sync.h>
+#else
+#include "mali_kbase_fence_defs.h"
+#endif
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#endif /* CONFIG_DEBUG_FS */
-#ifdef CONFIG_PM_DEVFREQ
+#ifdef CONFIG_MALI_DEVFREQ
#include <linux/devfreq.h>
-#endif /* CONFIG_DEVFREQ */
+#endif /* CONFIG_MALI_DEVFREQ */
#include <linux/clk.h>
#include <linux/regulator/consumer.h>
@@ -436,15 +437,26 @@ struct kbase_jd_atom {
struct kds_resource_set *kds_rset;
bool kds_dep_satisfied;
#endif /* CONFIG_KDS */
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC)
+ /* Stores either an input or output fence, depending on soft-job type */
struct sync_fence *fence;
struct sync_fence_waiter sync_waiter;
#endif /* CONFIG_SYNC */
-#ifdef CONFIG_MALI_DMA_FENCE
+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
struct {
- /* This points to the dma-buf fence for this atom. If this is
- * NULL then there is no fence for this atom and the other
- * fields related to dma_fence may have invalid data.
+ /* Use the functions/API defined in mali_kbase_fence.h to
+ * when working with this sub struct */
+#if defined(CONFIG_SYNC_FILE)
+ /* Input fence */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence_in;
+#else
+ struct dma_fence *fence_in;
+#endif
+#endif
+ /* This points to the dma-buf output fence for this atom. If
+ * this is NULL then there is no fence for this atom and the
+ * following fields related to dma_fence may have invalid data.
*
* The context and seqno fields contain the details for this
* fence.
@@ -453,7 +465,11 @@ struct kbase_jd_atom {
* regardless of the event_code of the katom (signal also on
* failure).
*/
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
/* The dma-buf fence context number for this atom. A unique
* context number is allocated to each katom in the context on
* context creation.
@@ -492,7 +508,7 @@ struct kbase_jd_atom {
*/
atomic_t dep_count;
} dma_fence;
-#endif /* CONFIG_MALI_DMA_FENCE */
+#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE*/
/* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */
enum base_jd_event_code event_code;
@@ -887,40 +903,6 @@ struct kbase_pm_device_data {
};
/**
- * struct kbase_protected_ops - Platform specific functions for GPU protected
- * mode operations
- * @protected_mode_enter: Callback to enter protected mode on the GPU
- * @protected_mode_reset: Callback to reset the GPU and exit protected mode.
- * @protected_mode_supported: Callback to check if protected mode is supported.
- */
-struct kbase_protected_ops {
- /**
- * protected_mode_enter() - Enter protected mode on the GPU
- * @kbdev: The kbase device
- *
- * Return: 0 on success, non-zero on error
- */
- int (*protected_mode_enter)(struct kbase_device *kbdev);
-
- /**
- * protected_mode_reset() - Reset the GPU and exit protected mode
- * @kbdev: The kbase device
- *
- * Return: 0 on success, non-zero on error
- */
- int (*protected_mode_reset)(struct kbase_device *kbdev);
-
- /**
- * protected_mode_supported() - Check if protected mode is supported
- * @kbdev: The kbase device
- *
- * Return: 0 on success, non-zero on error
- */
- bool (*protected_mode_supported)(struct kbase_device *kbdev);
-};
-
-
-/**
* struct kbase_mem_pool - Page based memory pool for kctx/kbdev
* @kbdev: Kbase device where memory is used
* @cur_size: Number of free pages currently in the pool (may exceed @max_size
@@ -1003,6 +985,14 @@ struct kbase_device {
struct kbase_mmu_mode const *mmu_mode;
struct kbase_as as[BASE_MAX_NR_AS];
+ /* The below variables (as_free and as_to_kctx) are managed by the
+ * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must
+ * be held whilst accessing these.
+ */
+ u16 as_free; /* Bitpattern of free Address Spaces */
+ /* Mapping from active Address Spaces to kbase_context */
+ struct kbase_context *as_to_kctx[BASE_MAX_NR_AS];
+
spinlock_t mmu_mask_change;
@@ -1097,7 +1087,7 @@ struct kbase_device {
struct list_head kctx_list;
struct mutex kctx_list_lock;
-#ifdef CONFIG_PM_DEVFREQ
+#ifdef CONFIG_MALI_DEVFREQ
struct devfreq_dev_profile devfreq_profile;
struct devfreq *devfreq;
unsigned long current_freq;
@@ -1117,14 +1107,11 @@ struct kbase_device {
struct {
/* Access to this struct must be with ipa.lock held */
struct mutex lock;
- unsigned long last_model_dyn_power_coeff;
- unsigned long last_static_power_coeff;
- struct list_head power_models;
struct kbase_ipa_model *configured_model;
struct kbase_ipa_model *fallback_model;
} ipa;
-#endif
-#endif
+#endif /* CONFIG_DEVFREQ_THERMAL */
+#endif /* CONFIG_MALI_DEVFREQ */
#ifdef CONFIG_MALI_TRACE_TIMELINE
@@ -1224,8 +1211,11 @@ struct kbase_device {
u32 snoop_enable_smc;
u32 snoop_disable_smc;
- /* Protected operations */
- struct kbase_protected_ops *protected_ops;
+ /* Protected mode operations */
+ struct protected_mode_ops *protected_ops;
+
+ /* Protected device attached to this kbase device */
+ struct protected_mode_device *protected_dev;
/*
* true when GPU is put into protected mode
@@ -1411,6 +1401,14 @@ struct kbase_context {
* you can take whilst doing this) */
int as_nr;
+ /* Keeps track of the number of users of this context. A user can be a
+ * job that is available for execution, instrumentation needing to 'pin'
+ * a context for counter collection, etc. If the refcount reaches 0 then
+ * this context is considered inactive and the previously programmed
+ * AS might be cleared at any point.
+ */
+ atomic_t refcount;
+
/* NOTE:
*
* Flags are in jctx.sched_info.ctx.flags
diff --git a/mali_kbase/mali_kbase_device.c b/mali_kbase/mali_kbase_device.c
index e53f876..d635fcc 100644
--- a/mali_kbase/mali_kbase_device.c
+++ b/mali_kbase/mali_kbase_device.c
@@ -245,11 +245,10 @@ int kbase_device_init(struct kbase_device * const kbdev)
kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- kbdev->mmu_mode = kbase_mmu_mode_get_aarch64();
-#else
- kbdev->mmu_mode = kbase_mmu_mode_get_lpae();
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64();
+ else
+ kbdev->mmu_mode = kbase_mmu_mode_get_lpae();
#ifdef CONFIG_MALI_DEBUG
init_waitqueue_head(&kbdev->driver_inactive_wait);
@@ -656,28 +655,6 @@ void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 va
}
}
-u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control)
-{
- u32 ret_value = 0;
-
- switch (control) {
- case FBDUMP_CONTROL_ENABLE:
- /* fall through */
- case FBDUMP_CONTROL_RATE:
- /* fall through */
- case SW_COUNTER_ENABLE:
- /* fall through */
- case FBDUMP_CONTROL_RESIZE_FACTOR:
- ret_value = kbdev->kbase_profiling_controls[control];
- break;
- default:
- dev_err(kbdev->dev, "Profiling control %d not found\n", control);
- break;
- }
-
- return ret_value;
-}
-
/*
* Called by gator to control the production of
* profiling information at runtime
diff --git a/mali_kbase/mali_kbase_dma_fence.c b/mali_kbase/mali_kbase_dma_fence.c
index 97bb6c5..9197743 100644
--- a/mali_kbase/mali_kbase_dma_fence.c
+++ b/mali_kbase/mali_kbase_dma_fence.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -22,7 +22,6 @@
#include "mali_kbase_dma_fence.h"
#include <linux/atomic.h>
-#include <linux/fence.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
@@ -34,10 +33,6 @@
#include <mali_kbase.h>
-
-/* Spin lock protecting all Mali fences as fence->lock. */
-static DEFINE_SPINLOCK(kbase_dma_fence_lock);
-
static void
kbase_dma_fence_work(struct work_struct *pwork);
@@ -49,67 +44,12 @@ kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom)
list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource);
}
-void
+static void
kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom)
{
list_del(&katom->queue);
}
-static const char *
-kbase_dma_fence_get_driver_name(struct fence *fence)
-{
- return kbase_drv_name;
-}
-
-static const char *
-kbase_dma_fence_get_timeline_name(struct fence *fence)
-{
- return kbase_timeline_name;
-}
-
-static bool
-kbase_dma_fence_enable_signaling(struct fence *fence)
-{
- /* If in the future we need to add code here remember to
- * to get a reference to the fence and release it when signaling
- * as stated in fence.h
- */
- return true;
-}
-
-static void
-kbase_dma_fence_fence_value_str(struct fence *fence, char *str, int size)
-{
- snprintf(str, size, "%u", fence->seqno);
-}
-
-static const struct fence_ops kbase_dma_fence_ops = {
- .get_driver_name = kbase_dma_fence_get_driver_name,
- .get_timeline_name = kbase_dma_fence_get_timeline_name,
- .enable_signaling = kbase_dma_fence_enable_signaling,
- /* Use the default wait */
- .wait = fence_default_wait,
- .fence_value_str = kbase_dma_fence_fence_value_str,
-};
-
-static struct fence *
-kbase_dma_fence_new(unsigned int context, unsigned int seqno)
-{
- struct fence *fence;
-
- fence = kzalloc(sizeof(*fence), GFP_KERNEL);
- if (!fence)
- return NULL;
-
- fence_init(fence,
- &kbase_dma_fence_ops,
- &kbase_dma_fence_lock,
- context,
- seqno);
-
- return fence;
-}
-
static int
kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info,
struct ww_acquire_ctx *ctx)
@@ -189,59 +129,6 @@ kbase_dma_fence_queue_work(struct kbase_jd_atom *katom)
}
/**
- * kbase_dma_fence_free_callbacks - Free dma-fence callbacks on a katom
- * @katom: Pointer to katom
- * @queue_worker: Boolean indicating if fence worker is to be queued when
- * dep_count reaches 0.
- *
- * This function will free all fence callbacks on the katom's list of
- * callbacks. Callbacks that have not yet been called, because their fence
- * hasn't yet signaled, will first be removed from the fence.
- *
- * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held.
- */
-static void
-kbase_dma_fence_free_callbacks(struct kbase_jd_atom *katom, bool queue_worker)
-{
- struct kbase_dma_fence_cb *cb, *tmp;
-
- lockdep_assert_held(&katom->kctx->jctx.lock);
-
- /* Clean up and free callbacks. */
- list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) {
- bool ret;
-
- /* Cancel callbacks that hasn't been called yet. */
- ret = fence_remove_callback(cb->fence, &cb->fence_cb);
- if (ret) {
- int ret;
-
- /* Fence had not signaled, clean up after
- * canceling.
- */
- ret = atomic_dec_return(&katom->dma_fence.dep_count);
-
- if (unlikely(queue_worker && ret == 0)) {
- /*
- * dep_count went to zero and queue_worker is
- * true. Queue the worker to handle the
- * completion of the katom.
- */
- kbase_dma_fence_queue_work(katom);
- }
- }
-
- /*
- * Release the reference taken in
- * kbase_dma_fence_add_callback().
- */
- fence_put(cb->fence);
- list_del(&cb->node);
- kfree(cb);
- }
-}
-
-/**
* kbase_dma_fence_cancel_atom() - Cancels waiting on an atom
* @katom: Katom to cancel
*
@@ -253,14 +140,12 @@ kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom)
lockdep_assert_held(&katom->kctx->jctx.lock);
/* Cancel callbacks and clean up. */
- kbase_dma_fence_free_callbacks(katom, false);
-
- KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == 0);
+ kbase_fence_free_callbacks(katom);
/* Mark the atom as handled in case all fences signaled just before
* canceling the callbacks and the worker was queued.
*/
- atomic_set(&katom->dma_fence.dep_count, -1);
+ kbase_fence_dep_count_set(katom, -1);
/* Prevent job_done_nolock from being called twice on an atom when
* there is a race between job completion and cancellation.
@@ -290,15 +175,15 @@ kbase_dma_fence_work(struct work_struct *pwork)
ctx = &katom->kctx->jctx;
mutex_lock(&ctx->lock);
- if (atomic_read(&katom->dma_fence.dep_count) != 0)
+ if (kbase_fence_dep_count_read(katom) != 0)
goto out;
- atomic_set(&katom->dma_fence.dep_count, -1);
+ kbase_fence_dep_count_set(katom, -1);
/* Remove atom from list of dma-fence waiting atoms. */
kbase_dma_fence_waiters_remove(katom);
/* Cleanup callbacks. */
- kbase_dma_fence_free_callbacks(katom, false);
+ kbase_fence_free_callbacks(katom);
/*
* Queue atom on GPU, unless it has already completed due to a failing
* dependency. Run jd_done_nolock() on the katom if it is completed.
@@ -312,64 +197,15 @@ out:
mutex_unlock(&ctx->lock);
}
-/**
- * kbase_dma_fence_add_callback() - Add callback on @fence to block @katom
- * @katom: Pointer to katom that will be blocked by @fence
- * @fence: Pointer to fence on which to set up the callback
- * @callback: Pointer to function to be called when fence is signaled
- *
- * Caller needs to hold a reference to @fence when calling this function, and
- * the caller is responsible for releasing that reference. An additional
- * reference to @fence will be taken when the callback was successfully set up
- * and @fence needs to be kept valid until the callback has been called and
- * cleanup have been done.
- *
- * Return: 0 on success: fence was either already signalled, or callback was
- * set up. Negative error code is returned on error.
- */
-static int
-kbase_dma_fence_add_callback(struct kbase_jd_atom *katom,
- struct fence *fence,
- fence_func_t callback)
-{
- int err = 0;
- struct kbase_dma_fence_cb *kbase_fence_cb;
-
- kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL);
- if (!kbase_fence_cb)
- return -ENOMEM;
-
- kbase_fence_cb->fence = fence;
- kbase_fence_cb->katom = katom;
- INIT_LIST_HEAD(&kbase_fence_cb->node);
-
- err = fence_add_callback(fence, &kbase_fence_cb->fence_cb, callback);
- if (err == -ENOENT) {
- /* Fence signaled, clear the error and return */
- err = 0;
- kbase_fence_cb->fence = NULL;
- kfree(kbase_fence_cb);
- } else if (err) {
- kfree(kbase_fence_cb);
- } else {
- /*
- * Get reference to fence that will be kept until callback gets
- * cleaned up in kbase_dma_fence_free_callbacks().
- */
- fence_get(fence);
- atomic_inc(&katom->dma_fence.dep_count);
- /* Add callback to katom's list of callbacks */
- list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks);
- }
-
- return err;
-}
-
static void
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb)
+#else
+kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
+#endif
{
- struct kbase_dma_fence_cb *kcb = container_of(cb,
- struct kbase_dma_fence_cb,
+ struct kbase_fence_cb *kcb = container_of(cb,
+ struct kbase_fence_cb,
fence_cb);
struct kbase_jd_atom *katom = kcb->katom;
@@ -377,7 +213,8 @@ kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb)
* preventing this callback from ever scheduling work. Which in turn
* would reschedule the atom.
*/
- if (atomic_dec_and_test(&katom->dma_fence.dep_count))
+
+ if (kbase_fence_dep_count_dec_and_test(katom))
kbase_dma_fence_queue_work(katom);
}
@@ -386,8 +223,13 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom,
struct reservation_object *resv,
bool exclusive)
{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *excl_fence = NULL;
struct fence **shared_fences = NULL;
+#else
+ struct dma_fence *excl_fence = NULL;
+ struct dma_fence **shared_fences = NULL;
+#endif
unsigned int shared_count = 0;
int err, i;
@@ -399,16 +241,16 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom,
return err;
if (excl_fence) {
- err = kbase_dma_fence_add_callback(katom,
- excl_fence,
- kbase_dma_fence_cb);
+ err = kbase_fence_add_callback(katom,
+ excl_fence,
+ kbase_dma_fence_cb);
/* Release our reference, taken by reservation_object_get_fences_rcu(),
* to the fence. We have set up our callback (if that was possible),
* and it's the fence's owner is responsible for singling the fence
* before allowing it to disappear.
*/
- fence_put(excl_fence);
+ dma_fence_put(excl_fence);
if (err)
goto out;
@@ -416,9 +258,9 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom,
if (exclusive) {
for (i = 0; i < shared_count; i++) {
- err = kbase_dma_fence_add_callback(katom,
- shared_fences[i],
- kbase_dma_fence_cb);
+ err = kbase_fence_add_callback(katom,
+ shared_fences[i],
+ kbase_dma_fence_cb);
if (err)
goto out;
}
@@ -431,7 +273,7 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom,
*/
out:
for (i = 0; i < shared_count; i++)
- fence_put(shared_fences[i]);
+ dma_fence_put(shared_fences[i]);
kfree(shared_fences);
if (err) {
@@ -439,7 +281,7 @@ out:
* On error, cancel and clean up all callbacks that was set up
* before the error.
*/
- kbase_dma_fence_free_callbacks(katom, false);
+ kbase_fence_free_callbacks(katom);
}
return err;
@@ -468,13 +310,16 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
struct kbase_dma_fence_resv_info *info)
{
int err, i;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
struct ww_acquire_ctx ww_ctx;
lockdep_assert_held(&katom->kctx->jctx.lock);
- fence = kbase_dma_fence_new(katom->dma_fence.context,
- atomic_inc_return(&katom->dma_fence.seqno));
+ fence = kbase_fence_out_new(katom);
if (!fence) {
err = -ENOMEM;
dev_err(katom->kctx->kbdev->dev,
@@ -482,15 +327,14 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
return err;
}
- katom->dma_fence.fence = fence;
- atomic_set(&katom->dma_fence.dep_count, 1);
+ kbase_fence_dep_count_set(katom, 1);
err = kbase_dma_fence_lock_reservations(info, &ww_ctx);
if (err) {
dev_err(katom->kctx->kbdev->dev,
"Error %d locking reservations.\n", err);
- atomic_set(&katom->dma_fence.dep_count, -1);
- fence_put(fence);
+ kbase_fence_dep_count_set(katom, -1);
+ kbase_fence_out_remove(katom);
return err;
}
@@ -512,7 +356,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
goto end;
}
- reservation_object_add_shared_fence(obj, katom->dma_fence.fence);
+ reservation_object_add_shared_fence(obj, fence);
} else {
err = kbase_dma_fence_add_reservation_callback(katom, obj, true);
if (err) {
@@ -521,7 +365,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
goto end;
}
- reservation_object_add_excl_fence(obj, katom->dma_fence.fence);
+ reservation_object_add_excl_fence(obj, fence);
}
}
@@ -530,9 +374,9 @@ end:
if (likely(!err)) {
/* Test if the callbacks are already triggered */
- if (atomic_dec_and_test(&katom->dma_fence.dep_count)) {
- atomic_set(&katom->dma_fence.dep_count, -1);
- kbase_dma_fence_free_callbacks(katom, false);
+ if (kbase_fence_dep_count_dec_and_test(katom)) {
+ kbase_fence_dep_count_set(katom, -1);
+ kbase_fence_free_callbacks(katom);
} else {
/* Add katom to the list of dma-buf fence waiting atoms
* only if it is still waiting.
@@ -545,8 +389,8 @@ end:
* kill it for us), signal the fence, free callbacks and the
* fence.
*/
- kbase_dma_fence_free_callbacks(katom, false);
- atomic_set(&katom->dma_fence.dep_count, -1);
+ kbase_fence_free_callbacks(katom);
+ kbase_fence_dep_count_set(katom, -1);
kbase_dma_fence_signal(katom);
}
@@ -569,7 +413,8 @@ void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx)
void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom)
{
/* Cancel callbacks and clean up. */
- kbase_dma_fence_free_callbacks(katom, true);
+ if (kbase_fence_free_callbacks(katom))
+ kbase_dma_fence_queue_work(katom);
}
void kbase_dma_fence_signal(struct kbase_jd_atom *katom)
@@ -577,14 +422,12 @@ void kbase_dma_fence_signal(struct kbase_jd_atom *katom)
if (!katom->dma_fence.fence)
return;
- KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == -1);
-
/* Signal the atom's fence. */
- fence_signal(katom->dma_fence.fence);
- fence_put(katom->dma_fence.fence);
- katom->dma_fence.fence = NULL;
+ dma_fence_signal(katom->dma_fence.fence);
+
+ kbase_fence_out_remove(katom);
- kbase_dma_fence_free_callbacks(katom, false);
+ kbase_fence_free_callbacks(katom);
}
void kbase_dma_fence_term(struct kbase_context *kctx)
diff --git a/mali_kbase/mali_kbase_dma_fence.h b/mali_kbase/mali_kbase_dma_fence.h
index 3b0a69b..c9ab403 100644
--- a/mali_kbase/mali_kbase_dma_fence.h
+++ b/mali_kbase/mali_kbase_dma_fence.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -20,9 +20,9 @@
#ifdef CONFIG_MALI_DMA_FENCE
-#include <linux/fence.h>
#include <linux/list.h>
#include <linux/reservation.h>
+#include <mali_kbase_fence.h>
/* Forward declaration from mali_kbase_defs.h */
@@ -30,20 +30,6 @@ struct kbase_jd_atom;
struct kbase_context;
/**
- * struct kbase_dma_fence_cb - Mali dma-fence callback data struct
- * @fence_cb: Callback function
- * @katom: Pointer to katom that is waiting on this callback
- * @fence: Pointer to the fence object on which this callback is waiting
- * @node: List head for linking this callback to the katom
- */
-struct kbase_dma_fence_cb {
- struct fence_cb fence_cb;
- struct kbase_jd_atom *katom;
- struct fence *fence;
- struct list_head node;
-};
-
-/**
* struct kbase_dma_fence_resv_info - Structure with list of reservation objects
* @resv_objs: Array of reservation objects to attach the
* new fence to.
@@ -131,11 +117,6 @@ void kbase_dma_fence_term(struct kbase_context *kctx);
*/
int kbase_dma_fence_init(struct kbase_context *kctx);
-/**
- * kbase_dma_fence_waiters_remove()- Remove katom from dma-fence wait list
- * @katom: Pointer to katom to remove from list
- */
-void kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom);
#else /* CONFIG_MALI_DMA_FENCE */
/* Dummy functions for when dma-buf fence isn't enabled. */
diff --git a/mali_kbase/mali_kbase_fence.c b/mali_kbase/mali_kbase_fence.c
new file mode 100644
index 0000000..fcb3733
--- /dev/null
+++ b/mali_kbase/mali_kbase_fence.c
@@ -0,0 +1,196 @@
+/*
+ *
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <linux/atomic.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <mali_kbase_fence_defs.h>
+#include <mali_kbase_fence.h>
+#include <mali_kbase.h>
+
+/* Spin lock protecting all Mali fences as fence->lock. */
+static DEFINE_SPINLOCK(kbase_fence_lock);
+
+static const char *
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+kbase_fence_get_driver_name(struct fence *fence)
+#else
+kbase_fence_get_driver_name(struct dma_fence *fence)
+#endif
+{
+ return kbase_drv_name;
+}
+
+static const char *
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+kbase_fence_get_timeline_name(struct fence *fence)
+#else
+kbase_fence_get_timeline_name(struct dma_fence *fence)
+#endif
+{
+ return kbase_timeline_name;
+}
+
+static bool
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+kbase_fence_enable_signaling(struct fence *fence)
+#else
+kbase_fence_enable_signaling(struct dma_fence *fence)
+#endif
+{
+ return true;
+}
+
+static void
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+kbase_fence_fence_value_str(struct fence *fence, char *str, int size)
+#else
+kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size)
+#endif
+{
+ snprintf(str, size, "%u", fence->seqno);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+const struct fence_ops kbase_fence_ops = {
+ .wait = fence_default_wait,
+#else
+const struct dma_fence_ops kbase_fence_ops = {
+ .wait = dma_fence_default_wait,
+#endif
+ .get_driver_name = kbase_fence_get_driver_name,
+ .get_timeline_name = kbase_fence_get_timeline_name,
+ .enable_signaling = kbase_fence_enable_signaling,
+ .fence_value_str = kbase_fence_fence_value_str
+};
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+struct fence *
+kbase_fence_out_new(struct kbase_jd_atom *katom)
+#else
+struct dma_fence *
+kbase_fence_out_new(struct kbase_jd_atom *katom)
+#endif
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
+
+ WARN_ON(katom->dma_fence.fence);
+
+ fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+ if (!fence)
+ return NULL;
+
+ dma_fence_init(fence,
+ &kbase_fence_ops,
+ &kbase_fence_lock,
+ katom->dma_fence.context,
+ atomic_inc_return(&katom->dma_fence.seqno));
+
+ katom->dma_fence.fence = fence;
+
+ return fence;
+}
+
+bool
+kbase_fence_free_callbacks(struct kbase_jd_atom *katom)
+{
+ struct kbase_fence_cb *cb, *tmp;
+ bool res = false;
+
+ lockdep_assert_held(&katom->kctx->jctx.lock);
+
+ /* Clean up and free callbacks. */
+ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) {
+ bool ret;
+
+ /* Cancel callbacks that hasn't been called yet. */
+ ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb);
+ if (ret) {
+ int ret;
+
+ /* Fence had not signaled, clean up after
+ * canceling.
+ */
+ ret = atomic_dec_return(&katom->dma_fence.dep_count);
+
+ if (unlikely(ret == 0))
+ res = true;
+ }
+
+ /*
+ * Release the reference taken in
+ * kbase_fence_add_callback().
+ */
+ dma_fence_put(cb->fence);
+ list_del(&cb->node);
+ kfree(cb);
+ }
+
+ return res;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+int
+kbase_fence_add_callback(struct kbase_jd_atom *katom,
+ struct fence *fence,
+ fence_func_t callback)
+#else
+int
+kbase_fence_add_callback(struct kbase_jd_atom *katom,
+ struct dma_fence *fence,
+ dma_fence_func_t callback)
+#endif
+{
+ int err = 0;
+ struct kbase_fence_cb *kbase_fence_cb;
+
+ if (!fence)
+ return -EINVAL;
+
+ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL);
+ if (!kbase_fence_cb)
+ return -ENOMEM;
+
+ kbase_fence_cb->fence = fence;
+ kbase_fence_cb->katom = katom;
+ INIT_LIST_HEAD(&kbase_fence_cb->node);
+
+ err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb,
+ callback);
+ if (err == -ENOENT) {
+ /* Fence signaled, clear the error and return */
+ err = 0;
+ kfree(kbase_fence_cb);
+ } else if (err) {
+ kfree(kbase_fence_cb);
+ } else {
+ /*
+ * Get reference to fence that will be kept until callback gets
+ * cleaned up in kbase_fence_free_callbacks().
+ */
+ dma_fence_get(fence);
+ atomic_inc(&katom->dma_fence.dep_count);
+ /* Add callback to katom's list of callbacks */
+ list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks);
+ }
+
+ return err;
+}
diff --git a/mali_kbase/mali_kbase_fence.h b/mali_kbase/mali_kbase_fence.h
new file mode 100644
index 0000000..8d39299
--- /dev/null
+++ b/mali_kbase/mali_kbase_fence.h
@@ -0,0 +1,266 @@
+/*
+ *
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_FENCE_H_
+#define _KBASE_FENCE_H_
+
+/*
+ * mali_kbase_fence.[hc] has common fence code used by both
+ * - CONFIG_MALI_DMA_FENCE - implicit DMA fences
+ * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel
+ */
+
+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
+
+#include <linux/list.h>
+#include "mali_kbase_fence_defs.h"
+#include "mali_kbase.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+extern const struct fence_ops kbase_fence_ops;
+#else
+extern const struct dma_fence_ops kbase_fence_ops;
+#endif
+
+/**
+* struct kbase_fence_cb - Mali dma-fence callback data struct
+* @fence_cb: Callback function
+* @katom: Pointer to katom that is waiting on this callback
+* @fence: Pointer to the fence object on which this callback is waiting
+* @node: List head for linking this callback to the katom
+*/
+struct kbase_fence_cb {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence_cb fence_cb;
+ struct fence *fence;
+#else
+ struct dma_fence_cb fence_cb;
+ struct dma_fence *fence;
+#endif
+ struct kbase_jd_atom *katom;
+ struct list_head node;
+};
+
+/**
+ * kbase_fence_out_new() - Creates a new output fence and puts it on the atom
+ * @katom: Atom to create an output fence for
+ *
+ * return: A new fence object on success, NULL on failure.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom);
+#else
+struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom);
+#endif
+
+#if defined(CONFIG_SYNC_FILE)
+/**
+ * kbase_fence_fence_in_set() - Assign input fence to atom
+ * @katom: Atom to assign input fence to
+ * @fence: Input fence to assign to atom
+ *
+ * This function will take ownership of one fence reference!
+ */
+#define kbase_fence_fence_in_set(katom, fence) \
+ do { \
+ WARN_ON((katom)->dma_fence.fence_in); \
+ (katom)->dma_fence.fence_in = fence; \
+ } while (0)
+#endif
+
+/**
+ * kbase_fence_out_remove() - Removes the output fence from atom
+ * @katom: Atom to remove output fence for
+ *
+ * This will also release the reference to this fence which the atom keeps
+ */
+static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom)
+{
+ if (katom->dma_fence.fence) {
+ dma_fence_put(katom->dma_fence.fence);
+ katom->dma_fence.fence = NULL;
+ }
+}
+
+#if defined(CONFIG_SYNC_FILE)
+/**
+ * kbase_fence_out_remove() - Removes the input fence from atom
+ * @katom: Atom to remove input fence for
+ *
+ * This will also release the reference to this fence which the atom keeps
+ */
+static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom)
+{
+ if (katom->dma_fence.fence_in) {
+ dma_fence_put(katom->dma_fence.fence_in);
+ katom->dma_fence.fence_in = NULL;
+ }
+}
+#endif
+
+/**
+ * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us
+ * @katom: Atom to check output fence for
+ *
+ * Return: true if fence exists and is valid, otherwise false
+ */
+static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom)
+{
+ return katom->dma_fence.fence &&
+ katom->dma_fence.fence->ops == &kbase_fence_ops;
+}
+
+/**
+ * kbase_fence_out_signal() - Signal output fence of atom
+ * @katom: Atom to signal output fence for
+ * @status: Status to signal with (0 for success, < 0 for error)
+ *
+ * Return: 0 on success, < 0 on error
+ */
+static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom,
+ int status)
+{
+ katom->dma_fence.fence->status = status;
+ return dma_fence_signal(katom->dma_fence.fence);
+}
+
+/**
+ * kbase_fence_add_callback() - Add callback on @fence to block @katom
+ * @katom: Pointer to katom that will be blocked by @fence
+ * @fence: Pointer to fence on which to set up the callback
+ * @callback: Pointer to function to be called when fence is signaled
+ *
+ * Caller needs to hold a reference to @fence when calling this function, and
+ * the caller is responsible for releasing that reference. An additional
+ * reference to @fence will be taken when the callback was successfully set up
+ * and @fence needs to be kept valid until the callback has been called and
+ * cleanup have been done.
+ *
+ * Return: 0 on success: fence was either already signaled, or callback was
+ * set up. Negative error code is returned on error.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+int kbase_fence_add_callback(struct kbase_jd_atom *katom,
+ struct fence *fence,
+ fence_func_t callback);
+#else
+int kbase_fence_add_callback(struct kbase_jd_atom *katom,
+ struct dma_fence *fence,
+ dma_fence_func_t callback);
+#endif
+
+/**
+ * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value
+ * @katom: Atom to set dep_count for
+ * @val: value to set dep_count to
+ *
+ * The dep_count is available to the users of this module so that they can
+ * synchronize completion of the wait with cancellation and adding of more
+ * callbacks. For instance, a user could do the following:
+ *
+ * dep_count set to 1
+ * callback #1 added, dep_count is increased to 2
+ * callback #1 happens, dep_count decremented to 1
+ * since dep_count > 0, no completion is done
+ * callback #2 is added, dep_count is increased to 2
+ * dep_count decremented to 1
+ * callback #2 happens, dep_count decremented to 0
+ * since dep_count now is zero, completion executes
+ *
+ * The dep_count can also be used to make sure that the completion only
+ * executes once. This is typically done by setting dep_count to -1 for the
+ * thread that takes on this responsibility.
+ */
+static inline void
+kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val)
+{
+ atomic_set(&katom->dma_fence.dep_count, val);
+}
+
+/**
+ * kbase_fence_dep_count_dec_and_test() - Decrements dep_count
+ * @katom: Atom to decrement dep_count for
+ *
+ * See @kbase_fence_dep_count_set for general description about dep_count
+ *
+ * Return: true if value was decremented to zero, otherwise false
+ */
+static inline bool
+kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom)
+{
+ return atomic_dec_and_test(&katom->dma_fence.dep_count);
+}
+
+/**
+ * kbase_fence_dep_count_read() - Returns the current dep_count value
+ * @katom: Pointer to katom
+ *
+ * See @kbase_fence_dep_count_set for general description about dep_count
+ *
+ * Return: The current dep_count value
+ */
+static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom)
+{
+ return atomic_read(&katom->dma_fence.dep_count);
+}
+
+/**
+ * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom
+ * @katom: Pointer to katom
+ *
+ * This function will free all fence callbacks on the katom's list of
+ * callbacks. Callbacks that have not yet been called, because their fence
+ * hasn't yet signaled, will first be removed from the fence.
+ *
+ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held.
+ *
+ * Return: true if dep_count reached 0, otherwise false.
+ */
+bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom);
+
+#if defined(CONFIG_SYNC_FILE)
+/**
+ * kbase_fence_in_get() - Retrieve input fence for atom.
+ * @katom: Atom to get input fence from
+ *
+ * A ref will be taken for the fence, so use @kbase_fence_put() to release it
+ *
+ * Return: The fence, or NULL if there is no input fence for atom
+ */
+#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in)
+#endif
+
+/**
+ * kbase_fence_out_get() - Retrieve output fence for atom.
+ * @katom: Atom to get output fence from
+ *
+ * A ref will be taken for the fence, so use @kbase_fence_put() to release it
+ *
+ * Return: The fence, or NULL if there is no output fence for atom
+ */
+#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence)
+
+/**
+ * kbase_fence_put() - Releases a reference to a fence
+ * @fence: Fence to release reference for.
+ */
+#define kbase_fence_put(fence) dma_fence_put(fence)
+
+
+#endif /* CONFIG_MALI_DMA_FENCE || defined(CONFIG_SYNC_FILE */
+
+#endif /* _KBASE_FENCE_H_ */
diff --git a/mali_kbase/mali_kbase_fence_defs.h b/mali_kbase/mali_kbase_fence_defs.h
new file mode 100644
index 0000000..fa2c6df
--- /dev/null
+++ b/mali_kbase/mali_kbase_fence_defs.h
@@ -0,0 +1,51 @@
+/*
+ *
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_FENCE_DEFS_H_
+#define _KBASE_FENCE_DEFS_H_
+
+/*
+ * There was a big rename in the 4.10 kernel (fence* -> dma_fence*)
+ * This file hides the compatibility issues with this for the rest the driver
+ */
+
+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+
+#include <linux/fence.h>
+
+#define dma_fence_context_alloc(a) fence_context_alloc(a)
+#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e)
+#define dma_fence_get(a) fence_get(a)
+#define dma_fence_put(a) fence_put(a)
+#define dma_fence_signal(a) fence_signal(a)
+#define dma_fence_is_signaled(a) fence_is_signaled(a)
+#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c)
+#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b)
+
+#else
+
+#include <linux/dma-fence.h>
+
+#endif /* < 4.10.0 */
+
+#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */
+
+#endif /* _KBASE_FENCE_DEFS_H_ */
diff --git a/mali_kbase/mali_kbase_gator_hwcnt_names.h b/mali_kbase/mali_kbase_gator_hwcnt_names.h
index e4003f6..cad19b6 100644
--- a/mali_kbase/mali_kbase_gator_hwcnt_names.h
+++ b/mali_kbase/mali_kbase_gator_hwcnt_names.h
@@ -2163,4 +2163,8 @@ static const char * const hardware_counters_mali_t88x[] = {
#include "mali_kbase_gator_hwcnt_names_tsix.h"
+#ifdef MALI_INCLUDE_TKAX
+#include "mali_kbase_gator_hwcnt_names_tkax.h"
+#endif /* MALI_INCLUDE_TKAX */
+
#endif
diff --git a/mali_kbase/mali_kbase_gpu_id.h b/mali_kbase/mali_kbase_gpu_id.h
index 60b4371..42f0111 100644
--- a/mali_kbase/mali_kbase_gpu_id.h
+++ b/mali_kbase/mali_kbase_gpu_id.h
@@ -101,6 +101,12 @@
#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0)
#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1)
#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0)
+#ifdef MALI_INCLUDE_TKAX
+#define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(9u, 0)
+#endif /* MALI_INCLUDE_TKAX */
+#ifdef MALI_INCLUDE_TTRX
+#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(10u, 0)
+#endif /* MALI_INCLUDE_TTRX */
/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */
#define GPU_ID_S_15DEV0 0x1
diff --git a/mali_kbase/mali_kbase_gpuprops.c b/mali_kbase/mali_kbase_gpuprops.c
index fc65223..e2f4209 100644
--- a/mali_kbase/mali_kbase_gpuprops.c
+++ b/mali_kbase/mali_kbase_gpuprops.c
@@ -26,6 +26,7 @@
#include <mali_kbase_gpuprops.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_hwaccess_gpuprops.h>
+#include "mali_kbase_ioctl.h"
#include <linux/clk.h>
/**
@@ -214,6 +215,14 @@ static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kb
gpu_props->raw_props.thread_features = regdump.thread_features;
}
+void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props)
+{
+ gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4);
+ gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8);
+ gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4);
+ gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16);
+}
+
/**
* kbase_gpuprops_calculate_props - Calculate the derived properties
* @gpu_props: The &base_gpu_props structure
@@ -227,10 +236,7 @@ static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, str
int i;
/* Populate the base_gpu_props structure */
- gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4);
- gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8);
- gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4);
- gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16);
+ kbase_gpuprops_update_core_props_gpu_id(gpu_props);
gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2;
gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT;
@@ -327,3 +333,178 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev)
gpu_props->raw_props.coherency_mode = regdump.coherency_features |
COHERENCY_FEATURE_BIT(COHERENCY_NONE);
}
+
+static struct {
+ u32 type;
+ size_t offset;
+ int size;
+} gpu_property_mapping[] = {
+#define PROP(name, member) \
+ {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \
+ sizeof(((struct mali_base_gpu_props *)0)->member)}
+ PROP(PRODUCT_ID, core_props.product_id),
+ PROP(VERSION_STATUS, core_props.version_status),
+ PROP(MINOR_REVISION, core_props.minor_revision),
+ PROP(MAJOR_REVISION, core_props.major_revision),
+ PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz),
+ PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max),
+ PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min),
+ PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size),
+ PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]),
+ PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]),
+ PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]),
+ PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size),
+
+ PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size),
+ PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size),
+ PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices),
+
+ PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes),
+ PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels),
+
+ PROP(MAX_THREADS, thread_props.max_threads),
+ PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size),
+ PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size),
+ PROP(MAX_REGISTERS, thread_props.max_registers),
+ PROP(MAX_TASK_QUEUE, thread_props.max_task_queue),
+ PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split),
+ PROP(IMPL_TECH, thread_props.impl_tech),
+
+ PROP(RAW_SHADER_PRESENT, raw_props.shader_present),
+ PROP(RAW_TILER_PRESENT, raw_props.tiler_present),
+ PROP(RAW_L2_PRESENT, raw_props.l2_present),
+ PROP(RAW_STACK_PRESENT, raw_props.stack_present),
+ PROP(RAW_L2_FEATURES, raw_props.l2_features),
+ PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size),
+ PROP(RAW_MEM_FEATURES, raw_props.mem_features),
+ PROP(RAW_MMU_FEATURES, raw_props.mmu_features),
+ PROP(RAW_AS_PRESENT, raw_props.as_present),
+ PROP(RAW_JS_PRESENT, raw_props.js_present),
+ PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]),
+ PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]),
+ PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]),
+ PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]),
+ PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]),
+ PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]),
+ PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]),
+ PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]),
+ PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]),
+ PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]),
+ PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]),
+ PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]),
+ PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]),
+ PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]),
+ PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]),
+ PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]),
+ PROP(RAW_TILER_FEATURES, raw_props.tiler_features),
+ PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]),
+ PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]),
+ PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]),
+ PROP(RAW_GPU_ID, raw_props.gpu_id),
+ PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads),
+ PROP(RAW_THREAD_MAX_WORKGROUP_SIZE,
+ raw_props.thread_max_workgroup_size),
+ PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size),
+ PROP(RAW_THREAD_FEATURES, raw_props.thread_features),
+ PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode),
+
+ PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups),
+ PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups),
+ PROP(COHERENCY_COHERENCY, coherency_info.coherency),
+ PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask),
+ PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask),
+ PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask),
+ PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask),
+ PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask),
+ PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask),
+ PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask),
+ PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask),
+ PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask),
+ PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask),
+ PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask),
+ PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask),
+ PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask),
+ PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask),
+ PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask),
+ PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask),
+
+#undef PROP
+};
+
+int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev)
+{
+ struct kbase_gpu_props *kprops = &kbdev->gpu_props;
+ struct mali_base_gpu_props *props = &kprops->props;
+ u32 count = ARRAY_SIZE(gpu_property_mapping);
+ u32 i;
+ u32 size = 0;
+ u8 *p;
+
+ for (i = 0; i < count; i++) {
+ /* 4 bytes for the ID, and the size of the property */
+ size += 4 + gpu_property_mapping[i].size;
+ }
+
+ kprops->prop_buffer_size = size;
+ kprops->prop_buffer = kmalloc(size, GFP_KERNEL);
+
+ if (!kprops->prop_buffer) {
+ kprops->prop_buffer_size = 0;
+ return -ENOMEM;
+ }
+
+ p = kprops->prop_buffer;
+
+#define WRITE_U8(v) (*p++ = (v) & 0xFF)
+#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0)
+#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0)
+#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0)
+
+ for (i = 0; i < count; i++) {
+ u32 type = gpu_property_mapping[i].type;
+ u8 type_size;
+ void *field = ((u8 *)props) + gpu_property_mapping[i].offset;
+
+ switch (gpu_property_mapping[i].size) {
+ case 1:
+ type_size = KBASE_GPUPROP_VALUE_SIZE_U8;
+ break;
+ case 2:
+ type_size = KBASE_GPUPROP_VALUE_SIZE_U16;
+ break;
+ case 4:
+ type_size = KBASE_GPUPROP_VALUE_SIZE_U32;
+ break;
+ case 8:
+ type_size = KBASE_GPUPROP_VALUE_SIZE_U64;
+ break;
+ default:
+ dev_err(kbdev->dev,
+ "Invalid gpu_property_mapping type=%d size=%d",
+ type, gpu_property_mapping[i].size);
+ return -EINVAL;
+ }
+
+ WRITE_U32((type<<2) | type_size);
+
+ switch (type_size) {
+ case KBASE_GPUPROP_VALUE_SIZE_U8:
+ WRITE_U8(*((u8 *)field));
+ break;
+ case KBASE_GPUPROP_VALUE_SIZE_U16:
+ WRITE_U16(*((u16 *)field));
+ break;
+ case KBASE_GPUPROP_VALUE_SIZE_U32:
+ WRITE_U32(*((u32 *)field));
+ break;
+ case KBASE_GPUPROP_VALUE_SIZE_U64:
+ WRITE_U64(*((u64 *)field));
+ break;
+ default: /* Cannot be reached */
+ WARN_ON(1);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
diff --git a/mali_kbase/mali_kbase_gpuprops.h b/mali_kbase/mali_kbase_gpuprops.h
index 44b5e9e..57b3eaf 100644
--- a/mali_kbase/mali_kbase_gpuprops.h
+++ b/mali_kbase/mali_kbase_gpuprops.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2015, 2017 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -61,4 +61,24 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev);
*/
int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props);
+/**
+ * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer
+ * @kbdev: The kbase device
+ *
+ * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user
+ * space to read.
+ */
+int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev);
+
+/**
+ * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value
+ * @gpu_props: the &base_gpu_props structure
+ *
+ * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into
+ * separate fields (version_status, minor_revision, major_revision, product_id)
+ * stored in base_gpu_props::core_props.
+ */
+void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props);
+
+
#endif /* _KBASE_GPUPROPS_H_ */
diff --git a/mali_kbase/mali_kbase_gpuprops_types.h b/mali_kbase/mali_kbase_gpuprops_types.h
index b06e511..10794fc 100644
--- a/mali_kbase/mali_kbase_gpuprops_types.h
+++ b/mali_kbase/mali_kbase_gpuprops_types.h
@@ -84,6 +84,9 @@ struct kbase_gpu_props {
/* Properties shared with userspace */
base_gpu_props props;
+
+ u32 prop_buffer_size;
+ void *prop_buffer;
};
#endif /* _KBASE_GPUPROPS_TYPES_H_ */
diff --git a/mali_kbase/mali_kbase_hw.c b/mali_kbase/mali_kbase_hw.c
index 89fe0d8..9a390d2 100644
--- a/mali_kbase/mali_kbase_hw.c
+++ b/mali_kbase/mali_kbase_hw.c
@@ -48,6 +48,16 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev)
case GPU_ID2_PRODUCT_TSIX:
features = base_hw_features_tSIx;
break;
+#ifdef MALI_INCLUDE_TKAX
+ case GPU_ID2_PRODUCT_TKAX:
+ features = base_hw_features_tKAx;
+ break;
+#endif /* MALI_INCLUDE_TKAX */
+#ifdef MALI_INCLUDE_TTRX
+ case GPU_ID2_PRODUCT_TTRX:
+ features = base_hw_features_tTRx;
+ break;
+#endif /* MALI_INCLUDE_TTRX */
default:
features = base_hw_features_generic;
break;
@@ -87,6 +97,142 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev)
set_bit(*features, &kbdev->hw_features_mask[0]);
}
+/**
+ * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID
+ * @kbdev: Device pointer
+ *
+ * Return: pointer to an array of hardware issues, terminated by
+ * BASE_HW_ISSUE_END.
+ *
+ * This function can only be used on new-format GPU IDs, i.e. those for which
+ * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev.
+ *
+ * In debugging versions of the driver, unknown versions of a known GPU will
+ * be treated as the most recent known version not later than the actual
+ * version. In such circumstances, the GPU ID in @kbdev will also be replaced
+ * with the most recent known version.
+ *
+ * Note: The GPU configuration must have been read by kbase_gpuprops_get_props()
+ * before calling this function.
+ */
+static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(
+ struct kbase_device *kbdev)
+{
+ const enum base_hw_issue *issues = NULL;
+
+ struct base_hw_product {
+ u32 product_model;
+ struct {
+ u32 version;
+ const enum base_hw_issue *issues;
+ } map[7];
+ };
+
+ static const struct base_hw_product base_hw_products[] = {
+ {GPU_ID2_PRODUCT_TMIX,
+ {{GPU_ID2_VERSION_MAKE(0, 0, 1),
+ base_hw_issues_tMIx_r0p0_05dev0},
+ {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0},
+ {U32_MAX /* sentinel value */, NULL} } },
+
+ {GPU_ID2_PRODUCT_THEX,
+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0},
+ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0},
+ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1},
+ {U32_MAX, NULL} } },
+
+ {GPU_ID2_PRODUCT_TSIX,
+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0},
+ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0},
+ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1},
+ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tSIx_r0p1},
+ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0},
+ {GPU_ID2_VERSION_MAKE(1, 0, 1), base_hw_issues_tSIx_r1p0},
+ {U32_MAX, NULL} } },
+
+
+#ifdef MALI_INCLUDE_TKAX
+ {GPU_ID2_PRODUCT_TKAX,
+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tKAx_r0p0},
+ {U32_MAX, NULL} } },
+#endif /* MALI_INCLUDE_TKAX */
+
+#ifdef MALI_INCLUDE_TTRX
+ {GPU_ID2_PRODUCT_TTRX,
+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0},
+ {U32_MAX, NULL} } },
+#endif /* MALI_INCLUDE_TTRX */
+ };
+
+ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
+ const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL;
+ const struct base_hw_product *product = NULL;
+ size_t p;
+
+ /* Stop when we reach the end of the products array. */
+ for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) {
+ if (product_model == base_hw_products[p].product_model) {
+ product = &base_hw_products[p];
+ break;
+ }
+ }
+
+ if (product != NULL) {
+ /* Found a matching product. */
+ const u32 version = gpu_id & GPU_ID2_VERSION;
+ u32 fallback_version = 0;
+ const enum base_hw_issue *fallback_issues = NULL;
+ size_t v;
+
+ /* Stop when we reach the end of the map. */
+ for (v = 0; product->map[v].version != U32_MAX; ++v) {
+
+ if (version == product->map[v].version) {
+ /* Exact match so stop. */
+ issues = product->map[v].issues;
+ break;
+ }
+
+ /* Check whether this is a candidate for most recent
+ known version not later than the actual
+ version. */
+ if ((version > product->map[v].version) &&
+ (product->map[v].version >= fallback_version)) {
+ fallback_version = product->map[v].version;
+ fallback_issues = product->map[v].issues;
+ }
+ }
+
+ if ((issues == NULL) && (fallback_issues != NULL)) {
+ /* Fall back to the issue set of the most recent known
+ version not later than the actual version. */
+ issues = fallback_issues;
+
+ dev_info(kbdev->dev,
+ "r%dp%d status %d is unknown; treating as r%dp%d status %d",
+ (gpu_id & GPU_ID2_VERSION_MAJOR) >>
+ GPU_ID2_VERSION_MAJOR_SHIFT,
+ (gpu_id & GPU_ID2_VERSION_MINOR) >>
+ GPU_ID2_VERSION_MINOR_SHIFT,
+ (gpu_id & GPU_ID2_VERSION_STATUS) >>
+ GPU_ID2_VERSION_STATUS_SHIFT,
+ (fallback_version & GPU_ID2_VERSION_MAJOR) >>
+ GPU_ID2_VERSION_MAJOR_SHIFT,
+ (fallback_version & GPU_ID2_VERSION_MINOR) >>
+ GPU_ID2_VERSION_MINOR_SHIFT,
+ (fallback_version & GPU_ID2_VERSION_STATUS) >>
+ GPU_ID2_VERSION_STATUS_SHIFT);
+
+ gpu_id &= ~GPU_ID2_VERSION;
+ gpu_id |= fallback_version;
+ kbdev->gpu_props.props.raw_props.gpu_id = gpu_id;
+
+ kbase_gpuprops_update_core_props_gpu_id(&kbdev->gpu_props.props);
+ }
+ }
+ return issues;
+}
+
int kbase_hw_set_issues_mask(struct kbase_device *kbdev)
{
const enum base_hw_issue *issues;
@@ -101,46 +247,17 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev)
if (impl_tech != IMPLEMENTATION_MODEL) {
if (GPU_ID_IS_NEW_FORMAT(product_id)) {
- switch (gpu_id) {
- case GPU_ID2_MAKE(6, 0, 10, 0, 0, 0, 1):
- issues = base_hw_issues_tMIx_r0p0_05dev0;
- break;
- case GPU_ID2_MAKE(6, 0, 10, 0, 0, 0, 2):
- issues = base_hw_issues_tMIx_r0p0;
- break;
- default:
- switch (gpu_id & GPU_ID2_PRODUCT_MODEL) {
- case GPU_ID2_PRODUCT_TMIX:
- issues = base_hw_issues_tMIx_r0p0;
- break;
- case GPU_ID2_PRODUCT_THEX:
- issues = base_hw_issues_tHEx_r0p0;
- break;
- case GPU_ID2_PRODUCT_TSIX:
- switch (gpu_id & (GPU_ID2_VERSION_MAJOR |
- GPU_ID2_VERSION_MINOR)) {
- case GPU_ID2_VERSION_MAKE(0, 0, 0):
- issues = base_hw_issues_tSIx_r0p0;
- break;
- case GPU_ID2_VERSION_MAKE(0, 1, 0):
- issues = base_hw_issues_tSIx_r0p1;
- break;
- case GPU_ID2_VERSION_MAKE(1, 0, 0):
- issues = base_hw_issues_tSIx_r1p0;
- break;
- default:
- dev_err(kbdev->dev,
- "Unknown GPU ID %x",
- gpu_id);
- return -EINVAL;
- }
- break;
- default:
- dev_err(kbdev->dev,
- "Unknown GPU ID %x", gpu_id);
- return -EINVAL;
- }
+ issues = kbase_hw_get_issues_for_new_id(kbdev);
+ if (issues == NULL) {
+ dev_err(kbdev->dev,
+ "Unknown GPU ID %x", gpu_id);
+ return -EINVAL;
}
+
+ /* The GPU ID might have been replaced with the last
+ known version of the same GPU. */
+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
+
} else {
switch (gpu_id) {
case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0):
@@ -250,6 +367,16 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev)
case GPU_ID2_PRODUCT_TSIX:
issues = base_hw_issues_model_tSIx;
break;
+#ifdef MALI_INCLUDE_TKAX
+ case GPU_ID2_PRODUCT_TKAX:
+ issues = base_hw_issues_model_tKAx;
+ break;
+#endif /* MALI_INCLUDE_TKAX */
+#ifdef MALI_INCLUDE_TTRX
+ case GPU_ID2_PRODUCT_TTRX:
+ issues = base_hw_issues_model_tTRx;
+ break;
+#endif /* MALI_INCLUDE_TTRX */
default:
dev_err(kbdev->dev,
"Unknown GPU ID %x", gpu_id);
@@ -289,7 +416,35 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev)
}
}
- dev_info(kbdev->dev, "GPU identified as 0x%04x r%dp%d status %d", (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, (gpu_id & GPU_ID_VERSION_STATUS) >> GPU_ID_VERSION_STATUS_SHIFT);
+ if (GPU_ID_IS_NEW_FORMAT(product_id)) {
+ dev_info(kbdev->dev,
+ "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d",
+ (gpu_id & GPU_ID2_PRODUCT_MAJOR) >>
+ GPU_ID2_PRODUCT_MAJOR_SHIFT,
+ (gpu_id & GPU_ID2_ARCH_MAJOR) >>
+ GPU_ID2_ARCH_MAJOR_SHIFT,
+ (gpu_id & GPU_ID2_ARCH_MINOR) >>
+ GPU_ID2_ARCH_MINOR_SHIFT,
+ (gpu_id & GPU_ID2_ARCH_REV) >>
+ GPU_ID2_ARCH_REV_SHIFT,
+ (gpu_id & GPU_ID2_VERSION_MAJOR) >>
+ GPU_ID2_VERSION_MAJOR_SHIFT,
+ (gpu_id & GPU_ID2_VERSION_MINOR) >>
+ GPU_ID2_VERSION_MINOR_SHIFT,
+ (gpu_id & GPU_ID2_VERSION_STATUS) >>
+ GPU_ID2_VERSION_STATUS_SHIFT);
+ } else {
+ dev_info(kbdev->dev,
+ "GPU identified as 0x%04x r%dp%d status %d",
+ (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
+ GPU_ID_VERSION_PRODUCT_ID_SHIFT,
+ (gpu_id & GPU_ID_VERSION_MAJOR) >>
+ GPU_ID_VERSION_MAJOR_SHIFT,
+ (gpu_id & GPU_ID_VERSION_MINOR) >>
+ GPU_ID_VERSION_MINOR_SHIFT,
+ (gpu_id & GPU_ID_VERSION_STATUS) >>
+ GPU_ID_VERSION_STATUS_SHIFT);
+ }
for (; *issues != BASE_HW_ISSUE_END; issues++)
set_bit(*issues, &kbdev->hw_issues_mask[0]);
diff --git a/mali_kbase/mali_kbase_hw.h b/mali_kbase/mali_kbase_hw.h
index fce7d29..754250c 100644
--- a/mali_kbase/mali_kbase_hw.h
+++ b/mali_kbase/mali_kbase_hw.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -40,7 +40,20 @@
test_bit(feature, &(kbdev)->hw_features_mask[0])
/**
- * @brief Set the HW issues mask depending on the GPU ID
+ * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID
+ * @kbdev: Device pointer
+ *
+ * Return: 0 if the GPU ID was recognized, otherwise -EINVAL.
+ *
+ * The GPU ID is read from the @kbdev.
+ *
+ * In debugging versions of the driver, unknown versions of a known GPU with a
+ * new-format ID will be treated as the most recent known version not later
+ * than the actual version. In such circumstances, the GPU ID in @kbdev will
+ * also be replaced with the most recent known version.
+ *
+ * Note: The GPU configuration must have been read by
+ * kbase_gpuprops_get_props() before calling this function.
*/
int kbase_hw_set_issues_mask(struct kbase_device *kbdev);
diff --git a/mali_kbase/mali_kbase_hwaccess_jm.h b/mali_kbase/mali_kbase_hwaccess_jm.h
index 821b68a..750fda2 100644
--- a/mali_kbase/mali_kbase_hwaccess_jm.h
+++ b/mali_kbase/mali_kbase_hwaccess_jm.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -45,33 +45,22 @@ void kbase_backend_run_atom(struct kbase_device *kbdev,
void kbase_backend_slot_update(struct kbase_device *kbdev);
/**
- * kbase_backend_find_free_address_space() - Find a free address space.
+ * kbase_backend_find_and_release_free_address_space() - Release a free AS
* @kbdev: Device pointer
* @kctx: Context pointer
*
- * If no address spaces are currently free, then this function can evict an
- * idle context from the runpool, freeing up the address space it was using.
+ * This function can evict an idle context from the runpool, freeing up the
+ * address space it was using.
*
* The address space is marked as in use. The caller must either assign a
* context using kbase_gpu_use_ctx(), or release it using
- * kbase_gpu_release_free_address_space()
+ * kbase_ctx_sched_release()
*
* Return: Number of free address space, or KBASEP_AS_NR_INVALID if none
* available
*/
-int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
- struct kbase_context *kctx);
-
-/**
- * kbase_backend_release_free_address_space() - Release an address space.
- * @kbdev: Device pointer
- * @as_nr: Address space to release
- *
- * The address space must have been returned by
- * kbase_gpu_find_free_address_space().
- */
-void kbase_backend_release_free_address_space(struct kbase_device *kbdev,
- int as_nr);
+int kbase_backend_find_and_release_free_address_space(
+ struct kbase_device *kbdev, struct kbase_context *kctx);
/**
* kbase_backend_use_ctx() - Activate a currently unscheduled context, using the
@@ -387,4 +376,6 @@ bool kbase_reset_gpu_active(struct kbase_device *kbdev);
void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
struct kbase_jd_atom *target_katom);
+extern struct protected_mode_ops kbase_native_protected_ops;
+
#endif /* _KBASE_HWACCESS_JM_H_ */
diff --git a/mali_kbase/mali_kbase_ioctl.h b/mali_kbase/mali_kbase_ioctl.h
new file mode 100644
index 0000000..dcbed9c
--- /dev/null
+++ b/mali_kbase/mali_kbase_ioctl.h
@@ -0,0 +1,656 @@
+/*
+ *
+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_IOCTL_H_
+#define _KBASE_IOCTL_H_
+
+#ifdef __cpluscplus
+extern "C" {
+#endif
+
+#include <linux/types.h>
+
+#define KBASE_IOCTL_TYPE 0x80
+
+#ifdef ANDROID
+/* Android's definition of ioctl is incorrect, specifying the type argument as
+ * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work
+ * round this by redefining _IOC to include a case to 'int'.
+ */
+#undef _IOC
+#define _IOC(dir, type, nr, size) \
+ ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \
+ ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT)))
+#endif
+
+/**
+ * struct kbase_ioctl_version_check - Check version compatibility with kernel
+ *
+ * @major: Major version number
+ * @minor: Minor version number
+ */
+struct kbase_ioctl_version_check {
+ __u16 major;
+ __u16 minor;
+};
+
+#define KBASE_IOCTL_VERSION_CHECK \
+ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check)
+
+/**
+ * struct kbase_ioctl_set_flags - Set kernel context creation flags
+ *
+ * @create_flags: Flags - see base_context_create_flags
+ */
+struct kbase_ioctl_set_flags {
+ __u32 create_flags;
+};
+
+#define KBASE_IOCTL_SET_FLAGS \
+ _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags)
+
+/**
+ * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel
+ *
+ * @addr: Memory address of an array of struct base_jd_atom_v2
+ * @nr_atoms: Number of entries in the array
+ * @stride: sizeof(struct base_jd_atom_v2)
+ */
+struct kbase_ioctl_job_submit {
+ union kbase_pointer addr;
+ __u32 nr_atoms;
+ __u32 stride;
+};
+
+#define KBASE_IOCTL_JOB_SUBMIT \
+ _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit)
+
+/**
+ * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel
+ *
+ * @buffer: Pointer to the buffer to store properties into
+ * @size: Size of the buffer
+ * @flags: Flags - must be zero for now
+ *
+ * The ioctl will return the number of bytes stored into @buffer or an error
+ * on failure (e.g. @size is too small). If @size is specified as 0 then no
+ * data will be written but the return value will be the number of bytes needed
+ * for all the properties.
+ *
+ * @flags may be used in the future to request a different format for the
+ * buffer. With @flags == 0 the following format is used.
+ *
+ * The buffer will be filled with pairs of values, a u32 key identifying the
+ * property followed by the value. The size of the value is identified using
+ * the bottom bits of the key. The value then immediately followed the key and
+ * is tightly packed (there is no padding). All keys and values are
+ * little-endian.
+ *
+ * 00 = u8
+ * 01 = u16
+ * 10 = u32
+ * 11 = u64
+ */
+struct kbase_ioctl_get_gpuprops {
+ union kbase_pointer buffer;
+ __u32 size;
+ __u32 flags;
+};
+
+#define KBASE_IOCTL_GET_GPUPROPS \
+ _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops)
+
+#define KBASE_IOCTL_POST_TERM \
+ _IO(KBASE_IOCTL_TYPE, 4)
+
+/**
+ * union kbase_ioctl_mem_alloc - Allocate memory on the GPU
+ *
+ * @va_pages: The number of pages of virtual address space to reserve
+ * @commit_pages: The number of physical pages to allocate
+ * @extent: The number of extra pages to allocate on each GPU fault which grows
+ * the region
+ * @flags: Flags
+ * @gpu_va: The GPU virtual address which is allocated
+ *
+ * @in: Input parameters
+ * @out: Output parameters
+ */
+union kbase_ioctl_mem_alloc {
+ struct {
+ __u64 va_pages;
+ __u64 commit_pages;
+ __u64 extent;
+ __u64 flags;
+ } in;
+ struct {
+ __u64 flags;
+ __u64 gpu_va;
+ } out;
+};
+
+#define KBASE_IOCTL_MEM_ALLOC \
+ _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc)
+
+/**
+ * struct kbase_ioctl_mem_query - Query properties of a GPU memory region
+ * @gpu_addr: A GPU address contained within the region
+ * @query: The type of query
+ * @value: The result of the query
+ *
+ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query.
+ *
+ * @in: Input parameters
+ * @out: Output parameters
+ */
+union kbase_ioctl_mem_query {
+ struct {
+ __u64 gpu_addr;
+ __u64 query;
+ } in;
+ struct {
+ __u64 value;
+ } out;
+};
+
+#define KBASE_IOCTL_MEM_QUERY \
+ _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query)
+
+#define KBASE_MEM_QUERY_COMMIT_SIZE 1
+#define KBASE_MEM_QUERY_VA_SIZE 2
+#define KBASE_MEM_QUERY_FLAGS 3
+
+/**
+ * struct kbase_ioctl_mem_free - Free a memory region
+ * @gpu_addr: Handle to the region to free
+ */
+struct kbase_ioctl_mem_free {
+ __u64 gpu_addr;
+};
+
+#define KBASE_IOCTL_MEM_FREE \
+ _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free)
+
+/**
+ * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader
+ * @buffer_count: requested number of dumping buffers
+ * @jm_bm: counters selection bitmask (JM)
+ * @shader_bm: counters selection bitmask (Shader)
+ * @tiler_bm: counters selection bitmask (Tiler)
+ * @mmu_l2_bm: counters selection bitmask (MMU_L2)
+ *
+ * A fd is returned from the ioctl if successful, or a negative value on error
+ */
+struct kbase_ioctl_hwcnt_reader_setup {
+ __u32 buffer_count;
+ __u32 jm_bm;
+ __u32 shader_bm;
+ __u32 tiler_bm;
+ __u32 mmu_l2_bm;
+};
+
+#define KBASE_IOCTL_HWCNT_READER_SETUP \
+ _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup)
+
+/**
+ * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection
+ * @dump_buffer: GPU address to write counters to
+ * @jm_bm: counters selection bitmask (JM)
+ * @shader_bm: counters selection bitmask (Shader)
+ * @tiler_bm: counters selection bitmask (Tiler)
+ * @mmu_l2_bm: counters selection bitmask (MMU_L2)
+ */
+struct kbase_ioctl_hwcnt_enable {
+ __u64 dump_buffer;
+ __u32 jm_bm;
+ __u32 shader_bm;
+ __u32 tiler_bm;
+ __u32 mmu_l2_bm;
+};
+
+#define KBASE_IOCTL_HWCNT_ENABLE \
+ _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable)
+
+#define KBASE_IOCTL_HWCNT_DUMP \
+ _IO(KBASE_IOCTL_TYPE, 10)
+
+#define KBASE_IOCTL_HWCNT_CLEAR \
+ _IO(KBASE_IOCTL_TYPE, 11)
+
+/**
+ * struct kbase_ioctl_disjoint_query - Query the disjoint counter
+ * @counter: A counter of disjoint events in the kernel
+ */
+struct kbase_ioctl_disjoint_query {
+ __u32 counter;
+};
+
+#define KBASE_IOCTL_DISJOINT_QUERY \
+ _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query)
+
+/**
+ * struct kbase_ioctl_get_ddk_version - Query the kernel version
+ * @version_buffer: Buffer to receive the kernel version string
+ * @size: Size of the buffer
+ *
+ * The ioctl will return the number of bytes written into version_buffer
+ * (which includes a NULL byte) or a negative error code
+ */
+struct kbase_ioctl_get_ddk_version {
+ union kbase_pointer version_buffer;
+ __u32 size;
+};
+
+#define KBASE_IOCTL_GET_DDK_VERSION \
+ _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version)
+
+/**
+ * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator
+ *
+ * @va_pages: Number of VA pages to reserve for JIT
+ *
+ * Note that depending on the VA size of the application and GPU, the value
+ * specified in @va_pages may be ignored.
+ */
+struct kbase_ioctl_mem_jit_init {
+ __u64 va_pages;
+};
+
+#define KBASE_IOCTL_MEM_JIT_INIT \
+ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init)
+
+/**
+ * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory
+ *
+ * @handle: GPU memory handle (GPU VA)
+ * @user_addr: The address where it is mapped in user space
+ * @size: The number of bytes to synchronise
+ * @type: The direction to synchronise: 0 is sync to memory (clean),
+ * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants.
+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero
+ */
+struct kbase_ioctl_mem_sync {
+ __u64 handle;
+ __u64 user_addr;
+ __u64 size;
+ __u8 type;
+ __u8 padding[7];
+};
+
+#define KBASE_IOCTL_MEM_SYNC \
+ _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync)
+
+/**
+ * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer
+ *
+ * @gpu_addr: The GPU address of the memory region
+ * @cpu_addr: The CPU address to locate
+ * @size: A size in bytes to validate is contained within the region
+ * @offset: The offset from the start of the memory region to @cpu_addr
+ *
+ * @in: Input parameters
+ * @out: Output parameters
+ */
+union kbase_ioctl_mem_find_cpu_offset {
+ struct {
+ __u64 gpu_addr;
+ __u64 cpu_addr;
+ __u64 size;
+ } in;
+ struct {
+ __u64 offset;
+ } out;
+};
+
+#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \
+ _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset)
+
+/**
+ * struct kbase_ioctl_get_context_id - Get the kernel context ID
+ *
+ * @id: The kernel context ID
+ */
+struct kbase_ioctl_get_context_id {
+ int id; /* This should really be __u32, but see GPUCORE-10048 */
+};
+
+#define KBASE_IOCTL_GET_CONTEXT_ID \
+ _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id)
+
+/**
+ * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd
+ *
+ * @flags: Flags
+ *
+ * The ioctl returns a file descriptor when successful
+ */
+struct kbase_ioctl_tlstream_acquire {
+ __u32 flags;
+};
+
+#define KBASE_IOCTL_TLSTREAM_ACQUIRE \
+ _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire)
+
+#define KBASE_IOCTL_TLSTREAM_FLUSH \
+ _IO(KBASE_IOCTL_TYPE, 19)
+
+/**
+ * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region
+ *
+ * @gpu_addr: The memory region to modify
+ * @pages: The number of physical pages that should be present
+ *
+ * The ioctl may return on the following error codes or 0 for success:
+ * -ENOMEM: Out of memory
+ * -EINVAL: Invalid arguments
+ */
+struct kbase_ioctl_mem_commit {
+ __u64 gpu_addr;
+ __u64 pages;
+};
+
+#define KBASE_IOCTL_MEM_COMMIT \
+ _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit)
+
+/**
+ * union kbase_ioctl_mem_alias - Create an alias of memory regions
+ * @flags: Flags, see BASE_MEM_xxx
+ * @stride: Bytes between start of each memory region
+ * @nents: The number of regions to pack together into the alias
+ * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info
+ * @gpu_va: Address of the new alias
+ * @va_pages: Size of the new alias
+ *
+ * @in: Input parameters
+ * @out: Output parameters
+ */
+union kbase_ioctl_mem_alias {
+ struct {
+ __u64 flags;
+ __u64 stride;
+ __u64 nents;
+ union kbase_pointer aliasing_info;
+ } in;
+ struct {
+ __u64 flags;
+ __u64 gpu_va;
+ __u64 va_pages;
+ } out;
+};
+
+#define KBASE_IOCTL_MEM_ALIAS \
+ _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias)
+
+/**
+ * union kbase_ioctl_mem_import - Import memory for use by the GPU
+ * @flags: Flags, see BASE_MEM_xxx
+ * @phandle: Handle to the external memory
+ * @type: Type of external memory, see base_mem_import_type
+ * @padding: Amount of extra VA pages to append to the imported buffer
+ * @gpu_va: Address of the new alias
+ * @va_pages: Size of the new alias
+ *
+ * @in: Input parameters
+ * @out: Output parameters
+ */
+union kbase_ioctl_mem_import {
+ struct {
+ __u64 flags;
+ union kbase_pointer phandle;
+ __u32 type;
+ __u32 padding;
+ } in;
+ struct {
+ __u64 flags;
+ __u64 gpu_va;
+ __u64 va_pages;
+ } out;
+};
+
+#define KBASE_IOCTL_MEM_IMPORT \
+ _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import)
+
+/**
+ * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region
+ * @gpu_va: The GPU region to modify
+ * @flags: The new flags to set
+ * @mask: Mask of the flags to modify
+ */
+struct kbase_ioctl_mem_flags_change {
+ __u64 gpu_va;
+ __u64 flags;
+ __u64 mask;
+};
+
+#define KBASE_IOCTL_MEM_FLAGS_CHANGE \
+ _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change)
+
+/**
+ * struct kbase_ioctl_stream_create - Create a synchronisation stream
+ * @name: A name to identify this stream. Must be NULL-terminated.
+ *
+ * Note that this is also called a "timeline", but is named stream to avoid
+ * confusion with other uses of the word.
+ *
+ * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes.
+ *
+ * The ioctl returns a file descriptor.
+ */
+struct kbase_ioctl_stream_create {
+ char name[32];
+};
+
+#define KBASE_IOCTL_STREAM_CREATE \
+ _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create)
+
+/**
+ * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence
+ * @fd: The file descriptor to validate
+ */
+struct kbase_ioctl_fence_validate {
+ int fd;
+};
+
+#define KBASE_IOCTL_FENCE_VALIDATE \
+ _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate)
+
+/**
+ * struct kbase_ioctl_get_profiling_controls - Get the profiling controls
+ * @count: The size of @buffer in u32 words
+ * @buffer: The buffer to receive the profiling controls
+ */
+struct kbase_ioctl_get_profiling_controls {
+ union kbase_pointer buffer;
+ __u32 count;
+};
+
+#define KBASE_IOCTL_GET_PROFILING_CONTROLS \
+ _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls)
+
+/**
+ * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel
+ * @buffer: Pointer to the information
+ * @len: Length
+ * @padding: Padding
+ *
+ * The data provided is accessible through a debugfs file
+ */
+struct kbase_ioctl_mem_profile_add {
+ union kbase_pointer buffer;
+ __u32 len;
+ __u32 padding;
+};
+
+#define KBASE_IOCTL_MEM_PROFILE_ADD \
+ _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add)
+
+/**
+ * struct kbase_ioctl_soft_event_update - Update the status of a soft-event
+ * @event: GPU address of the event which has been updated
+ * @new_status: The new status to set
+ * @flags: Flags for future expansion
+ */
+struct kbase_ioctl_soft_event_update {
+ __u64 event;
+ __u32 new_status;
+ __u32 flags;
+};
+
+#define KBASE_IOCTL_SOFT_EVENT_UPDATE \
+ _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update)
+
+/***************
+ * test ioctls *
+ ***************/
+#if MALI_UNIT_TEST
+/* These ioctls are purely for test purposes and are not used in the production
+ * driver, they therefore may change without notice
+ */
+
+#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1)
+
+/**
+ * struct kbase_ioctl_tlstream_test - Start a timeline stream test
+ *
+ * @tpw_count: number of trace point writers in each context
+ * @msg_delay: time delay between tracepoints from one writer in milliseconds
+ * @msg_count: number of trace points written by one writer
+ * @aux_msg: if non-zero aux messages will be included
+ */
+struct kbase_ioctl_tlstream_test {
+ __u32 tpw_count;
+ __u32 msg_delay;
+ __u32 msg_count;
+ __u32 aux_msg;
+};
+
+#define KBASE_IOCTL_TLSTREAM_TEST \
+ _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test)
+
+/**
+ * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes
+ * @bytes_collected: number of bytes read by user
+ * @bytes_generated: number of bytes generated by tracepoints
+ */
+struct kbase_ioctl_tlstream_stats {
+ __u32 bytes_collected;
+ __u32 bytes_generated;
+};
+
+#define KBASE_IOCTL_TLSTREAM_STATS \
+ _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats)
+
+#endif
+
+/**********************************
+ * Definitions for GPU properties *
+ **********************************/
+#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0)
+#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1)
+#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2)
+#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3)
+
+#define KBASE_GPUPROP_PRODUCT_ID 1
+#define KBASE_GPUPROP_VERSION_STATUS 2
+#define KBASE_GPUPROP_MINOR_REVISION 3
+#define KBASE_GPUPROP_MAJOR_REVISION 4
+#define KBASE_GPUPROP_GPU_SPEED_MHZ 5
+#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6
+#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7
+#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8
+#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9
+#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10
+#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11
+#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12
+
+#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13
+#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14
+#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15
+
+#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16
+#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17
+
+#define KBASE_GPUPROP_MAX_THREADS 18
+#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19
+#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20
+#define KBASE_GPUPROP_MAX_REGISTERS 21
+#define KBASE_GPUPROP_MAX_TASK_QUEUE 22
+#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23
+#define KBASE_GPUPROP_IMPL_TECH 24
+
+#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25
+#define KBASE_GPUPROP_RAW_TILER_PRESENT 26
+#define KBASE_GPUPROP_RAW_L2_PRESENT 27
+#define KBASE_GPUPROP_RAW_STACK_PRESENT 28
+#define KBASE_GPUPROP_RAW_L2_FEATURES 29
+#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30
+#define KBASE_GPUPROP_RAW_MEM_FEATURES 31
+#define KBASE_GPUPROP_RAW_MMU_FEATURES 32
+#define KBASE_GPUPROP_RAW_AS_PRESENT 33
+#define KBASE_GPUPROP_RAW_JS_PRESENT 34
+#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35
+#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36
+#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37
+#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38
+#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39
+#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40
+#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41
+#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42
+#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43
+#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44
+#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45
+#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46
+#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47
+#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48
+#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49
+#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50
+#define KBASE_GPUPROP_RAW_TILER_FEATURES 51
+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52
+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53
+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54
+#define KBASE_GPUPROP_RAW_GPU_ID 55
+#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56
+#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57
+#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58
+#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59
+#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60
+
+#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61
+#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62
+#define KBASE_GPUPROP_COHERENCY_COHERENCY 63
+#define KBASE_GPUPROP_COHERENCY_GROUP_0 64
+#define KBASE_GPUPROP_COHERENCY_GROUP_1 65
+#define KBASE_GPUPROP_COHERENCY_GROUP_2 66
+#define KBASE_GPUPROP_COHERENCY_GROUP_3 67
+#define KBASE_GPUPROP_COHERENCY_GROUP_4 68
+#define KBASE_GPUPROP_COHERENCY_GROUP_5 69
+#define KBASE_GPUPROP_COHERENCY_GROUP_6 70
+#define KBASE_GPUPROP_COHERENCY_GROUP_7 71
+#define KBASE_GPUPROP_COHERENCY_GROUP_8 72
+#define KBASE_GPUPROP_COHERENCY_GROUP_9 73
+#define KBASE_GPUPROP_COHERENCY_GROUP_10 74
+#define KBASE_GPUPROP_COHERENCY_GROUP_11 75
+#define KBASE_GPUPROP_COHERENCY_GROUP_12 76
+#define KBASE_GPUPROP_COHERENCY_GROUP_13 77
+#define KBASE_GPUPROP_COHERENCY_GROUP_14 78
+#define KBASE_GPUPROP_COHERENCY_GROUP_15 79
+
+#ifdef __cpluscplus
+}
+#endif
+
+#endif
diff --git a/mali_kbase/mali_kbase_jd.c b/mali_kbase/mali_kbase_jd.c
index c2a5571..f39f1b0 100644
--- a/mali_kbase/mali_kbase_jd.c
+++ b/mali_kbase/mali_kbase_jd.c
@@ -596,7 +596,7 @@ static inline void jd_resolve_dep(struct list_head *out_list,
#ifdef CONFIG_MALI_DMA_FENCE
int dep_count;
- dep_count = atomic_read(&dep_atom->dma_fence.dep_count);
+ dep_count = kbase_fence_dep_count_read(dep_atom);
if (likely(dep_count == -1)) {
dep_satisfied = true;
} else {
@@ -735,8 +735,8 @@ static void jd_try_submitting_deps(struct list_head *out_list,
#ifdef CONFIG_MALI_DMA_FENCE
int dep_count;
- dep_count = atomic_read(
- &dep_atom->dma_fence.dep_count);
+ dep_count = kbase_fence_dep_count_read(
+ dep_atom);
if (likely(dep_count == -1)) {
dep_satisfied = true;
} else {
@@ -975,7 +975,11 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us
* the scheduler: 'not ready to run' and 'dependency-only' jobs. */
jctx->job_nr++;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
katom->start_timestamp.tv64 = 0;
+#else
+ katom->start_timestamp = 0;
+#endif
katom->udata = user_atom->udata;
katom->kctx = kctx;
katom->nr_extres = user_atom->nr_extres;
@@ -1007,7 +1011,7 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us
katom->kds_rset = NULL;
#endif /* CONFIG_KDS */
#ifdef CONFIG_MALI_DMA_FENCE
- atomic_set(&katom->dma_fence.dep_count, -1);
+ kbase_fence_dep_count_set(katom, -1);
#endif
/* Don't do anything if there is a mess up with dependencies.
@@ -1233,7 +1237,7 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us
#ifdef CONFIG_MALI_DMA_FENCE
- if (atomic_read(&katom->dma_fence.dep_count) != -1) {
+ if (kbase_fence_dep_count_read(katom) != -1) {
ret = false;
goto out;
}
@@ -1270,26 +1274,20 @@ bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *us
return ret;
}
-#ifdef BASE_LEGACY_UK6_SUPPORT
-int kbase_jd_submit(struct kbase_context *kctx,
- const struct kbase_uk_job_submit *submit_data,
- int uk6_atom)
-#else
int kbase_jd_submit(struct kbase_context *kctx,
- const struct kbase_uk_job_submit *submit_data)
-#endif /* BASE_LEGACY_UK6_SUPPORT */
+ void __user *user_addr, u32 nr_atoms, u32 stride,
+ bool uk6_atom)
{
struct kbase_jd_context *jctx = &kctx->jctx;
int err = 0;
int i;
bool need_to_try_schedule_context = false;
struct kbase_device *kbdev;
- void __user *user_addr;
u32 latest_flush;
/*
- * kbase_jd_submit isn't expected to fail and so all errors with the jobs
- * are reported by immediately falling them (through event system)
+ * kbase_jd_submit isn't expected to fail and so all errors with the
+ * jobs are reported by immediately failing them (through event system)
*/
kbdev = kctx->kbdev;
@@ -1300,29 +1298,25 @@ int kbase_jd_submit(struct kbase_context *kctx,
return -EINVAL;
}
-#ifdef BASE_LEGACY_UK6_SUPPORT
- if ((uk6_atom && submit_data->stride !=
- sizeof(struct base_jd_atom_v2_uk6)) ||
- submit_data->stride != sizeof(base_jd_atom_v2)) {
-#else
- if (submit_data->stride != sizeof(base_jd_atom_v2)) {
-#endif /* BASE_LEGACY_UK6_SUPPORT */
+ if (stride != sizeof(base_jd_atom_v2)) {
dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel");
return -EINVAL;
}
- user_addr = get_compat_pointer(kctx, &submit_data->addr);
-
- KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(submit_data->nr_atoms, &kctx->timeline.jd_atoms_in_flight));
+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms,
+ &kctx->timeline.jd_atoms_in_flight));
/* All atoms submitted in this call have the same flush ID */
latest_flush = kbase_backend_get_current_flush_id(kbdev);
- for (i = 0; i < submit_data->nr_atoms; i++) {
+ for (i = 0; i < nr_atoms; i++) {
struct base_jd_atom_v2 user_atom;
struct kbase_jd_atom *katom;
#ifdef BASE_LEGACY_UK6_SUPPORT
+ BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) !=
+ sizeof(base_jd_atom_v2));
+
if (uk6_atom) {
struct base_jd_atom_v2_uk6 user_atom_v6;
base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA};
@@ -1332,7 +1326,7 @@ int kbase_jd_submit(struct kbase_context *kctx,
err = -EINVAL;
KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
atomic_sub_return(
- submit_data->nr_atoms - i,
+ nr_atoms - i,
&kctx->timeline.jd_atoms_in_flight));
break;
}
@@ -1364,14 +1358,17 @@ int kbase_jd_submit(struct kbase_context *kctx,
user_atom.device_nr = user_atom_v6.device_nr;
} else {
#endif /* BASE_LEGACY_UK6_SUPPORT */
- if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) {
- err = -EINVAL;
- KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(submit_data->nr_atoms - i, &kctx->timeline.jd_atoms_in_flight));
- break;
- }
+ if (copy_from_user(&user_atom, user_addr,
+ sizeof(user_atom)) != 0) {
+ err = -EINVAL;
+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
+ atomic_sub_return(nr_atoms - i,
+ &kctx->timeline.jd_atoms_in_flight));
+ break;
+ }
#ifdef BASE_LEGACY_UK6_SUPPORT
}
-#endif /* BASE_LEGACY_UK6_SUPPORT */
+#endif
#ifdef BASE_LEGACY_UK10_2_SUPPORT
if (KBASE_API_VERSION(10, 3) > kctx->api_version)
@@ -1379,7 +1376,7 @@ int kbase_jd_submit(struct kbase_context *kctx,
& 0x7fff);
#endif /* BASE_LEGACY_UK10_2_SUPPORT */
- user_addr = (void __user *)((uintptr_t) user_addr + submit_data->stride);
+ user_addr = (void __user *)((uintptr_t) user_addr + stride);
mutex_lock(&jctx->lock);
#ifndef compiletime_assert
@@ -1846,8 +1843,9 @@ int kbase_jd_init(struct kbase_context *kctx)
kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
-#ifdef CONFIG_MALI_DMA_FENCE
- kctx->jctx.atoms[i].dma_fence.context = fence_context_alloc(1);
+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
+ kctx->jctx.atoms[i].dma_fence.context =
+ dma_fence_context_alloc(1);
atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0);
INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks);
#endif
diff --git a/mali_kbase/mali_kbase_jd_debugfs.c b/mali_kbase/mali_kbase_jd_debugfs.c
index 47ea426..c8b37c4 100644
--- a/mali_kbase/mali_kbase_jd_debugfs.c
+++ b/mali_kbase/mali_kbase_jd_debugfs.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -15,63 +15,58 @@
-#include <linux/seq_file.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/seq_file.h>
#include <mali_kbase.h>
-
#include <mali_kbase_jd_debugfs.h>
#include <mali_kbase_dma_fence.h>
-
-#ifdef CONFIG_DEBUG_FS
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+#include <mali_kbase_sync.h>
+#endif
struct kbase_jd_debugfs_depinfo {
u8 id;
char type;
};
-#ifdef CONFIG_SYNC
-#include <mali_kbase_sync.h>
-#include "sync.h"
-#endif /* CONFIG_SYNC */
-
-
static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom,
struct seq_file *sfile)
{
-#ifdef CONFIG_SYNC
- struct sync_fence *fence = atom->fence;
- int status;
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+ struct kbase_sync_fence_info info;
+ int res;
switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
- if (!fence)
+ res = kbase_sync_fence_out_info_get(atom, &info);
+ if (0 == res) {
+ seq_printf(sfile, "Sa([%p]%d) ",
+ info.fence, info.status);
break;
-
- status = kbase_fence_get_status(fence);
-
- seq_printf(sfile, "Sa([%p]%d) ",
- fence, status);
- break;
+ }
case BASE_JD_REQ_SOFT_FENCE_WAIT:
- if (!fence)
+ res = kbase_sync_fence_in_info_get(atom, &info);
+ if (0 == res) {
+ seq_printf(sfile, "Wa([%p]%d) ",
+ info.fence, info.status);
break;
-
- status = kbase_fence_get_status(fence);
-
- seq_printf(sfile, "Wa([%p]%d) ",
- fence, status);
- break;
+ }
default:
break;
}
-#endif /* CONFIG_SYNC */
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
#ifdef CONFIG_MALI_DMA_FENCE
if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
- struct kbase_dma_fence_cb *cb;
+ struct kbase_fence_cb *cb;
if (atom->dma_fence.fence) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *fence = atom->dma_fence.fence;
+#else
+ struct dma_fence *fence = atom->dma_fence.fence;
+#endif
seq_printf(sfile,
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
@@ -81,13 +76,17 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom,
#endif
fence->context,
fence->seqno,
- fence_is_signaled(fence) ?
+ dma_fence_is_signaled(fence) ?
"signaled" : "active");
}
list_for_each_entry(cb, &atom->dma_fence.callbacks,
node) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *fence = cb->fence;
+#else
+ struct dma_fence *fence = cb->fence;
+#endif
seq_printf(sfile,
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
@@ -97,7 +96,7 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom,
#endif
fence->context,
fence->seqno,
- fence_is_signaled(fence) ?
+ dma_fence_is_signaled(fence) ?
"signaled" : "active");
}
}
diff --git a/mali_kbase/mali_kbase_js.c b/mali_kbase/mali_kbase_js.c
index d75dcba..b053ce1 100644
--- a/mali_kbase/mali_kbase_js.c
+++ b/mali_kbase/mali_kbase_js.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -27,6 +27,7 @@
#endif
#include <mali_kbase_tlstream.h>
#include <mali_kbase_hw.h>
+#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_config_defaults.h>
@@ -82,46 +83,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
- unsigned long flags;
- struct kbasep_js_device_data *js_devdata;
- int as_nr;
- int refcnt = 0;
-
- js_devdata = &kbdev->js_data;
-
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID) {
- struct kbasep_js_per_as_data *js_per_as_data;
-
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- refcnt = js_per_as_data->as_busy_refcount;
- }
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-
- return refcnt;
-}
-
-static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
- struct kbase_context *kctx)
-{
- struct kbasep_js_device_data *js_devdata;
- int as_nr;
- int refcnt = 0;
-
- js_devdata = &kbdev->js_data;
-
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID) {
- struct kbasep_js_per_as_data *js_per_as_data;
-
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- refcnt = js_per_as_data->as_busy_refcount;
- }
-
- return refcnt;
+ return atomic_read(&kctx->refcount);
}
#else /* KBASE_TRACE_ENABLE */
static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
@@ -131,13 +93,6 @@ static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
CSTD_UNUSED(kctx);
return 0;
}
-static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
- struct kbase_context *kctx)
-{
- CSTD_UNUSED(kbdev);
- CSTD_UNUSED(kctx);
- return 0;
-}
#endif /* KBASE_TRACE_ENABLE */
/*
@@ -181,12 +136,11 @@ static void kbase_js_sync_timers(struct kbase_device *kbdev)
mutex_unlock(&kbdev->js_data.runpool_mutex);
}
-/* Hold the hwaccess_lock for this */
+/* Hold the mmu_hw_mutex and hwaccess_lock for this */
bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_per_as_data *js_per_as_data;
bool result = false;
int as_nr;
@@ -194,19 +148,15 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID) {
- int new_refcnt;
+ lockdep_assert_held(&kbdev->hwaccess_lock);
+ as_nr = kctx->as_nr;
+ if (atomic_read(&kctx->refcount) > 0) {
KBASE_DEBUG_ASSERT(as_nr >= 0);
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- KBASE_DEBUG_ASSERT(js_per_as_data->kctx != NULL);
-
- new_refcnt = ++(js_per_as_data->as_busy_refcount);
+ kbase_ctx_sched_retain_ctx_refcount(kctx);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx,
- NULL, 0u, new_refcnt);
+ NULL, 0u, atomic_read(&kctx->refcount));
result = true;
}
@@ -470,25 +420,11 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev)
{
struct kbasep_js_device_data *jsdd;
int i;
- u16 as_present;
KBASE_DEBUG_ASSERT(kbdev != NULL);
jsdd = &kbdev->js_data;
- /* These two must be recalculated if nr_hw_address_spaces changes
- * (e.g. for HW workarounds) */
- as_present = (1U << kbdev->nr_hw_address_spaces) - 1;
- kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces;
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- bool use_workaround;
-
- use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
- if (use_workaround) {
- dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only");
- kbdev->nr_user_address_spaces = 1;
- }
- }
#ifdef CONFIG_MALI_DEBUG
/* Soft-stop will be disabled on a single context by default unless
* softstop_always is set */
@@ -498,8 +434,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev)
jsdd->nr_user_contexts_running = 0;
jsdd->nr_contexts_pullable = 0;
atomic_set(&jsdd->nr_contexts_runnable, 0);
- /* All ASs initially free */
- jsdd->as_free = as_present;
/* No ctx allowed to submit */
jsdd->runpool_irq.submit_allowed = 0u;
memset(jsdd->runpool_irq.ctx_attr_ref_count, 0,
@@ -577,10 +511,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev)
dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing.");
#endif
- /* Clear the AS data, including setting NULL pointers */
- memset(&jsdd->runpool_irq.per_as_data[0], 0,
- sizeof(jsdd->runpool_irq.per_as_data));
-
for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i)
jsdd->js_reqs[i] = core_reqs_from_jsn_features(
kbdev->gpu_props.props.raw_props.js_features[i]);
@@ -1189,7 +1119,7 @@ bool kbasep_js_add_job(struct kbase_context *kctx,
enqueue_required = kbase_js_dep_resolved_submit(kctx, atom);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc,
- kbasep_js_trace_get_refcnt_nolock(kbdev, kctx));
+ kbasep_js_trace_get_refcnt(kbdev, kctx));
/* Context Attribute Refcounting */
kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom);
@@ -1303,11 +1233,11 @@ bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev,
KBASE_DEBUG_ASSERT(kbdev != NULL);
js_devdata = &kbdev->js_data;
- /* KBASE_TRACE_ADD_REFCOUNT( kbdev, JS_RETAIN_CTX, kctx, NULL, 0,
- kbasep_js_trace_get_refcnt(kbdev, kctx)); */
+ mutex_lock(&kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
return result;
}
@@ -1318,48 +1248,23 @@ struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev,
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
struct kbase_context *found_kctx = NULL;
- struct kbasep_js_per_as_data *js_per_as_data;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
js_devdata = &kbdev->js_data;
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- found_kctx = js_per_as_data->kctx;
+ found_kctx = kbdev->as_to_kctx[as_nr];
if (found_kctx != NULL)
- ++(js_per_as_data->as_busy_refcount);
+ kbase_ctx_sched_retain_ctx_refcount(found_kctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return found_kctx;
}
-struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
- struct kbase_device *kbdev, int as_nr)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbase_context *found_kctx = NULL;
- struct kbasep_js_per_as_data *js_per_as_data;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
-
- lockdep_assert_held(&kbdev->hwaccess_lock);
-
- js_devdata = &kbdev->js_data;
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- found_kctx = js_per_as_data->kctx;
-
- if (found_kctx != NULL)
- ++(js_per_as_data->as_busy_refcount);
-
- return found_kctx;
-}
-
/**
* kbasep_js_release_result - Try running more jobs after releasing a context
* and/or atom
@@ -1445,7 +1350,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
- struct kbasep_js_per_as_data *js_per_as_data;
kbasep_js_release_result release_result = 0u;
bool runpool_ctx_attr_change = false;
@@ -1461,16 +1365,9 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
- /* kctx->as_nr and js_per_as_data are only read from here. The caller's
- * js_ctx_mutex provides a barrier that ensures they are up-to-date.
- *
- * They will not change whilst we're reading them, because the refcount
- * is non-zero (and we ASSERT on that last fact).
- */
kctx_as_nr = kctx->as_nr;
KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID);
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[kctx_as_nr];
- KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0);
+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0);
/*
* Transaction begins on AS and runpool_irq
@@ -1479,14 +1376,14 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
*/
current_as = &kbdev->as[kctx_as_nr];
mutex_lock(&kbdev->pm.lock);
- mutex_lock(&kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr);
- KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0);
+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0);
/* Update refcount */
- new_ref_count = --(js_per_as_data->as_busy_refcount);
+ kbase_ctx_sched_release_ctx(kctx);
+ new_ref_count = atomic_read(&kctx->refcount);
/* Release the atom if it finished (i.e. wasn't soft-stopped) */
if (kbasep_js_has_atom_finished(katom_retained_state))
@@ -1496,7 +1393,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u,
new_ref_count);
- if (new_ref_count == 1 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) &&
+ if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) &&
!kbase_pm_is_suspending(kbdev)) {
/* Context is kept scheduled into an address space even when
* there are no jobs, in this case we have to handle the
@@ -1509,8 +1406,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
kbasep_js_set_submit_allowed(js_devdata, kctx);
}
- /* Make a set of checks to see if the context should be scheduled out */
- if (new_ref_count == 0 &&
+ /* Make a set of checks to see if the context should be scheduled out.
+ * Note that there'll always be at least 1 reference to the context
+ * which was previously acquired by kbasep_js_schedule_ctx(). */
+ if (new_ref_count == 1 &&
(!kbasep_js_is_submit_allowed(js_devdata, kctx) ||
kbdev->pm.suspending)) {
int num_slots = kbdev->gpu_props.num_job_slots;
@@ -1518,7 +1417,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
/* Last reference, and we've been told to remove this context
* from the Run Pool */
- dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d",
+ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d",
kctx, new_ref_count, js_kctx_info->ctx.nr_jobs,
kbasep_js_is_submit_allowed(js_devdata, kctx));
@@ -1571,7 +1470,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
kbase_backend_release_ctx_noirq(kbdev, kctx);
- mutex_unlock(&kbdev->mmu_hw_mutex);
mutex_unlock(&kbdev->pm.lock);
/* Note: Don't reuse kctx_as_nr now */
@@ -1594,7 +1492,6 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
katom_retained_state, runpool_ctx_attr_change);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- mutex_unlock(&kbdev->mmu_hw_mutex);
mutex_unlock(&kbdev->pm.lock);
}
@@ -1740,8 +1637,26 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
js_kctx_info = &kctx->jctx.sched_info;
/* Pick available address space for this context */
- as_nr = kbase_backend_find_free_address_space(kbdev, kctx);
+ mutex_lock(&kbdev->mmu_hw_mutex);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ as_nr = kbase_ctx_sched_retain_ctx(kctx);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
+ if (as_nr == KBASEP_AS_NR_INVALID) {
+ as_nr = kbase_backend_find_and_release_free_address_space(
+ kbdev, kctx);
+ if (as_nr != KBASEP_AS_NR_INVALID) {
+ /* Attempt to retain the context again, this should
+ * succeed */
+ mutex_lock(&kbdev->mmu_hw_mutex);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ as_nr = kbase_ctx_sched_retain_ctx(kctx);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
+ WARN_ON(as_nr == KBASEP_AS_NR_INVALID);
+ }
+ }
if (as_nr == KBASEP_AS_NR_INVALID)
return false; /* No address spaces currently available */
@@ -1752,12 +1667,16 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
*/
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->mmu_hw_mutex);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
/* Check to see if context is dying due to kbase_job_zap_context() */
if (kbase_ctx_flag(kctx, KCTX_DYING)) {
/* Roll back the transaction so far and return */
- kbase_backend_release_free_address_space(kbdev, as_nr);
+ kbase_ctx_sched_release_ctx(kctx);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
@@ -1770,21 +1689,17 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
kbase_ctx_flag_set(kctx, KCTX_SCHEDULED);
- mutex_lock(&kbdev->mmu_hw_mutex);
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-
/* Assign context to previously chosen address space */
if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) {
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- mutex_unlock(&kbdev->mmu_hw_mutex);
/* Roll back the transaction so far and return */
+ kbase_ctx_sched_release_ctx(kctx);
kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED);
- kbase_backend_release_free_address_space(kbdev, as_nr);
-
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kbdev->mmu_hw_mutex);
mutex_unlock(&js_devdata->runpool_mutex);
-
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
return false;
}
@@ -1851,7 +1766,8 @@ static bool kbase_js_use_ctx(struct kbase_device *kbdev,
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- if (kbase_backend_use_ctx_sched(kbdev, kctx)) {
+ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) &&
+ kbase_backend_use_ctx_sched(kbdev, kctx)) {
/* Context already has ASID - mark as active */
kbdev->hwaccess.active_kctx = kctx;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@@ -1953,14 +1869,12 @@ void kbasep_js_suspend(struct kbase_device *kbdev)
/* Retain each of the contexts, so we can cause it to leave even if it
* had no refcount to begin with */
for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) {
- struct kbasep_js_per_as_data *js_per_as_data =
- &js_devdata->runpool_irq.per_as_data[i];
- struct kbase_context *kctx = js_per_as_data->kctx;
+ struct kbase_context *kctx = kbdev->as_to_kctx[i];
retained = retained << 1;
if (kctx) {
- ++(js_per_as_data->as_busy_refcount);
+ kbase_ctx_sched_retain_ctx_refcount(kctx);
retained |= 1u;
/* We can only cope with up to 1 privileged context -
* the instrumented context. It'll be suspended by
@@ -1980,9 +1894,7 @@ void kbasep_js_suspend(struct kbase_device *kbdev)
for (i = 0;
i < BASE_MAX_NR_AS;
++i, retained = retained >> 1) {
- struct kbasep_js_per_as_data *js_per_as_data =
- &js_devdata->runpool_irq.per_as_data[i];
- struct kbase_context *kctx = js_per_as_data->kctx;
+ struct kbase_context *kctx = kbdev->as_to_kctx[i];
if (retained & 1u)
kbasep_js_runpool_release_ctx(kbdev, kctx);
@@ -2189,16 +2101,19 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
{
struct kbase_jd_atom *katom;
struct kbasep_js_device_data *js_devdata;
+ struct kbase_device *kbdev;
int pulled;
KBASE_DEBUG_ASSERT(kctx);
- js_devdata = &kctx->kbdev->js_data;
- lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
+ kbdev = kctx->kbdev;
+
+ js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&kbdev->hwaccess_lock);
if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
return NULL;
- if (kbase_pm_is_suspending(kctx->kbdev))
+ if (kbase_pm_is_suspending(kbdev))
return NULL;
katom = jsctx_rb_peek(kctx, js);
@@ -2214,7 +2129,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
* present on the same slot */
if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) {
struct kbase_jd_atom *prev_atom =
- kbase_backend_inspect_tail(kctx->kbdev, js);
+ kbase_backend_inspect_tail(kbdev, js);
if (prev_atom && prev_atom->kctx != kctx)
return NULL;
@@ -2226,7 +2141,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
katom->x_pre_dep->will_fail_event_code)
return NULL;
if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
- kbase_backend_nr_atoms_on_slot(kctx->kbdev, js))
+ kbase_backend_nr_atoms_on_slot(kbdev, js))
return NULL;
}
@@ -2236,13 +2151,14 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
if (pulled == 1 && !kctx->slots_pullable) {
WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF);
- atomic_inc(&kctx->kbdev->js_data.nr_contexts_runnable);
+ atomic_inc(&kbdev->js_data.nr_contexts_runnable);
}
atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]);
kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++;
jsctx_rb_pull(kctx, katom);
- kbasep_js_runpool_retain_ctx_nolock(kctx->kbdev, kctx);
+ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+
katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF;
katom->ticks = 0;
@@ -2868,22 +2784,7 @@ void kbase_js_zap_context(struct kbase_context *kctx)
static inline int trace_get_refcnt(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
- struct kbasep_js_device_data *js_devdata;
- int as_nr;
- int refcnt = 0;
-
- js_devdata = &kbdev->js_data;
-
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID) {
- struct kbasep_js_per_as_data *js_per_as_data;
-
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- refcnt = js_per_as_data->as_busy_refcount;
- }
-
- return refcnt;
+ return atomic_read(&kctx->refcount);
}
/**
diff --git a/mali_kbase/mali_kbase_js.h b/mali_kbase/mali_kbase_js.h
index d092fff..ddada8e 100644
--- a/mali_kbase/mali_kbase_js.h
+++ b/mali_kbase/mali_kbase_js.h
@@ -227,7 +227,8 @@ bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
* @note This function can safely be called from IRQ context.
*
* The following locking conditions are made on the caller:
- * - it must \em not hold the hwaccess_lock, because it will be used internally.
+ * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be
+ * used internally.
*
* @return value != false if the retain succeeded, and the context will not be scheduled out.
* @return false if the retain failed (because the context is being/has been scheduled out).
@@ -241,7 +242,7 @@ bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_conte
* @note This function can safely be called from IRQ context.
*
* The following locks must be held by the caller:
- * - hwaccess_lock
+ * - mmu_hw_mutex, hwaccess_lock
*
* @return value != false if the retain succeeded, and the context will not be scheduled out.
* @return false if the retain failed (because the context is being/has been scheduled out).
@@ -269,28 +270,6 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbas
struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr);
/**
- * kbasep_js_runpool_lookup_ctx_nolock - Lookup a context in the Run Pool based
- * upon its current address space and ensure that is stays scheduled in.
- * @kbdev: Device pointer
- * @as_nr: Address space to lookup
- *
- * The context is refcounted as being busy to prevent it from scheduling
- * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
- * longer required to stay scheduled in.
- *
- * Note: This function can safely be called from IRQ context.
- *
- * The following locking conditions are made on the caller:
- * - it must the hold the hwaccess_lock
- *
- * Return: a valid struct kbase_context on success, which has been refcounted as
- * being busy.
- * NULL on failure, indicating that no context was found in \a as_nr
- */
-struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
- struct kbase_device *kbdev, int as_nr);
-
-/**
* @brief Handling the requeuing/killing of a context that was evicted from the
* policy queue or runpool.
*
@@ -782,41 +761,9 @@ static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_a
return (bool) (js >= 0);
}
-#if KBASE_DEBUG_DISABLE_ASSERTS == 0
-/**
- * Debug Check the refcount of a context. Only use within ASSERTs
- *
- * Obtains hwaccess_lock
- *
- * @return negative value if the context is not scheduled in
- * @return current refcount of the context if it is scheduled in. The refcount
- * is not guarenteed to be kept constant.
- */
-static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, struct kbase_context *kctx)
-{
- unsigned long flags;
- struct kbasep_js_device_data *js_devdata;
- int result = -1;
- int as_nr;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
- js_devdata = &kbdev->js_data;
-
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID)
- result = js_devdata->runpool_irq.per_as_data[as_nr].as_busy_refcount;
-
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-
- return result;
-}
-#endif /* KBASE_DEBUG_DISABLE_ASSERTS == 0 */
-
/**
* @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the
- * context is guarenteed to be already previously retained.
+ * context is guaranteed to be already previously retained.
*
* It is a programming error to supply the \a as_nr of a context that has not
* been previously retained/has a busy refcount of zero. The only exception is
@@ -825,28 +772,20 @@ static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev,
* The following locking conditions are made on the caller:
* - it must \em not hold the hwaccess_lock, because it will be used internally.
*
- * @return a valid struct kbase_context on success, with a refcount that is guarenteed
+ * @return a valid struct kbase_context on success, with a refcount that is guaranteed
* to be non-zero and unmodified by this function.
* @return NULL on failure, indicating that no context was found in \a as_nr
*/
static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr)
{
- unsigned long flags;
- struct kbasep_js_device_data *js_devdata;
struct kbase_context *found_kctx;
- struct kbasep_js_per_as_data *js_per_as_data;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
- js_devdata = &kbdev->js_data;
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
-
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-
- found_kctx = js_per_as_data->kctx;
- KBASE_DEBUG_ASSERT(found_kctx == NULL || js_per_as_data->as_busy_refcount > 0);
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ found_kctx = kbdev->as_to_kctx[as_nr];
+ KBASE_DEBUG_ASSERT(found_kctx == NULL ||
+ atomic_read(&found_kctx->refcount) > 0);
return found_kctx;
}
diff --git a/mali_kbase/mali_kbase_js_defs.h b/mali_kbase/mali_kbase_js_defs.h
index 6f5feba..ba8b644 100644
--- a/mali_kbase/mali_kbase_js_defs.h
+++ b/mali_kbase/mali_kbase_js_defs.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -142,26 +142,6 @@ enum {
typedef u32 kbasep_js_atom_done_code;
/**
- * Data used by the scheduler that is unique for each Address Space.
- *
- * This is used in IRQ context and hwaccess_lock must be held whilst accessing
- * this data (inculding reads and atomic decisions based on the read).
- */
-struct kbasep_js_per_as_data {
- /**
- * Ref count of whether this AS is busy, and must not be scheduled out
- *
- * When jobs are running this is always positive. However, it can still be
- * positive when no jobs are running. If all you need is a heuristic to
- * tell you whether jobs might be running, this should be sufficient.
- */
- int as_busy_refcount;
-
- /** Pointer to the current context on this address space, or NULL for no context */
- struct kbase_context *kctx;
-};
-
-/**
* @brief KBase Device Data Job Scheduler sub-structure
*
* This encapsulates the current context of the Job Scheduler on a particular
@@ -181,10 +161,8 @@ struct kbasep_js_device_data {
struct runpool_irq {
/** Bitvector indicating whether a currently scheduled context is allowed to submit jobs.
* When bit 'N' is set in this, it indicates whether the context bound to address space
- * 'N' (per_as_data[N].kctx) is allowed to submit jobs.
- *
- * It is placed here because it's much more memory efficient than having a u8 in
- * struct kbasep_js_per_as_data to store this flag */
+ * 'N' is allowed to submit jobs.
+ */
u16 submit_allowed;
/** Context Attributes:
@@ -203,9 +181,6 @@ struct kbasep_js_device_data {
* is required on updating the variable) */
s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT];
- /** Data that is unique for each AS */
- struct kbasep_js_per_as_data per_as_data[BASE_MAX_NR_AS];
-
/*
* Affinity management and tracking
*/
@@ -256,8 +231,6 @@ struct kbasep_js_device_data {
*/
struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS];
- u16 as_free; /**< Bitpattern of free Address Spaces */
-
/** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */
s8 nr_user_contexts_running;
/** Number of currently scheduled contexts (including ones that are not submitting jobs) */
diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c
index 20a18a9..e76294d 100644
--- a/mali_kbase/mali_kbase_mem.c
+++ b/mali_kbase/mali_kbase_mem.c
@@ -961,6 +961,9 @@ bad_insert:
KBASE_EXPORT_TEST_API(kbase_gpu_mmap);
+static void kbase_jd_user_buf_unmap(struct kbase_context *kctx,
+ struct kbase_mem_phy_alloc *alloc, bool writeable);
+
int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
{
int err;
@@ -981,6 +984,20 @@ int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc);
}
+ if (reg->gpu_alloc && reg->gpu_alloc->type ==
+ KBASE_MEM_TYPE_IMPORTED_USER_BUF) {
+ struct kbase_alloc_import_user_buf *user_buf =
+ &reg->gpu_alloc->imported.user_buf;
+
+ if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) {
+ user_buf->current_mapping_usage_count &=
+ ~PINNED_ON_IMPORT;
+
+ kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc,
+ (reg->flags & KBASE_REG_GPU_WR));
+ }
+ }
+
if (err)
return err;
@@ -994,6 +1011,10 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping(
{
struct vm_area_struct *vma;
struct kbase_cpu_mapping *map;
+ unsigned long vm_pgoff_in_region;
+ unsigned long vm_off_in_region;
+ unsigned long map_start;
+ size_t map_size;
lockdep_assert_held(&current->mm->mmap_sem);
@@ -1014,8 +1035,16 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping(
/* Not from this context! */
return NULL;
- *offset = (uaddr - vma->vm_start) +
- ((vma->vm_pgoff - map->region->start_pfn)<<PAGE_SHIFT);
+ vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn;
+ vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT;
+ map_start = vma->vm_start - vm_off_in_region;
+ map_size = map->region->nr_pages << PAGE_SHIFT;
+
+ if ((uaddr + size) > (map_start + map_size))
+ /* Not within the CPU mapping */
+ return NULL;
+
+ *offset = (uaddr - vma->vm_start) + vm_off_in_region;
return map;
}
@@ -1092,10 +1121,9 @@ void kbase_sync_single(struct kbase_context *kctx,
}
static int kbase_do_syncset(struct kbase_context *kctx,
- struct base_syncset *set, enum kbase_sync_type sync_fn)
+ struct basep_syncset *sset, enum kbase_sync_type sync_fn)
{
int err = 0;
- struct basep_syncset *sset = &set->basep_sset;
struct kbase_va_region *reg;
struct kbase_cpu_mapping *map;
unsigned long start;
@@ -1179,23 +1207,26 @@ out_unlock:
return err;
}
-int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset)
+int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset)
{
int err = -EINVAL;
- struct basep_syncset *sset;
- KBASE_DEBUG_ASSERT(NULL != kctx);
- KBASE_DEBUG_ASSERT(NULL != syncset);
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+ KBASE_DEBUG_ASSERT(sset != NULL);
- sset = &syncset->basep_sset;
+ if (sset->mem_handle.basep.handle & ~PAGE_MASK) {
+ dev_warn(kctx->kbdev->dev,
+ "mem_handle: passed parameter is invalid");
+ return -EINVAL;
+ }
switch (sset->type) {
case BASE_SYNCSET_OP_MSYNC:
- err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_DEVICE);
+ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE);
break;
case BASE_SYNCSET_OP_CSYNC:
- err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_CPU);
+ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU);
break;
default:
@@ -1266,6 +1297,11 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr)
KBASE_DEBUG_ASSERT(kctx != NULL);
+ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) {
+ dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid");
+ return -EINVAL;
+ }
+
if (0 == gpu_addr) {
dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n");
return -EINVAL;
@@ -2154,12 +2190,18 @@ static int kbase_jd_user_buf_map(struct kbase_context *kctx,
alloc->imported.user_buf.nr_pages,
reg->flags & KBASE_REG_GPU_WR,
0, pages, NULL);
-#else
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
pinned_pages = get_user_pages_remote(NULL, mm,
address,
alloc->imported.user_buf.nr_pages,
reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0,
pages, NULL);
+#else
+ pinned_pages = get_user_pages_remote(NULL, mm,
+ address,
+ alloc->imported.user_buf.nr_pages,
+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0,
+ pages, NULL, NULL);
#endif
if (pinned_pages <= 0)
@@ -2289,28 +2331,44 @@ static int kbase_jd_umm_map(struct kbase_context *kctx,
alloc->imported.umm.dma_buf->size);
}
- if (WARN_ONCE(count < reg->nr_pages,
+ if (!(reg->flags & KBASE_REG_IMPORT_PAD) &&
+ WARN_ONCE(count < reg->nr_pages,
"sg list from dma_buf_map_attachment < dma_buf->size=%zu\n",
alloc->imported.umm.dma_buf->size)) {
err = -EINVAL;
- goto out;
+ goto err_unmap_attachment;
}
/* Update nents as we now have pages to map */
- alloc->nents = count;
+ alloc->nents = reg->nr_pages;
err = kbase_mmu_insert_pages(kctx, reg->start_pfn,
kbase_get_gpu_phy_pages(reg),
- kbase_reg_current_backed_size(reg),
+ count,
reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);
-
-out:
- if (err) {
- dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment,
- alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
- alloc->imported.umm.sgt = NULL;
+ if (err)
+ goto err_unmap_attachment;
+
+ if (reg->flags & KBASE_REG_IMPORT_PAD) {
+ err = kbase_mmu_insert_single_page(kctx,
+ reg->start_pfn + count,
+ page_to_phys(kctx->aliasing_sink_page),
+ reg->nr_pages - count,
+ (reg->flags | KBASE_REG_GPU_RD) &
+ ~KBASE_REG_GPU_WR);
+ if (err)
+ goto err_teardown_orig_pages;
}
+ return 0;
+
+err_teardown_orig_pages:
+ kbase_mmu_teardown_pages(kctx, reg->start_pfn, count);
+err_unmap_attachment:
+ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment,
+ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
+ alloc->imported.umm.sgt = NULL;
+
return err;
}
@@ -2434,11 +2492,15 @@ void kbase_unmap_external_resource(struct kbase_context *kctx,
alloc->imported.umm.current_mapping_usage_count--;
if (0 == alloc->imported.umm.current_mapping_usage_count) {
- if (reg && reg->gpu_alloc == alloc)
- kbase_mmu_teardown_pages(
+ if (reg && reg->gpu_alloc == alloc) {
+ int err;
+
+ err = kbase_mmu_teardown_pages(
kctx,
reg->start_pfn,
- kbase_reg_current_backed_size(reg));
+ alloc->nents);
+ WARN_ON(err);
+ }
kbase_jd_umm_unmap(kctx, alloc);
}
diff --git a/mali_kbase/mali_kbase_mem.h b/mali_kbase/mali_kbase_mem.h
index eefb4a2..e293577 100644
--- a/mali_kbase/mali_kbase_mem.h
+++ b/mali_kbase/mali_kbase_mem.h
@@ -151,18 +151,31 @@ struct kbase_mem_phy_alloc {
} alias;
/* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */
struct kbase_context *kctx;
- struct {
+ struct kbase_alloc_import_user_buf {
unsigned long address;
unsigned long size;
unsigned long nr_pages;
struct page **pages;
- unsigned int current_mapping_usage_count;
+ /* top bit (1<<31) of current_mapping_usage_count
+ * specifies that this import was pinned on import
+ * See PINNED_ON_IMPORT
+ */
+ u32 current_mapping_usage_count;
struct mm_struct *mm;
dma_addr_t *dma_addrs;
} user_buf;
} imported;
};
+/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is
+ * used to signify that a buffer was pinned when it was imported. Since the
+ * reference count is limited by the number of atoms that can be submitted at
+ * once there should be no danger of overflowing into this bit.
+ * Stealing the top bit also has the benefit that
+ * current_mapping_usage_count != 0 if and only if the buffer is mapped.
+ */
+#define PINNED_ON_IMPORT (1<<31)
+
static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc)
{
KBASE_DEBUG_ASSERT(alloc);
@@ -255,6 +268,9 @@ struct kbase_va_region {
#define KBASE_REG_DONT_NEED (1ul << 20)
+/* Imported buffer is padded? */
+#define KBASE_REG_IMPORT_PAD (1ul << 21)
+
#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0)
/* only used with 32-bit clients */
@@ -726,7 +742,16 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
*/
void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages);
-int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset);
+/**
+ * kbase_sync_now - Perform cache maintenance on a memory region
+ *
+ * @kctx: The kbase context of the region
+ * @sset: A syncset structure describing the region and direction of the
+ * synchronisation required
+ *
+ * Return: 0 on success or error code
+ */
+int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset);
void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa,
phys_addr_t gpu_pa, off_t offset, size_t size,
enum kbase_sync_type sync_fn);
diff --git a/mali_kbase/mali_kbase_mem_linux.c b/mali_kbase/mali_kbase_mem_linux.c
index 0b3a7da..eea429a 100644
--- a/mali_kbase/mali_kbase_mem_linux.c
+++ b/mali_kbase/mali_kbase_mem_linux.c
@@ -264,6 +264,11 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * c
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(out);
+ if (gpu_addr & ~PAGE_MASK) {
+ dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid");
+ return -EINVAL;
+ }
+
kbase_gpu_vm_lock(kctx);
/* Validate the region */
@@ -723,6 +728,9 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in
if (!gpu_addr)
return -EINVAL;
+ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE))
+ return -EINVAL;
+
/* nuke other bits */
flags &= mask;
@@ -939,7 +947,8 @@ bad_flags:
#endif /* CONFIG_UMP */
#ifdef CONFIG_DMA_SHARED_BUFFER
-static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags)
+static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx,
+ int fd, u64 *va_pages, u64 *flags, u32 padding)
{
struct kbase_va_region *reg;
struct dma_buf *dma_buf;
@@ -954,7 +963,7 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in
if (!dma_attachment)
goto no_attachment;
- *va_pages = PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT;
+ *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding;
if (!*va_pages)
goto bad_size;
@@ -1008,6 +1017,9 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in
if (*flags & BASE_MEM_SECURE)
reg->flags |= KBASE_REG_SECURE;
+ if (padding)
+ reg->flags |= KBASE_REG_IMPORT_PAD;
+
reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM;
reg->gpu_alloc->imported.umm.sgt = NULL;
reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
@@ -1046,11 +1058,14 @@ static struct kbase_va_region *kbase_mem_from_user_buffer(
struct kbase_context *kctx, unsigned long address,
unsigned long size, u64 *va_pages, u64 *flags)
{
+ long i;
struct kbase_va_region *reg;
long faulted_pages;
int zone = KBASE_REG_ZONE_CUSTOM_VA;
bool shared_zone = false;
u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx);
+ struct kbase_alloc_import_user_buf *user_buf;
+ struct page **pages = NULL;
if ((address & (cache_line_alignment - 1)) != 0 ||
(size & (cache_line_alignment - 1)) != 0) {
@@ -1116,50 +1131,101 @@ static struct kbase_va_region *kbase_mem_from_user_buffer(
reg->flags &= ~KBASE_REG_FREE;
reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */
reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */
+ reg->flags &= ~KBASE_REG_CPU_CACHED;
+
+ user_buf = &reg->gpu_alloc->imported.user_buf;
+
+ user_buf->size = size;
+ user_buf->address = address;
+ user_buf->nr_pages = *va_pages;
+ user_buf->mm = current->mm;
+ user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *),
+ GFP_KERNEL);
+
+ if (!user_buf->pages)
+ goto no_page_array;
+
+ /* If the region is coherent with the CPU then the memory is imported
+ * and mapped onto the GPU immediately.
+ * Otherwise get_user_pages is called as a sanity check, but with
+ * NULL as the pages argument which will fault the pages, but not
+ * pin them. The memory will then be pinned only around the jobs that
+ * specify the region as an external resource.
+ */
+ if (reg->flags & KBASE_REG_SHARE_BOTH) {
+ pages = user_buf->pages;
+ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
+ }
down_read(&current->mm->mmap_sem);
- /* A sanity check that get_user_pages will work on the memory */
- /* (so the initial import fails on weird memory regions rather than */
- /* the job failing when we try to handle the external resources). */
- /* It doesn't take a reference to the pages (because the page list is NULL). */
- /* We can't really store the page list because that would involve */
- /* keeping the pages pinned - instead we pin/unpin around the job */
- /* (as part of the external resources handling code) */
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
- reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
+ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL);
#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0)
faulted_pages = get_user_pages(address, *va_pages,
- reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
+ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL);
#else
faulted_pages = get_user_pages(address, *va_pages,
reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0,
- NULL, NULL);
+ pages, NULL);
#endif
+
up_read(&current->mm->mmap_sem);
if (faulted_pages != *va_pages)
goto fault_mismatch;
- reg->gpu_alloc->imported.user_buf.size = size;
- reg->gpu_alloc->imported.user_buf.address = address;
- reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages;
- reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages,
- sizeof(struct page *), GFP_KERNEL);
- reg->gpu_alloc->imported.user_buf.mm = current->mm;
atomic_inc(&current->mm->mm_count);
- if (!reg->gpu_alloc->imported.user_buf.pages)
- goto no_page_array;
-
reg->gpu_alloc->nents = 0;
reg->extent = 0;
+ if (pages) {
+ struct device *dev = kctx->kbdev->dev;
+ unsigned long local_size = user_buf->size;
+ unsigned long offset = user_buf->address & ~PAGE_MASK;
+ phys_addr_t *pa = kbase_get_gpu_phy_pages(reg);
+
+ /* Top bit signifies that this was pinned on import */
+ user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT;
+
+ for (i = 0; i < faulted_pages; i++) {
+ dma_addr_t dma_addr;
+ unsigned long min;
+
+ min = MIN(PAGE_SIZE - offset, local_size);
+ dma_addr = dma_map_page(dev, pages[i],
+ offset, min,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, dma_addr))
+ goto unwind_dma_map;
+
+ user_buf->dma_addrs[i] = dma_addr;
+ pa[i] = page_to_phys(pages[i]);
+
+ local_size -= min;
+ offset = 0;
+ }
+
+ reg->gpu_alloc->nents = faulted_pages;
+ }
+
return reg;
-no_page_array:
+unwind_dma_map:
+ while (i--) {
+ dma_unmap_page(kctx->kbdev->dev,
+ user_buf->dma_addrs[i],
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ }
fault_mismatch:
+ if (pages) {
+ for (i = 0; i < faulted_pages; i++)
+ put_page(pages[i]);
+ }
+ kfree(user_buf->pages);
+no_page_array:
invalid_flags:
kbase_mem_phy_alloc_put(reg->cpu_alloc);
kbase_mem_phy_alloc_put(reg->gpu_alloc);
@@ -1362,7 +1428,7 @@ bad_flags:
}
int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
- void __user *phandle, u64 *gpu_va, u64 *va_pages,
+ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages,
u64 *flags)
{
struct kbase_va_region *reg;
@@ -1384,6 +1450,24 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
goto bad_flags;
}
+ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
+ !kbase_device_is_cpu_coherent(kctx->kbdev)) {
+ dev_warn(kctx->kbdev->dev,
+ "kbase_mem_import call required coherent mem when unavailable");
+ goto bad_flags;
+ }
+ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
+ !kbase_device_is_cpu_coherent(kctx->kbdev)) {
+ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
+ *flags &= ~BASE_MEM_COHERENT_SYSTEM;
+ }
+
+ if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) {
+ dev_warn(kctx->kbdev->dev,
+ "padding is only supported for UMM");
+ goto bad_flags;
+ }
+
switch (type) {
#ifdef CONFIG_UMP
case BASE_MEM_IMPORT_TYPE_UMP: {
@@ -1403,7 +1487,8 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
if (get_user(fd, (int __user *)phandle))
reg = NULL;
else
- reg = kbase_mem_from_umm(kctx, fd, va_pages, flags);
+ reg = kbase_mem_from_umm(kctx, fd, va_pages, flags,
+ padding);
}
break;
#endif /* CONFIG_DMA_SHARED_BUFFER */
@@ -1535,7 +1620,7 @@ static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
return ret;
}
-int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages)
{
u64 old_pages;
u64 delta;
@@ -1544,48 +1629,40 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en
bool read_locked = false;
KBASE_DEBUG_ASSERT(kctx);
- KBASE_DEBUG_ASSERT(failure_reason);
KBASE_DEBUG_ASSERT(gpu_addr != 0);
+ if (gpu_addr & ~PAGE_MASK) {
+ dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid");
+ return -EINVAL;
+ }
+
down_write(&current->mm->mmap_sem);
kbase_gpu_vm_lock(kctx);
/* Validate the region */
reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
- if (!reg || (reg->flags & KBASE_REG_FREE)) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
+ if (!reg || (reg->flags & KBASE_REG_FREE))
goto out_unlock;
- }
KBASE_DEBUG_ASSERT(reg->cpu_alloc);
KBASE_DEBUG_ASSERT(reg->gpu_alloc);
- if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
+ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
goto out_unlock;
- }
- if (0 == (reg->flags & KBASE_REG_GROWABLE)) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
+ if (0 == (reg->flags & KBASE_REG_GROWABLE))
goto out_unlock;
- }
- if (new_pages > reg->nr_pages) {
- /* Would overflow the VA region */
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
+ /* Would overflow the VA region */
+ if (new_pages > reg->nr_pages)
goto out_unlock;
- }
/* can't be mapped more than once on the GPU */
- if (atomic_read(&reg->gpu_alloc->gpu_mappings) > 1) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
+ if (atomic_read(&reg->gpu_alloc->gpu_mappings) > 1)
goto out_unlock;
- }
/* can't grow regions which are ephemeral */
- if (reg->flags & KBASE_REG_DONT_NEED) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
+ if (reg->flags & KBASE_REG_DONT_NEED)
goto out_unlock;
- }
if (new_pages == reg->gpu_alloc->nents) {
/* no change */
@@ -1606,13 +1683,13 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en
/* Allocate some more pages */
if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
+ res = -ENOMEM;
goto out_unlock;
}
if (reg->cpu_alloc != reg->gpu_alloc) {
if (kbase_alloc_phy_pages_helper(
reg->gpu_alloc, delta) != 0) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
+ res = -ENOMEM;
kbase_free_phy_pages_helper(reg->cpu_alloc,
delta);
goto out_unlock;
@@ -1631,7 +1708,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en
if (reg->cpu_alloc != reg->gpu_alloc)
kbase_free_phy_pages_helper(reg->gpu_alloc,
delta);
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
+ res = -ENOMEM;
goto out_unlock;
}
} else {
@@ -1645,7 +1722,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, en
res = kbase_mem_shrink_gpu_mapping(kctx, reg,
new_pages, old_pages);
if (res) {
- *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
+ res = -ENOMEM;
goto out_unlock;
}
@@ -1735,7 +1812,11 @@ static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
/* insert all valid pages from the fault location */
i = rel_pgoff;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT);
+#else
+ addr = (pgoff_t)(vmf->address >> PAGE_SHIFT);
+#endif
while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) {
int ret = vm_insert_pfn(vma, addr << PAGE_SHIFT,
PFN_DOWN(map->alloc->pages[i]));
diff --git a/mali_kbase/mali_kbase_mem_linux.h b/mali_kbase/mali_kbase_mem_linux.h
index 97e0bec..a41d646 100644
--- a/mali_kbase/mali_kbase_mem_linux.h
+++ b/mali_kbase/mali_kbase_mem_linux.h
@@ -37,11 +37,22 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx,
u64 *gpu_va);
int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages);
int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
- void __user *phandle, u64 *gpu_va, u64 *va_pages,
+ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages,
u64 *flags);
u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages);
int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask);
-int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason);
+
+/**
+ * kbase_mem_commit - Change the physical backing size of a region
+ *
+ * @kctx: The kernel context
+ * @gpu_addr: Handle to the memory region
+ * @new_pages: Number of physical pages to back the region with
+ *
+ * Return: 0 on success or error code
+ */
+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages);
+
int kbase_mmap(struct file *file, struct vm_area_struct *vma);
/**
diff --git a/mali_kbase/mali_kbase_mem_lowlevel.h b/mali_kbase/mali_kbase_mem_lowlevel.h
index 441180b..9725fd3 100644
--- a/mali_kbase/mali_kbase_mem_lowlevel.h
+++ b/mali_kbase/mali_kbase_mem_lowlevel.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014, 2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_mem_pool.c b/mali_kbase/mali_kbase_mem_pool.c
index b5a7b8c..25c3128 100644
--- a/mali_kbase/mali_kbase_mem_pool.c
+++ b/mali_kbase/mali_kbase_mem_pool.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_mem_profile_debugfs.c b/mali_kbase/mali_kbase_mem_profile_debugfs.c
index 092da9a..d58fd8d 100644
--- a/mali_kbase/mali_kbase_mem_profile_debugfs.c
+++ b/mali_kbase/mali_kbase_mem_profile_debugfs.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -84,6 +84,8 @@ int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data,
kfree(kctx->mem_profile_data);
kctx->mem_profile_data = data;
kctx->mem_profile_size = size;
+ } else {
+ kfree(data);
}
dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d",
diff --git a/mali_kbase/mali_kbase_mmu.c b/mali_kbase/mali_kbase_mmu.c
index 0329dfd..79d2d18 100644
--- a/mali_kbase/mali_kbase_mmu.c
+++ b/mali_kbase/mali_kbase_mmu.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -172,18 +172,23 @@ void page_fault_worker(struct work_struct *data)
dev_warn(kbdev->dev, "Access flag unexpectedly set");
goto fault_done;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT:
-
- kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Address size fault");
+ else
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ "Unknown fault code");
goto fault_done;
case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT:
- kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory attributes fault");
+ else
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ "Unknown fault code");
goto fault_done;
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
default:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
@@ -1628,7 +1633,8 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
e = "TRANSLATION_FAULT";
break;
case 0xC8:
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
+ e = "PERMISSION_FAULT";
+ break;
case 0xC9:
case 0xCA:
case 0xCB:
@@ -1636,8 +1642,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
case 0xCD:
case 0xCE:
case 0xCF:
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
- e = "PERMISSION_FAULT";
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ e = "PERMISSION_FAULT";
+ else
+ e = "UNKNOWN";
break;
case 0xD0:
case 0xD1:
@@ -1650,7 +1658,8 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
e = "TRANSTAB_BUS_FAULT";
break;
case 0xD8:
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
+ e = "ACCESS_FLAG";
+ break;
case 0xD9:
case 0xDA:
case 0xDB:
@@ -1658,10 +1667,11 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
case 0xDD:
case 0xDE:
case 0xDF:
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
- e = "ACCESS_FLAG";
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ e = "ACCESS_FLAG";
+ else
+ e = "UNKNOWN";
break;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
case 0xE0:
case 0xE1:
case 0xE2:
@@ -1670,7 +1680,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
case 0xE5:
case 0xE6:
case 0xE7:
- e = "ADDRESS_SIZE_FAULT";
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ e = "ADDRESS_SIZE_FAULT";
+ else
+ e = "UNKNOWN";
break;
case 0xE8:
case 0xE9:
@@ -1680,8 +1693,10 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
case 0xED:
case 0xEE:
case 0xEF:
- e = "MEMORY_ATTRIBUTES_FAULT";
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ e = "MEMORY_ATTRIBUTES_FAULT";
+ else
+ e = "UNKNOWN";
break;
default:
e = "UNKNOWN";
@@ -1696,11 +1711,10 @@ static const char *access_type_name(struct kbase_device *kbdev,
{
switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- return "ATOMIC";
-#else
- return "UNKNOWN";
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ return "ATOMIC";
+ else
+ return "UNKNOWN";
case AS_FAULTSTATUS_ACCESS_TYPE_READ:
return "READ";
case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
@@ -1736,7 +1750,7 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
js_devdata = &kbdev->js_data;
/* ASSERT that the context won't leave the runpool */
- KBASE_DEBUG_ASSERT(kbasep_js_debug_check_ctx_refcount(kbdev, kctx) > 0);
+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0);
/* decode the fault status */
exception_type = as->fault_status & 0xFF;
@@ -2035,15 +2049,14 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_contex
*/
kbasep_js_clear_submit_allowed(js_devdata, kctx);
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- dev_warn(kbdev->dev,
- "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n",
- as->number, as->fault_addr,
- as->fault_extra_addr);
-#else
- dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
- as->number, as->fault_addr);
-#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
+ dev_warn(kbdev->dev,
+ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n",
+ as->number, as->fault_addr,
+ as->fault_extra_addr);
+ else
+ dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
+ as->number, as->fault_addr);
/*
* We need to switch to UNMAPPED mode - but we do this in a
diff --git a/mali_kbase/mali_kbase_mmu_mode.h b/mali_kbase/mali_kbase_mmu_mode.h
index 2449c60..b487c00 100644
--- a/mali_kbase/mali_kbase_mmu_mode.h
+++ b/mali_kbase/mali_kbase_mmu_mode.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_mmu_mode_aarch64.c b/mali_kbase/mali_kbase_mmu_mode_aarch64.c
index 791f3ed..60df171 100644
--- a/mali_kbase/mali_kbase_mmu_mode_aarch64.c
+++ b/mali_kbase/mali_kbase_mmu_mode_aarch64.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2014, 2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2014, 2016, 2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
diff --git a/mali_kbase/mali_kbase_mmu_mode_lpae.c b/mali_kbase/mali_kbase_mmu_mode_lpae.c
index 683cabb..53fbbc7 100644
--- a/mali_kbase/mali_kbase_mmu_mode_lpae.c
+++ b/mali_kbase/mali_kbase_mmu_mode_lpae.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -91,11 +91,7 @@ static void mmu_get_as_setup(struct kbase_context *kctx,
AS_TRANSTAB_LPAE_ADRMODE_TABLE |
AS_TRANSTAB_LPAE_READ_INNER;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- setup->transcfg = AS_TRANSCFG_ADRMODE_LEGACY;
-#else
setup->transcfg = 0;
-#endif
}
static void mmu_update(struct kbase_context *kctx)
@@ -117,10 +113,6 @@ static void mmu_disable_as(struct kbase_device *kbdev, int as_nr)
current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED;
-#ifdef CONFIG_MALI_GPU_MMU_AARCH64
- current_setup->transcfg = AS_TRANSCFG_ADRMODE_LEGACY;
-#endif
-
/* Apply the address space setting */
kbase_mmu_hw_configure(kbdev, as, NULL);
}
diff --git a/mali_kbase/mali_kbase_softjobs.c b/mali_kbase/mali_kbase_softjobs.c
index 8a9a47d..72809f7 100644
--- a/mali_kbase/mali_kbase_softjobs.c
+++ b/mali_kbase/mali_kbase_softjobs.c
@@ -23,15 +23,14 @@
#include <linux/dma-buf.h>
#include <asm/cacheflush.h>
#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
-#include <linux/dma-mapping.h>
-#ifdef CONFIG_SYNC
-#include "sync.h"
-#include <linux/syscalls.h>
-#include "mali_kbase_sync.h"
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+#include <mali_kbase_sync.h>
#endif
+#include <linux/dma-mapping.h>
#include <mali_base_kernel.h>
#include <mali_kbase_hwaccess_time.h>
#include <mali_kbase_mem_linux.h>
+#include <mali_kbase_tlstream.h>
#include <linux/version.h>
#include <linux/ktime.h>
#include <linux/pfn.h>
@@ -47,7 +46,7 @@
* executed within the driver rather than being handed over to the GPU.
*/
-void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom)
+static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom)
{
struct kbase_context *kctx = katom->kctx;
unsigned long lflags;
@@ -194,48 +193,11 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom)
return 0;
}
-#ifdef CONFIG_SYNC
-
-static enum base_jd_event_code kbase_fence_trigger(struct kbase_jd_atom *katom, int result)
-{
- struct sync_pt *pt;
- struct sync_timeline *timeline;
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
- if (!list_is_singular(&katom->fence->pt_list_head)) {
-#else
- if (katom->fence->num_fences != 1) {
-#endif
- /* Not exactly one item in the list - so it didn't (directly) come from us */
- return BASE_JD_EVENT_JOB_CANCELLED;
- }
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
- pt = list_first_entry(&katom->fence->pt_list_head, struct sync_pt, pt_list);
-#else
- pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base);
-#endif
- timeline = sync_pt_parent(pt);
-
- if (!kbase_sync_timeline_is_ours(timeline)) {
- /* Fence has a sync_pt which isn't ours! */
- return BASE_JD_EVENT_JOB_CANCELLED;
- }
-
- kbase_sync_signal_pt(pt, result);
-
- sync_timeline_signal(timeline);
-
- return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE;
-}
-
-static void kbase_fence_wait_worker(struct work_struct *data)
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
+/* Called by the explicit fence mechanism when a fence wait has completed */
+void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom)
{
- struct kbase_jd_atom *katom;
- struct kbase_context *kctx;
-
- katom = container_of(data, struct kbase_jd_atom, work);
- kctx = katom->kctx;
+ struct kbase_context *kctx = katom->kctx;
mutex_lock(&kctx->jctx.lock);
kbasep_remove_waiting_soft_job(katom);
@@ -244,89 +206,8 @@ static void kbase_fence_wait_worker(struct work_struct *data)
kbase_js_sched_all(kctx->kbdev);
mutex_unlock(&kctx->jctx.lock);
}
-
-static void kbase_fence_wait_callback(struct sync_fence *fence, struct sync_fence_waiter *waiter)
-{
- struct kbase_jd_atom *katom = container_of(waiter, struct kbase_jd_atom, sync_waiter);
- struct kbase_context *kctx;
-
- KBASE_DEBUG_ASSERT(NULL != katom);
-
- kctx = katom->kctx;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
-
- /* Propagate the fence status to the atom.
- * If negative then cancel this atom and its dependencies.
- */
- if (kbase_fence_get_status(fence) < 0)
- katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
-
- /* To prevent a potential deadlock we schedule the work onto the job_done_wq workqueue
- *
- * The issue is that we may signal the timeline while holding kctx->jctx.lock and
- * the callbacks are run synchronously from sync_timeline_signal. So we simply defer the work.
- */
-
- KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
- INIT_WORK(&katom->work, kbase_fence_wait_worker);
- queue_work(kctx->jctx.job_done_wq, &katom->work);
-}
-
-static int kbase_fence_wait(struct kbase_jd_atom *katom)
-{
- int ret;
-
- KBASE_DEBUG_ASSERT(NULL != katom);
- KBASE_DEBUG_ASSERT(NULL != katom->kctx);
-
- sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback);
-
- ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter);
-
- if (ret == 1) {
- /* Already signalled */
- return 0;
- }
-
- if (ret < 0) {
- katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
- /* We should cause the dependent jobs in the bag to be failed,
- * to do this we schedule the work queue to complete this job */
- KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
- INIT_WORK(&katom->work, kbase_fence_wait_worker);
- queue_work(katom->kctx->jctx.job_done_wq, &katom->work);
- }
-
-#ifdef CONFIG_MALI_FENCE_DEBUG
- /* The timeout code will add this job to the list of waiting soft jobs.
- */
- kbasep_add_waiting_with_timeout(katom);
-#else
- kbasep_add_waiting_soft_job(katom);
#endif
- return 1;
-}
-
-static void kbase_fence_cancel_wait(struct kbase_jd_atom *katom)
-{
- if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) {
- /* The wait wasn't cancelled - leave the cleanup for kbase_fence_wait_callback */
- return;
- }
-
- /* Wait was cancelled - zap the atoms */
- katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
-
- kbasep_remove_waiting_soft_job(katom);
- kbase_finish_soft_job(katom);
-
- if (jd_done_nolock(katom, NULL))
- kbase_js_sched_all(katom->kctx->kbdev);
-}
-#endif /* CONFIG_SYNC */
-
static void kbasep_soft_event_complete_job(struct work_struct *work)
{
struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom,
@@ -403,15 +284,17 @@ static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom)
if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE)
== BASE_JD_REQ_SOFT_FENCE_TRIGGER) {
- struct sync_fence *fence = dep->fence;
- int status = kbase_fence_get_status(fence);
-
/* Found blocked trigger fence. */
- dev_warn(dev,
- "\tVictim trigger atom %d fence [%p] %s: %s\n",
- kbase_jd_atom_id(kctx, dep),
- fence, fence->name,
- kbase_sync_status_string(status));
+ struct kbase_sync_fence_info info;
+
+ if (!kbase_sync_fence_in_info_get(dep, &info)) {
+ dev_warn(dev,
+ "\tVictim trigger atom %d fence [%p] %s: %s\n",
+ kbase_jd_atom_id(kctx, dep),
+ info.fence,
+ info.name,
+ kbase_sync_status_string(info.status));
+ }
}
kbase_fence_debug_check_atom(dep);
@@ -423,40 +306,32 @@ static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom)
{
struct kbase_context *kctx = katom->kctx;
struct device *dev = katom->kctx->kbdev->dev;
- struct sync_fence *fence;
int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms);
- int status;
unsigned long lflags;
+ struct kbase_sync_fence_info info;
spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags);
- fence = katom->fence;
- if (!fence) {
+ if (kbase_sync_fence_in_info_get(katom, &info)) {
/* Fence must have signaled just after timeout. */
spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags);
return;
}
- status = kbase_fence_get_status(fence);
-
dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n",
kctx->tgid, kctx->id,
kbase_jd_atom_id(kctx, katom),
- fence, timeout_ms);
+ info.fence, timeout_ms);
dev_warn(dev, "\tGuilty fence [%p] %s: %s\n",
- fence, fence->name,
- kbase_sync_status_string(status));
+ info.fence, info.name,
+ kbase_sync_status_string(info.status));
/* Search for blocked trigger atoms */
kbase_fence_debug_check_atom(katom);
spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags);
- /* Dump out the full state of all the Android sync fences.
- * The function sync_dump() isn't exported to modules, so force
- * sync_fence_wait() to time out to trigger sync_dump().
- */
- sync_fence_wait(fence, 1);
+ kbase_sync_fence_in_dump(katom);
}
struct kbase_fence_debug_work {
@@ -1073,7 +948,7 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom)
struct base_jit_alloc_info *info;
struct kbase_va_region *reg;
struct kbase_vmap_struct mapping;
- u64 *ptr;
+ u64 *ptr, new_addr;
if (katom->jit_blocked) {
list_del(&katom->queue);
@@ -1152,7 +1027,10 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom)
return 0;
}
- *ptr = reg->start_pfn << PAGE_SHIFT;
+ new_addr = reg->start_pfn << PAGE_SHIFT;
+ *ptr = new_addr;
+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(
+ katom, info->gpu_alloc_addr, new_addr);
kbase_vunmap(kctx, &mapping);
katom->event_code = BASE_JD_EVENT_DONE;
@@ -1389,17 +1267,28 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom)
switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
return kbase_dump_cpu_gpu_time(katom);
-#ifdef CONFIG_SYNC
+
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
- KBASE_DEBUG_ASSERT(katom->fence != NULL);
- katom->event_code = kbase_fence_trigger(katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT);
- /* Release the reference as we don't need it any more */
- sync_fence_put(katom->fence);
- katom->fence = NULL;
+ katom->event_code = kbase_sync_fence_out_trigger(katom,
+ katom->event_code == BASE_JD_EVENT_DONE ?
+ 0 : -EFAULT);
break;
case BASE_JD_REQ_SOFT_FENCE_WAIT:
- return kbase_fence_wait(katom);
-#endif /* CONFIG_SYNC */
+ {
+ int ret = kbase_sync_fence_in_wait(katom);
+
+ if (ret == 1) {
+#ifdef CONFIG_MALI_FENCE_DEBUG
+ kbasep_add_waiting_with_timeout(katom);
+#else
+ kbasep_add_waiting_soft_job(katom);
+#endif
+ }
+ return ret;
+ }
+#endif
+
case BASE_JD_REQ_SOFT_REPLAY:
return kbase_replay_process(katom);
case BASE_JD_REQ_SOFT_EVENT_WAIT:
@@ -1438,9 +1327,9 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom)
void kbase_cancel_soft_job(struct kbase_jd_atom *katom)
{
switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
case BASE_JD_REQ_SOFT_FENCE_WAIT:
- kbase_fence_cancel_wait(katom);
+ kbase_sync_fence_in_cancel_wait(katom);
break;
#endif
case BASE_JD_REQ_SOFT_EVENT_WAIT:
@@ -1461,7 +1350,7 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
return -EINVAL;
}
break;
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
{
struct base_fence fence;
@@ -1470,21 +1359,16 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
return -EINVAL;
- fd = kbase_stream_create_fence(fence.basep.stream_fd);
+ fd = kbase_sync_fence_out_create(katom,
+ fence.basep.stream_fd);
if (fd < 0)
return -EINVAL;
- katom->fence = sync_fence_fdget(fd);
-
- if (katom->fence == NULL) {
- /* The only way the fence can be NULL is if userspace closed it for us.
- * So we don't need to clear it up */
- return -EINVAL;
- }
fence.basep.fd = fd;
if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) {
- katom->fence = NULL;
- sys_close(fd);
+ kbase_sync_fence_out_remove(katom);
+ kbase_sync_fence_close_fd(fd);
+ fence.basep.fd = -EINVAL;
return -EINVAL;
}
}
@@ -1492,14 +1376,16 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
case BASE_JD_REQ_SOFT_FENCE_WAIT:
{
struct base_fence fence;
+ int ret;
if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
return -EINVAL;
/* Get a reference to the fence object */
- katom->fence = sync_fence_fdget(fence.basep.fd);
- if (katom->fence == NULL)
- return -EINVAL;
+ ret = kbase_sync_fence_in_from_fd(katom,
+ fence.basep.fd);
+ if (ret < 0)
+ return ret;
#ifdef CONFIG_MALI_DMA_FENCE
/*
@@ -1513,7 +1399,7 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
#endif /* CONFIG_MALI_DMA_FENCE */
}
break;
-#endif /* CONFIG_SYNC */
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
case BASE_JD_REQ_SOFT_JIT_ALLOC:
return kbase_jit_allocate_prepare(katom);
case BASE_JD_REQ_SOFT_REPLAY:
@@ -1545,23 +1431,17 @@ void kbase_finish_soft_job(struct kbase_jd_atom *katom)
case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
/* Nothing to do */
break;
-#ifdef CONFIG_SYNC
+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
- /* If fence has not yet been signalled, do it now */
- if (katom->fence) {
- kbase_fence_trigger(katom, katom->event_code ==
- BASE_JD_EVENT_DONE ? 0 : -EFAULT);
- sync_fence_put(katom->fence);
- katom->fence = NULL;
- }
+ /* If fence has not yet been signaled, do it now */
+ kbase_sync_fence_out_trigger(katom, katom->event_code ==
+ BASE_JD_EVENT_DONE ? 0 : -EFAULT);
break;
case BASE_JD_REQ_SOFT_FENCE_WAIT:
- /* Release the reference to the fence object */
- sync_fence_put(katom->fence);
- katom->fence = NULL;
+ /* Release katom's reference to fence object */
+ kbase_sync_fence_in_remove(katom);
break;
-#endif /* CONFIG_SYNC */
-
+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
case BASE_JD_REQ_SOFT_DEBUG_COPY:
kbase_debug_copy_finish(katom);
break;
diff --git a/mali_kbase/mali_kbase_sync.c b/mali_kbase/mali_kbase_sync.c
deleted file mode 100644
index 97e301d..0000000
--- a/mali_kbase/mali_kbase_sync.c
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#ifdef CONFIG_SYNC
-
-#include <linux/seq_file.h>
-#include "sync.h"
-#include <mali_kbase.h>
-#include <mali_kbase_sync.h>
-
-struct mali_sync_timeline {
- struct sync_timeline timeline;
- atomic_t counter;
- atomic_t signalled;
-};
-
-struct mali_sync_pt {
- struct sync_pt pt;
- int order;
- int result;
-};
-
-static struct mali_sync_timeline *to_mali_sync_timeline(struct sync_timeline *timeline)
-{
- return container_of(timeline, struct mali_sync_timeline, timeline);
-}
-
-static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt)
-{
- return container_of(pt, struct mali_sync_pt, pt);
-}
-
-static struct sync_pt *timeline_dup(struct sync_pt *pt)
-{
- struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
- struct mali_sync_pt *new_mpt;
- struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), sizeof(struct mali_sync_pt));
-
- if (!new_pt)
- return NULL;
-
- new_mpt = to_mali_sync_pt(new_pt);
- new_mpt->order = mpt->order;
- new_mpt->result = mpt->result;
-
- return new_pt;
-}
-
-static int timeline_has_signaled(struct sync_pt *pt)
-{
- struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt));
- int result = mpt->result;
-
- int diff = atomic_read(&mtl->signalled) - mpt->order;
-
- if (diff >= 0)
- return (result < 0) ? result : 1;
-
- return 0;
-}
-
-static int timeline_compare(struct sync_pt *a, struct sync_pt *b)
-{
- struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt);
- struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt);
-
- int diff = ma->order - mb->order;
-
- if (diff == 0)
- return 0;
-
- return (diff < 0) ? -1 : 1;
-}
-
-static void timeline_value_str(struct sync_timeline *timeline, char *str,
- int size)
-{
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline);
-
- snprintf(str, size, "%d", atomic_read(&mtl->signalled));
-}
-
-static void pt_value_str(struct sync_pt *pt, char *str, int size)
-{
- struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
-
- snprintf(str, size, "%d(%d)", mpt->order, mpt->result);
-}
-
-static struct sync_timeline_ops mali_timeline_ops = {
- .driver_name = "Mali",
- .dup = timeline_dup,
- .has_signaled = timeline_has_signaled,
- .compare = timeline_compare,
- .timeline_value_str = timeline_value_str,
- .pt_value_str = pt_value_str,
-};
-
-int kbase_sync_timeline_is_ours(struct sync_timeline *timeline)
-{
- return timeline->ops == &mali_timeline_ops;
-}
-
-struct sync_timeline *kbase_sync_timeline_alloc(const char *name)
-{
- struct sync_timeline *tl;
- struct mali_sync_timeline *mtl;
-
- tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline), name);
- if (!tl)
- return NULL;
-
- /* Set the counter in our private struct */
- mtl = to_mali_sync_timeline(tl);
- atomic_set(&mtl->counter, 0);
- atomic_set(&mtl->signalled, 0);
-
- return tl;
-}
-
-struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent)
-{
- struct sync_pt *pt = sync_pt_create(parent, sizeof(struct mali_sync_pt));
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent);
- struct mali_sync_pt *mpt;
-
- if (!pt)
- return NULL;
-
- mpt = to_mali_sync_pt(pt);
- mpt->order = atomic_inc_return(&mtl->counter);
- mpt->result = 0;
-
- return pt;
-}
-
-void kbase_sync_signal_pt(struct sync_pt *pt, int result)
-{
- struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt));
- int signalled;
- int diff;
-
- mpt->result = result;
-
- do {
- signalled = atomic_read(&mtl->signalled);
-
- diff = signalled - mpt->order;
-
- if (diff > 0) {
- /* The timeline is already at or ahead of this point.
- * This should not happen unless userspace has been
- * signalling fences out of order, so warn but don't
- * violate the sync_pt API.
- * The warning is only in debug builds to prevent
- * a malicious user being able to spam dmesg.
- */
-#ifdef CONFIG_MALI_DEBUG
- pr_err("Fences were triggered in a different order to allocation!");
-#endif /* CONFIG_MALI_DEBUG */
- return;
- }
- } while (atomic_cmpxchg(&mtl->signalled, signalled, mpt->order) != signalled);
-}
-
-const char *kbase_sync_status_string(int status)
-{
- if (status == 0)
- return "signaled";
- else if (status > 0)
- return "active";
- else
- return "error";
-}
-
-#endif /* CONFIG_SYNC */
diff --git a/mali_kbase/mali_kbase_sync.h b/mali_kbase/mali_kbase_sync.h
index 07e5158..de72147 100644
--- a/mali_kbase/mali_kbase_sync.h
+++ b/mali_kbase/mali_kbase_sync.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -15,94 +15,189 @@
-
-
/**
* @file mali_kbase_sync.h
*
+ * This file contains our internal "API" for explicit fences.
+ * It hides the implementation details of the actual explicit fence mechanism
+ * used (Android fences or sync file with DMA fences).
*/
#ifndef MALI_KBASE_SYNC_H
#define MALI_KBASE_SYNC_H
-#include "sync.h"
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
-/* For backwards compatiblility with kernels before 3.17. After 3.17
- * sync_pt_parent is included in the kernel. */
-static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt)
-{
- return pt->parent;
-}
+#include <linux/syscalls.h>
+#ifdef CONFIG_SYNC
+#include <sync.h>
#endif
-
-static inline int kbase_fence_get_status(struct sync_fence *fence)
-{
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
- return fence->status;
-#else
- return atomic_read(&fence->status);
+#ifdef CONFIG_SYNC_FILE
+#include "mali_kbase_fence_defs.h"
+#include <linux/sync_file.h>
#endif
-}
-/*
- * Create a stream object.
- * Built on top of timeline object.
+#include "mali_kbase.h"
+
+/**
+ * struct kbase_sync_fence_info - Information about a fence
+ * @fence: Pointer to fence (type is void*, as underlaying struct can differ)
+ * @name: The name given to this fence when it was created
+ * @status: < 0 means error, 0 means active, 1 means signaled
+ *
+ * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get()
+ * to get the information.
+ */
+struct kbase_sync_fence_info {
+ void *fence;
+ char name[32];
+ int status;
+};
+
+/**
+ * kbase_sync_fence_stream_create() - Create a stream object
+ * @name: Name of stream (only used to ease debugging/visualization)
+ * @out_fd: A file descriptor representing the created stream object
+ *
+ * Can map down to a timeline implementation in some implementations.
* Exposed as a file descriptor.
* Life-time controlled via the file descriptor:
* - dup to add a ref
* - close to remove a ref
+ *
+ * return: 0 on success, < 0 on error
*/
-int kbase_stream_create(const char *name, int *const out_fd);
+int kbase_sync_fence_stream_create(const char *name, int *const out_fd);
-/*
- * Create a fence in a stream object
+/**
+ * kbase_sync_fence_out_create Create an explicit output fence to specified atom
+ * @katom: Atom to assign the new explicit fence to
+ * @stream_fd: File descriptor for stream object to create fence on
+ *
+ * return: Valid file descriptor to fence or < 0 on error
*/
-int kbase_stream_create_fence(int tl_fd);
+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd);
-/*
- * Validate a fd to be a valid fence
- * No reference is taken.
+/**
+ * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom
+ * @katom: Atom to assign the existing explicit fence to
+ * @fd: File descriptor to an existing fence
+ *
+ * Assigns an explicit input fence to atom.
+ * This can later be waited for by calling @kbase_sync_fence_in_wait
+ *
+ * return: 0 on success, < 0 on error
+ */
+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd);
+
+/**
+ * kbase_sync_fence_validate() - Validate a fd to be a valid fence
+ * @fd: File descriptor to check
*
* This function is only usable to catch unintentional user errors early,
* it does not stop malicious code changing the fd after this function returns.
+ *
+ * return 0: if fd is for a valid fence, < 0 if invalid
*/
-int kbase_fence_validate(int fd);
+int kbase_sync_fence_validate(int fd);
-/* Returns true if the specified timeline is allocated by Mali */
-int kbase_sync_timeline_is_ours(struct sync_timeline *timeline);
+/**
+ * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom
+ * @katom: Atom with an explicit fence to signal
+ * @result: < 0 means signal with error, 0 >= indicates success
+ *
+ * Signal output fence attached on katom and remove the fence from the atom.
+ *
+ * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE
+ */
+enum base_jd_event_code
+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result);
-/* Allocates a timeline for Mali
+/**
+ * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled
+ * @katom: Atom with explicit fence to wait for
*
- * One timeline should be allocated per API context.
+ * If the fence is already signaled, then 0 is returned, and the caller must
+ * continue processing of the katom.
+ *
+ * If the fence isn't already signaled, then this kbase_sync framework will
+ * take responsibility to continue the processing once the fence is signaled.
+ *
+ * return: 0 if already signaled, otherwise 1
*/
-struct sync_timeline *kbase_sync_timeline_alloc(const char *name);
+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom);
-/* Allocates a sync point within the timeline.
+/**
+ * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits
+ * @katom: Atom to cancel wait for
*
- * The timeline must be the one allocated by kbase_sync_timeline_alloc
+ * This function is fully responsible for continuing processing of this atom
+ * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all)
+ */
+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom);
+
+/**
+ * kbase_sync_fence_in_remove() - Remove the input fence from the katom
+ * @katom: Atom to remove explicit input fence for
*
- * Sync points must be triggered in *exactly* the same order as they are allocated.
+ * This will also release the corresponding reference.
*/
-struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent);
+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom);
-/* Signals a particular sync point
+/**
+ * kbase_sync_fence_out_remove() - Remove the output fence from the katom
+ * @katom: Atom to remove explicit output fence for
*
- * Sync points must be triggered in *exactly* the same order as they are allocated.
+ * This will also release the corresponding reference.
+ */
+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom);
+
+/**
+ * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence
+ * @fd: File descriptor to close
+ */
+static inline void kbase_sync_fence_close_fd(int fd)
+{
+ sys_close(fd);
+}
+
+/**
+ * kbase_sync_fence_in_info_get() - Retrieves information about input fence
+ * @katom: Atom to get fence information from
+ * @info: Struct to be filled with fence information
*
- * If they are signalled in the wrong order then a message will be printed in debug
- * builds and otherwise attempts to signal order sync_pts will be ignored.
+ * return: 0 on success, < 0 on error
+ */
+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info);
+
+/**
+ * kbase_sync_fence_out_info_get() - Retrieves information about output fence
+ * @katom: Atom to get fence information from
+ * @info: Struct to be filled with fence information
*
- * result can be negative to indicate error, any other value is interpreted as success.
+ * return: 0 on success, < 0 on error
*/
-void kbase_sync_signal_pt(struct sync_pt *pt, int result);
+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info);
/**
* kbase_sync_status_string() - Get string matching @status
* @status: Value of fence status.
*
- * Return: Pointer to string describing @status.
+ * return: Pointer to string describing @status.
*/
const char *kbase_sync_status_string(int status);
+/*
+ * Internal worker used to continue processing of atom.
+ */
+void kbase_sync_fence_wait_worker(struct work_struct *data);
+
+#ifdef CONFIG_MALI_FENCE_DEBUG
+/**
+ * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state
+ * @katom: Atom to trigger fence debug dump for
+ */
+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom);
#endif
+
+#endif /* MALI_KBASE_SYNC_H */
diff --git a/mali_kbase/mali_kbase_sync_android.c b/mali_kbase/mali_kbase_sync_android.c
new file mode 100644
index 0000000..d7349dc
--- /dev/null
+++ b/mali_kbase/mali_kbase_sync_android.c
@@ -0,0 +1,537 @@
+/*
+ *
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Code for supporting explicit Android fences (CONFIG_SYNC)
+ * Known to be good for kernels 4.5 and earlier.
+ * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels
+ * (see mali_kbase_sync_file.c)
+ */
+
+#include <linux/sched.h>
+#include <linux/fdtable.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/anon_inodes.h>
+#include <linux/version.h>
+#include "sync.h"
+#include <mali_kbase.h>
+#include <mali_kbase_sync.h>
+
+struct mali_sync_timeline {
+ struct sync_timeline timeline;
+ atomic_t counter;
+ atomic_t signaled;
+};
+
+struct mali_sync_pt {
+ struct sync_pt pt;
+ int order;
+ int result;
+};
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+/* For backwards compatibility with kernels before 3.17. After 3.17
+ * sync_pt_parent is included in the kernel. */
+static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt)
+{
+ return pt->parent;
+}
+#endif
+
+static struct mali_sync_timeline *to_mali_sync_timeline(
+ struct sync_timeline *timeline)
+{
+ return container_of(timeline, struct mali_sync_timeline, timeline);
+}
+
+static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt)
+{
+ return container_of(pt, struct mali_sync_pt, pt);
+}
+
+static struct sync_pt *timeline_dup(struct sync_pt *pt)
+{
+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
+ struct mali_sync_pt *new_mpt;
+ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt),
+ sizeof(struct mali_sync_pt));
+
+ if (!new_pt)
+ return NULL;
+
+ new_mpt = to_mali_sync_pt(new_pt);
+ new_mpt->order = mpt->order;
+ new_mpt->result = mpt->result;
+
+ return new_pt;
+}
+
+static int timeline_has_signaled(struct sync_pt *pt)
+{
+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(
+ sync_pt_parent(pt));
+ int result = mpt->result;
+
+ int diff = atomic_read(&mtl->signaled) - mpt->order;
+
+ if (diff >= 0)
+ return (result < 0) ? result : 1;
+
+ return 0;
+}
+
+static int timeline_compare(struct sync_pt *a, struct sync_pt *b)
+{
+ struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt);
+ struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt);
+
+ int diff = ma->order - mb->order;
+
+ if (diff == 0)
+ return 0;
+
+ return (diff < 0) ? -1 : 1;
+}
+
+static void timeline_value_str(struct sync_timeline *timeline, char *str,
+ int size)
+{
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline);
+
+ snprintf(str, size, "%d", atomic_read(&mtl->signaled));
+}
+
+static void pt_value_str(struct sync_pt *pt, char *str, int size)
+{
+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
+
+ snprintf(str, size, "%d(%d)", mpt->order, mpt->result);
+}
+
+static struct sync_timeline_ops mali_timeline_ops = {
+ .driver_name = "Mali",
+ .dup = timeline_dup,
+ .has_signaled = timeline_has_signaled,
+ .compare = timeline_compare,
+ .timeline_value_str = timeline_value_str,
+ .pt_value_str = pt_value_str,
+};
+
+/* Allocates a timeline for Mali
+ *
+ * One timeline should be allocated per API context.
+ */
+static struct sync_timeline *mali_sync_timeline_alloc(const char *name)
+{
+ struct sync_timeline *tl;
+ struct mali_sync_timeline *mtl;
+
+ tl = sync_timeline_create(&mali_timeline_ops,
+ sizeof(struct mali_sync_timeline), name);
+ if (!tl)
+ return NULL;
+
+ /* Set the counter in our private struct */
+ mtl = to_mali_sync_timeline(tl);
+ atomic_set(&mtl->counter, 0);
+ atomic_set(&mtl->signaled, 0);
+
+ return tl;
+}
+
+static int kbase_stream_close(struct inode *inode, struct file *file)
+{
+ struct sync_timeline *tl;
+
+ tl = (struct sync_timeline *)file->private_data;
+ sync_timeline_destroy(tl);
+ return 0;
+}
+
+static const struct file_operations stream_fops = {
+ .owner = THIS_MODULE,
+ .release = kbase_stream_close,
+};
+
+int kbase_sync_fence_stream_create(const char *name, int *const out_fd)
+{
+ struct sync_timeline *tl;
+
+ if (!out_fd)
+ return -EINVAL;
+
+ tl = mali_sync_timeline_alloc(name);
+ if (!tl)
+ return -EINVAL;
+
+ *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC);
+
+ if (*out_fd < 0) {
+ sync_timeline_destroy(tl);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* Allocates a sync point within the timeline.
+ *
+ * The timeline must be the one allocated by kbase_sync_timeline_alloc
+ *
+ * Sync points must be triggered in *exactly* the same order as they are
+ * allocated.
+ */
+static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent)
+{
+ struct sync_pt *pt = sync_pt_create(parent,
+ sizeof(struct mali_sync_pt));
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent);
+ struct mali_sync_pt *mpt;
+
+ if (!pt)
+ return NULL;
+
+ mpt = to_mali_sync_pt(pt);
+ mpt->order = atomic_inc_return(&mtl->counter);
+ mpt->result = 0;
+
+ return pt;
+}
+
+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd)
+{
+ struct sync_timeline *tl;
+ struct sync_pt *pt;
+ struct sync_fence *fence;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0)
+ struct files_struct *files;
+ struct fdtable *fdt;
+#endif
+ int fd;
+ struct file *tl_file;
+
+ tl_file = fget(tl_fd);
+ if (tl_file == NULL)
+ return -EBADF;
+
+ if (tl_file->f_op != &stream_fops) {
+ fd = -EBADF;
+ goto out;
+ }
+
+ tl = tl_file->private_data;
+
+ pt = kbase_sync_pt_alloc(tl);
+ if (!pt) {
+ fd = -EFAULT;
+ goto out;
+ }
+
+ fence = sync_fence_create("mali_fence", pt);
+ if (!fence) {
+ sync_pt_free(pt);
+ fd = -EFAULT;
+ goto out;
+ }
+
+ /* from here the fence owns the sync_pt */
+
+ /* create a fd representing the fence */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
+ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
+ if (fd < 0) {
+ sync_fence_put(fence);
+ goto out;
+ }
+#else
+ fd = get_unused_fd();
+ if (fd < 0) {
+ sync_fence_put(fence);
+ goto out;
+ }
+
+ files = current->files;
+ spin_lock(&files->file_lock);
+ fdt = files_fdtable(files);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
+ __set_close_on_exec(fd, fdt);
+#else
+ FD_SET(fd, fdt->close_on_exec);
+#endif
+ spin_unlock(&files->file_lock);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */
+
+ /* bind fence to the new fd */
+ sync_fence_install(fence, fd);
+
+ katom->fence = sync_fence_fdget(fd);
+ if (katom->fence == NULL) {
+ /* The only way the fence can be NULL is if userspace closed it
+ * for us, so we don't need to clear it up */
+ fd = -EINVAL;
+ goto out;
+ }
+
+out:
+ fput(tl_file);
+
+ return fd;
+}
+
+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd)
+{
+ katom->fence = sync_fence_fdget(fd);
+ return katom->fence ? 0 : -ENOENT;
+}
+
+int kbase_sync_fence_validate(int fd)
+{
+ struct sync_fence *fence;
+
+ fence = sync_fence_fdget(fd);
+ if (!fence)
+ return -EINVAL;
+
+ sync_fence_put(fence);
+ return 0;
+}
+
+/* Returns true if the specified timeline is allocated by Mali */
+static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline)
+{
+ return timeline->ops == &mali_timeline_ops;
+}
+
+/* Signals a particular sync point
+ *
+ * Sync points must be triggered in *exactly* the same order as they are
+ * allocated.
+ *
+ * If they are signaled in the wrong order then a message will be printed in
+ * debug builds and otherwise attempts to signal order sync_pts will be ignored.
+ *
+ * result can be negative to indicate error, any other value is interpreted as
+ * success.
+ */
+static void kbase_sync_signal_pt(struct sync_pt *pt, int result)
+{
+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(
+ sync_pt_parent(pt));
+ int signaled;
+ int diff;
+
+ mpt->result = result;
+
+ do {
+ signaled = atomic_read(&mtl->signaled);
+
+ diff = signaled - mpt->order;
+
+ if (diff > 0) {
+ /* The timeline is already at or ahead of this point.
+ * This should not happen unless userspace has been
+ * signaling fences out of order, so warn but don't
+ * violate the sync_pt API.
+ * The warning is only in debug builds to prevent
+ * a malicious user being able to spam dmesg.
+ */
+#ifdef CONFIG_MALI_DEBUG
+ pr_err("Fences were triggered in a different order to allocation!");
+#endif /* CONFIG_MALI_DEBUG */
+ return;
+ }
+ } while (atomic_cmpxchg(&mtl->signaled,
+ signaled, mpt->order) != signaled);
+}
+
+enum base_jd_event_code
+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result)
+{
+ struct sync_pt *pt;
+ struct sync_timeline *timeline;
+
+ if (!katom->fence)
+ return BASE_JD_EVENT_JOB_CANCELLED;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+ if (!list_is_singular(&katom->fence->pt_list_head)) {
+#else
+ if (katom->fence->num_fences != 1) {
+#endif
+ /* Not exactly one item in the list - so it didn't (directly)
+ * come from us */
+ return BASE_JD_EVENT_JOB_CANCELLED;
+ }
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+ pt = list_first_entry(&katom->fence->pt_list_head,
+ struct sync_pt, pt_list);
+#else
+ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base);
+#endif
+ timeline = sync_pt_parent(pt);
+
+ if (!kbase_sync_timeline_is_ours(timeline)) {
+ /* Fence has a sync_pt which isn't ours! */
+ return BASE_JD_EVENT_JOB_CANCELLED;
+ }
+
+ kbase_sync_signal_pt(pt, result);
+
+ sync_timeline_signal(timeline);
+
+ kbase_sync_fence_out_remove(katom);
+
+ return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE;
+}
+
+static inline int kbase_fence_get_status(struct sync_fence *fence)
+{
+ if (!fence)
+ return -ENOENT;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+ return fence->status;
+#else
+ return atomic_read(&fence->status);
+#endif
+}
+
+static void kbase_fence_wait_callback(struct sync_fence *fence,
+ struct sync_fence_waiter *waiter)
+{
+ struct kbase_jd_atom *katom = container_of(waiter,
+ struct kbase_jd_atom, sync_waiter);
+ struct kbase_context *kctx = katom->kctx;
+
+ /* Propagate the fence status to the atom.
+ * If negative then cancel this atom and its dependencies.
+ */
+ if (kbase_fence_get_status(fence) < 0)
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ /* To prevent a potential deadlock we schedule the work onto the
+ * job_done_wq workqueue
+ *
+ * The issue is that we may signal the timeline while holding
+ * kctx->jctx.lock and the callbacks are run synchronously from
+ * sync_timeline_signal. So we simply defer the work.
+ */
+
+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker);
+ queue_work(kctx->jctx.job_done_wq, &katom->work);
+}
+
+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom)
+{
+ int ret;
+
+ sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback);
+
+ ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter);
+
+ if (ret == 1) {
+ /* Already signaled */
+ return 0;
+ }
+
+ if (ret < 0) {
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+ /* We should cause the dependent jobs in the bag to be failed,
+ * to do this we schedule the work queue to complete this job */
+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker);
+ queue_work(katom->kctx->jctx.job_done_wq, &katom->work);
+ }
+
+ return 1;
+}
+
+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom)
+{
+ if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) {
+ /* The wait wasn't cancelled - leave the cleanup for
+ * kbase_fence_wait_callback */
+ return;
+ }
+
+ /* Wait was cancelled - zap the atoms */
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ kbasep_remove_waiting_soft_job(katom);
+ kbase_finish_soft_job(katom);
+
+ if (jd_done_nolock(katom, NULL))
+ kbase_js_sched_all(katom->kctx->kbdev);
+}
+
+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom)
+{
+ if (katom->fence) {
+ sync_fence_put(katom->fence);
+ katom->fence = NULL;
+ }
+}
+
+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom)
+{
+ if (katom->fence) {
+ sync_fence_put(katom->fence);
+ katom->fence = NULL;
+ }
+}
+
+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info)
+{
+ if (!katom->fence)
+ return -ENOENT;
+
+ info->fence = katom->fence;
+ info->status = kbase_fence_get_status(katom->fence);
+ strlcpy(info->name, katom->fence->name, sizeof(info->name));
+
+ return 0;
+}
+
+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info)
+{
+ if (!katom->fence)
+ return -ENOENT;
+
+ info->fence = katom->fence;
+ info->status = kbase_fence_get_status(katom->fence);
+ strlcpy(info->name, katom->fence->name, sizeof(info->name));
+
+ return 0;
+}
+
+#ifdef CONFIG_MALI_FENCE_DEBUG
+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom)
+{
+ /* Dump out the full state of all the Android sync fences.
+ * The function sync_dump() isn't exported to modules, so force
+ * sync_fence_wait() to time out to trigger sync_dump().
+ */
+ if (katom->fence)
+ sync_fence_wait(katom->fence, 1);
+}
+#endif
diff --git a/mali_kbase/mali_kbase_sync_common.c b/mali_kbase/mali_kbase_sync_common.c
new file mode 100644
index 0000000..457def2
--- /dev/null
+++ b/mali_kbase/mali_kbase_sync_common.c
@@ -0,0 +1,43 @@
+/*
+ *
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * @file mali_kbase_sync_common.c
+ *
+ * Common code for our explicit fence functionality
+ */
+
+#include <linux/workqueue.h>
+#include "mali_kbase.h"
+
+void kbase_sync_fence_wait_worker(struct work_struct *data)
+{
+ struct kbase_jd_atom *katom;
+
+ katom = container_of(data, struct kbase_jd_atom, work);
+ kbase_soft_event_wait_callback(katom);
+}
+
+const char *kbase_sync_status_string(int status)
+{
+ if (status == 0)
+ return "signaled";
+ else if (status > 0)
+ return "active";
+ else
+ return "error";
+}
diff --git a/mali_kbase/mali_kbase_sync_file.c b/mali_kbase/mali_kbase_sync_file.c
new file mode 100644
index 0000000..4e1621c
--- /dev/null
+++ b/mali_kbase/mali_kbase_sync_file.c
@@ -0,0 +1,339 @@
+/*
+ *
+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE)
+ * Introduced in kernel 4.9.
+ * Android explicit fences (CONFIG_SYNC) can be used for older kernels
+ * (see mali_kbase_sync_android.c)
+ */
+
+#include <linux/sched.h>
+#include <linux/fdtable.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/anon_inodes.h>
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <linux/sync_file.h>
+#include <linux/slab.h>
+#include "mali_kbase_fence_defs.h"
+#include "mali_kbase_sync.h"
+#include "mali_kbase_fence.h"
+#include "mali_kbase.h"
+
+static const struct file_operations stream_fops = {
+ .owner = THIS_MODULE
+};
+
+int kbase_sync_fence_stream_create(const char *name, int *const out_fd)
+{
+ if (!out_fd)
+ return -EINVAL;
+
+ *out_fd = anon_inode_getfd(name, &stream_fops, NULL,
+ O_RDONLY | O_CLOEXEC);
+ if (*out_fd < 0)
+ return -EINVAL;
+
+ return 0;
+}
+
+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
+ struct sync_file *sync_file;
+ int fd;
+
+ fence = kbase_fence_out_new(katom);
+ if (!fence)
+ return -ENOMEM;
+
+ /* Take an extra reference to the fence on behalf of the katom.
+ * This is needed because sync_file_create() will take ownership of
+ * one of these refs */
+ dma_fence_get(fence);
+
+ /* create a sync_file fd representing the fence */
+ sync_file = sync_file_create(fence);
+ if (!sync_file) {
+ dma_fence_put(fence);
+ kbase_fence_out_remove(katom);
+ return -ENOMEM;
+ }
+
+ fd = get_unused_fd_flags(O_CLOEXEC);
+ if (fd < 0) {
+ fput(sync_file->file);
+ kbase_fence_out_remove(katom);
+ return fd;
+ }
+
+ fd_install(fd, sync_file->file);
+
+ return fd;
+}
+
+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence = sync_file_get_fence(fd);
+#else
+ struct dma_fence *fence = sync_file_get_fence(fd);
+#endif
+
+ if (!fence)
+ return -ENOENT;
+
+ kbase_fence_fence_in_set(katom, fence);
+
+ return 0;
+}
+
+int kbase_sync_fence_validate(int fd)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence = sync_file_get_fence(fd);
+#else
+ struct dma_fence *fence = sync_file_get_fence(fd);
+#endif
+
+ if (!fence)
+ return -EINVAL;
+
+ dma_fence_put(fence);
+
+ return 0; /* valid */
+}
+
+enum base_jd_event_code
+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result)
+{
+ int res;
+
+ if (!kbase_fence_out_is_ours(katom)) {
+ /* Not our fence */
+ return BASE_JD_EVENT_JOB_CANCELLED;
+ }
+
+ res = kbase_fence_out_signal(katom, result);
+ if (unlikely(res < 0)) {
+ dev_warn(katom->kctx->kbdev->dev,
+ "fence_signal() failed with %d\n", res);
+ }
+
+ kbase_sync_fence_out_remove(katom);
+
+ return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+static void kbase_fence_wait_callback(struct fence *fence,
+ struct fence_cb *cb)
+#else
+static void kbase_fence_wait_callback(struct dma_fence *fence,
+ struct dma_fence_cb *cb)
+#endif
+{
+ struct kbase_fence_cb *kcb = container_of(cb,
+ struct kbase_fence_cb,
+ fence_cb);
+ struct kbase_jd_atom *katom = kcb->katom;
+ struct kbase_context *kctx = katom->kctx;
+
+ /* Cancel atom if fence is erroneous */
+ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0)
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ if (kbase_fence_dep_count_dec_and_test(katom)) {
+ /* We take responsibility of handling this */
+ kbase_fence_dep_count_set(katom, -1);
+
+ /* To prevent a potential deadlock we schedule the work onto the
+ * job_done_wq workqueue
+ *
+ * The issue is that we may signal the timeline while holding
+ * kctx->jctx.lock and the callbacks are run synchronously from
+ * sync_timeline_signal. So we simply defer the work.
+ */
+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker);
+ queue_work(kctx->jctx.job_done_wq, &katom->work);
+ }
+}
+
+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom)
+{
+ int err;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
+
+ fence = kbase_fence_in_get(katom);
+ if (!fence)
+ return 0; /* no input fence to wait for, good to go! */
+
+ kbase_fence_dep_count_set(katom, 1);
+
+ err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback);
+
+ kbase_fence_put(fence);
+
+ if (likely(!err)) {
+ /* Test if the callbacks are already triggered */
+ if (kbase_fence_dep_count_dec_and_test(katom)) {
+ kbase_fence_free_callbacks(katom);
+ kbase_fence_dep_count_set(katom, -1);
+ return 0; /* Already signaled, good to go right now */
+ }
+
+ /* Callback installed, so we just need to wait for it... */
+ } else {
+ /* Failure */
+ kbase_fence_free_callbacks(katom);
+ kbase_fence_dep_count_set(katom, -1);
+
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ /* We should cause the dependent jobs in the bag to be failed,
+ * to do this we schedule the work queue to complete this job */
+
+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker);
+ queue_work(katom->kctx->jctx.job_done_wq, &katom->work);
+ }
+
+ return 1; /* completion to be done later by callback/worker */
+}
+
+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom)
+{
+ if (!kbase_fence_free_callbacks(katom)) {
+ /* The wait wasn't cancelled -
+ * leave the cleanup for kbase_fence_wait_callback */
+ return;
+ }
+
+ /* Take responsibility of completion */
+ kbase_fence_dep_count_set(katom, -1);
+
+ /* Wait was cancelled - zap the atoms */
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ kbasep_remove_waiting_soft_job(katom);
+ kbase_finish_soft_job(katom);
+
+ if (jd_done_nolock(katom, NULL))
+ kbase_js_sched_all(katom->kctx->kbdev);
+}
+
+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom)
+{
+ kbase_fence_out_remove(katom);
+}
+
+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom)
+{
+ kbase_fence_free_callbacks(katom);
+ kbase_fence_in_remove(katom);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+static void kbase_sync_fence_info_get(struct fence *fence,
+ struct kbase_sync_fence_info *info)
+#else
+static void kbase_sync_fence_info_get(struct dma_fence *fence,
+ struct kbase_sync_fence_info *info)
+#endif
+{
+ info->fence = fence;
+
+ /* translate into CONFIG_SYNC status:
+ * < 0 : error
+ * 0 : active
+ * 1 : signaled
+ */
+ if (dma_fence_is_signaled(fence)) {
+ if (fence->status < 0)
+ info->status = fence->status; /* signaled with error */
+ else
+ info->status = 1; /* signaled with success */
+ } else {
+ info->status = 0; /* still active (unsignaled) */
+ }
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
+ scnprintf(info->name, sizeof(info->name), "%u#%u",
+ fence->context, fence->seqno);
+#else
+ scnprintf(info->name, sizeof(info->name), "%llu#%u",
+ fence->context, fence->seqno);
+#endif
+}
+
+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
+
+ fence = kbase_fence_in_get(katom);
+ if (!fence)
+ return -ENOENT;
+
+ kbase_sync_fence_info_get(fence, info);
+
+ kbase_fence_put(fence);
+
+ return 0;
+}
+
+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom,
+ struct kbase_sync_fence_info *info)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+ struct fence *fence;
+#else
+ struct dma_fence *fence;
+#endif
+
+ fence = kbase_fence_out_get(katom);
+ if (!fence)
+ return -ENOENT;
+
+ kbase_sync_fence_info_get(fence, info);
+
+ kbase_fence_put(fence);
+
+ return 0;
+}
+
+
+#ifdef CONFIG_MALI_FENCE_DEBUG
+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom)
+{
+ /* Not implemented */
+}
+#endif
diff --git a/mali_kbase/mali_kbase_sync_user.c b/mali_kbase/mali_kbase_sync_user.c
deleted file mode 100644
index b9baa91..0000000
--- a/mali_kbase/mali_kbase_sync_user.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_sync_user.c
- *
- */
-
-#ifdef CONFIG_SYNC
-
-#include <linux/sched.h>
-#include <linux/fdtable.h>
-#include <linux/file.h>
-#include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/anon_inodes.h>
-#include <linux/version.h>
-#include <linux/uaccess.h>
-#include <mali_kbase_sync.h>
-
-static int kbase_stream_close(struct inode *inode, struct file *file)
-{
- struct sync_timeline *tl;
-
- tl = (struct sync_timeline *)file->private_data;
- BUG_ON(!tl);
- sync_timeline_destroy(tl);
- return 0;
-}
-
-static const struct file_operations stream_fops = {
- .owner = THIS_MODULE,
- .release = kbase_stream_close,
-};
-
-int kbase_stream_create(const char *name, int *const out_fd)
-{
- struct sync_timeline *tl;
-
- BUG_ON(!out_fd);
-
- tl = kbase_sync_timeline_alloc(name);
- if (!tl)
- return -EINVAL;
-
- *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY | O_CLOEXEC);
-
- if (*out_fd < 0) {
- sync_timeline_destroy(tl);
- return -EINVAL;
- }
-
- return 0;
-}
-
-int kbase_stream_create_fence(int tl_fd)
-{
- struct sync_timeline *tl;
- struct sync_pt *pt;
- struct sync_fence *fence;
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0)
- struct files_struct *files;
- struct fdtable *fdt;
-#endif
- int fd;
- struct file *tl_file;
-
- tl_file = fget(tl_fd);
- if (tl_file == NULL)
- return -EBADF;
-
- if (tl_file->f_op != &stream_fops) {
- fd = -EBADF;
- goto out;
- }
-
- tl = tl_file->private_data;
-
- pt = kbase_sync_pt_alloc(tl);
- if (!pt) {
- fd = -EFAULT;
- goto out;
- }
-
- fence = sync_fence_create("mali_fence", pt);
- if (!fence) {
- sync_pt_free(pt);
- fd = -EFAULT;
- goto out;
- }
-
- /* from here the fence owns the sync_pt */
-
- /* create a fd representing the fence */
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
- fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
- if (fd < 0) {
- sync_fence_put(fence);
- goto out;
- }
-#else
- fd = get_unused_fd();
- if (fd < 0) {
- sync_fence_put(fence);
- goto out;
- }
-
- files = current->files;
- spin_lock(&files->file_lock);
- fdt = files_fdtable(files);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
- __set_close_on_exec(fd, fdt);
-#else
- FD_SET(fd, fdt->close_on_exec);
-#endif
- spin_unlock(&files->file_lock);
-#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */
-
- /* bind fence to the new fd */
- sync_fence_install(fence, fd);
-
- out:
- fput(tl_file);
-
- return fd;
-}
-
-int kbase_fence_validate(int fd)
-{
- struct sync_fence *fence;
-
- fence = sync_fence_fdget(fd);
- if (!fence)
- return -EINVAL;
-
- sync_fence_put(fence);
- return 0;
-}
-
-#endif /* CONFIG_SYNC */
diff --git a/mali_kbase/mali_kbase_tlstream.c b/mali_kbase/mali_kbase_tlstream.c
index aa7850a..c952993 100644
--- a/mali_kbase/mali_kbase_tlstream.c
+++ b/mali_kbase/mali_kbase_tlstream.c
@@ -145,6 +145,7 @@ enum tl_msg_id_obj {
KBASE_TL_ATTRIB_ATOM_PRIORITY,
KBASE_TL_ATTRIB_ATOM_STATE,
KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE,
+ KBASE_TL_ATTRIB_ATOM_JIT,
KBASE_TL_ATTRIB_AS_CONFIG,
KBASE_TL_EVENT_LPU_SOFTSTOP,
KBASE_TL_EVENT_ATOM_SOFTSTOP_EX,
@@ -459,6 +460,13 @@ static const struct tp_desc tp_desc_obj[] = {
"atom"
},
{
+ KBASE_TL_ATTRIB_ATOM_JIT,
+ __stringify(KBASE_TL_ATTRIB_ATOM_JIT),
+ "jit done for atom",
+ "@pLL",
+ "atom,edit_addr,new_addr"
+ },
+ {
KBASE_TL_ATTRIB_AS_CONFIG,
__stringify(KBASE_TL_ATTRIB_AS_CONFIG),
"address space attributes",
@@ -1380,21 +1388,77 @@ void kbase_tlstream_term(void)
}
}
-int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags)
+static void kbase_create_timeline_objects(struct kbase_context *kctx)
{
+ struct kbase_device *kbdev = kctx->kbdev;
+ unsigned int lpu_id;
+ unsigned int as_nr;
+ struct kbasep_kctx_list_element *element;
+
+ /* Create LPU objects. */
+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
+ u32 *lpu =
+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
+ KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu);
+ }
+
+ /* Create Address Space objects. */
+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++)
+ KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr);
+
+ /* Create GPU object and make it retain all LPUs and address spaces. */
+ KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(
+ kbdev,
+ kbdev->gpu_props.props.raw_props.gpu_id,
+ kbdev->gpu_props.num_cores);
+
+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
+ void *lpu =
+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
+ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev);
+ }
+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++)
+ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(
+ &kbdev->as[as_nr],
+ kbdev);
+
+ /* Create object for each known context. */
+ mutex_lock(&kbdev->kctx_list_lock);
+ list_for_each_entry(element, &kbdev->kctx_list, link) {
+ KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(
+ element->kctx,
+ (u32)(element->kctx->id),
+ (u32)(element->kctx->tgid));
+ }
+ /* Before releasing the lock, reset body stream buffers.
+ * This will prevent context creation message to be directed to both
+ * summary and body stream.
+ */
+ kbase_tlstream_reset_body_streams();
+ mutex_unlock(&kbdev->kctx_list_lock);
+ /* Static object are placed into summary packet that needs to be
+ * transmitted first. Flush all streams to make it available to
+ * user space.
+ */
+ kbase_tlstream_flush_streams();
+}
+
+int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags)
+{
+ int ret;
u32 tlstream_enabled = TLSTREAM_ENABLED | flags;
if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) {
int rcode;
- *fd = anon_inode_getfd(
+ ret = anon_inode_getfd(
"[mali_tlstream]",
&kbasep_tlstream_fops,
kctx,
O_RDONLY | O_CLOEXEC);
- if (0 > *fd) {
+ if (ret < 0) {
atomic_set(&kbase_tlstream_enabled, 0);
- return *fd;
+ return ret;
}
/* Reset and initialize header streams. */
@@ -1430,11 +1494,17 @@ int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags)
1800000);
}
+ /* Summary stream was cleared during acquire.
+ * Create static timeline objects that will be
+ * read by client.
+ */
+ kbase_create_timeline_objects(kctx);
+
} else {
- *fd = -EBUSY;
+ ret = -EBUSY;
}
- return 0;
+ return ret;
}
void kbase_tlstream_flush_streams(void)
@@ -2161,6 +2231,35 @@ void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom)
kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}
+void __kbase_tlstream_tl_attrib_atom_jit(
+ void *atom, u64 edit_addr, u64 new_addr)
+{
+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom)
+ + sizeof(edit_addr) + sizeof(new_addr);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &edit_addr, sizeof(edit_addr));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &new_addr, sizeof(new_addr));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
void __kbase_tlstream_tl_attrib_as_config(
void *as, u64 transtab, u64 memattr, u64 transcfg)
{
diff --git a/mali_kbase/mali_kbase_tlstream.h b/mali_kbase/mali_kbase_tlstream.h
index 69e188e..c0a1117 100644
--- a/mali_kbase/mali_kbase_tlstream.h
+++ b/mali_kbase/mali_kbase_tlstream.h
@@ -38,7 +38,6 @@ void kbase_tlstream_term(void);
/**
* kbase_tlstream_acquire - acquire timeline stream file descriptor
* @kctx: kernel common context
- * @fd: timeline stream file descriptor
* @flags: timeline stream flags
*
* This descriptor is meant to be used by userspace timeline to gain access to
@@ -46,12 +45,11 @@ void kbase_tlstream_term(void);
* timeline client.
* Only one entity can own the descriptor at any given time. Descriptor shall be
* closed if unused. If descriptor cannot be obtained (i.e. when it is already
- * being used) argument fd will contain negative value.
+ * being used) return will be a negative value.
*
- * Return: zero on success (this does not necessarily mean that stream
- * descriptor could be returned), negative number on error
+ * Return: file descriptor on success, negative number on error
*/
-int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd, u32 flags);
+int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags);
/**
* kbase_tlstream_flush_streams - flush timeline streams.
@@ -137,6 +135,8 @@ void __kbase_tlstream_tl_attrib_atom_config(
void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio);
void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state);
void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom);
+void __kbase_tlstream_tl_attrib_atom_jit(
+ void *atom, u64 edit_addr, u64 new_addr);
void __kbase_tlstream_tl_attrib_as_config(
void *as, u64 transtab, u64 memattr, u64 transcfg);
void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom);
@@ -170,6 +170,13 @@ extern atomic_t kbase_tlstream_enabled;
__kbase_tlstream_##trace_name(__VA_ARGS__); \
} while (0)
+#define __TRACE_IF_ENABLED_JD(trace_name, ...) \
+ do { \
+ int enabled = atomic_read(&kbase_tlstream_enabled); \
+ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \
+ __kbase_tlstream_##trace_name(__VA_ARGS__); \
+ } while (0)
+
/*****************************************************************************/
/**
@@ -482,6 +489,15 @@ extern atomic_t kbase_tlstream_enabled;
__TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom)
/**
+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom
+ * @atom: atom identifier
+ * @edit_addr: address edited by jit
+ * @new_addr: address placed into the edited location
+ */
+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \
+ __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr)
+
+/**
* KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes
* @as: assigned address space
* @transtab: configuration of the TRANSTAB register
diff --git a/mali_kbase/mali_kbase_uku.h b/mali_kbase/mali_kbase_uku.h
index 3f2cb15..f3fcde4 100644
--- a/mali_kbase/mali_kbase_uku.h
+++ b/mali_kbase/mali_kbase_uku.h
@@ -66,6 +66,8 @@
#define BASE_UK_VERSION_MAJOR 10
#define BASE_UK_VERSION_MINOR 6
+#define LINUX_UK_BASE_MAGIC 0x80
+
struct kbase_uk_mem_alloc {
union uk_header header;
/* IN */
diff --git a/mali_kbase/mali_kbase_vinstr.c b/mali_kbase/mali_kbase_vinstr.c
index 7f7efc8..b2d46ed 100644
--- a/mali_kbase/mali_kbase_vinstr.c
+++ b/mali_kbase/mali_kbase_vinstr.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -1160,6 +1160,28 @@ static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
return HRTIMER_NORESTART;
}
+#ifdef CONFIG_DEBUG_OBJECT_TIMERS
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
+/**
+ * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(),
+ * rewritten
+ *
+ * @timer: high resolution timer
+ *
+ * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for
+ * earlier kernel versions it is not possible to call it explicitly.
+ * Since this function must accompany hrtimer_init_on_stack(), which
+ * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS
+ * is defined in order to avoid the warning about object on stack not being
+ * annotated, we rewrite it here to be used for earlier kernel versions.
+ */
+static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer)
+{
+ debug_object_free(timer, &hrtimer_debug_descr);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */
+#endif /* CONFIG_DEBUG_OBJECT_TIMERS */
+
/**
* kbasep_vinstr_service_task - HWC dumping service thread
*
@@ -1174,7 +1196,8 @@ static int kbasep_vinstr_service_task(void *data)
KBASE_DEBUG_ASSERT(vinstr_ctx);
- hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+
timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
timer.vinstr_ctx = vinstr_ctx;
@@ -1277,6 +1300,14 @@ static int kbasep_vinstr_service_task(void *data)
mutex_unlock(&vinstr_ctx->lock);
}
+#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
+ kbase_destroy_hrtimer_on_stack(&timer.hrtimer);
+#else
+ destroy_hrtimer_on_stack(&timer.hrtimer);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */
+#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */
+
return 0;
}
diff --git a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h
index 34f6d57..49e107f 100644
--- a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h
+++ b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -71,10 +71,3 @@
#define PLATFORM_FUNCS (NULL)
extern struct kbase_pm_callback_conf pm_callbacks;
-
-/**
- * Protected mode switch
- *
- * Attached value: pointer to @ref kbase_protected_ops
- */
-#define PROTECTED_CALLBACKS (NULL)
diff --git a/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c b/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c
deleted file mode 100644
index d237a38..0000000
--- a/mali_kbase/platform/juno_soc/mali_kbase_config_juno_soc.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <linux/ioport.h>
-#include <linux/thermal.h>
-#include <mali_kbase.h>
-#include <mali_kbase_defs.h>
-#include <mali_kbase_config.h>
-#include <mali_kbase_smc.h>
-
-/* Versatile Express (VE) Juno Development Platform */
-
-#define HARD_RESET_AT_POWER_OFF 0
-
-#ifndef CONFIG_OF
-static struct kbase_io_resources io_resources = {
- .job_irq_number = 65,
- .mmu_irq_number = 66,
- .gpu_irq_number = 64,
- .io_memory_region = {
- .start = 0x2D000000,
- .end = 0x2D000000 + (4096 * 4) - 1}
-};
-#endif
-
-static int pm_callback_power_on(struct kbase_device *kbdev)
-{
- /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */
- return 1;
-}
-
-static void pm_callback_power_off(struct kbase_device *kbdev)
-{
-#if HARD_RESET_AT_POWER_OFF
- /* Cause a GPU hard reset to test whether we have actually idled the GPU
- * and that we properly reconfigure the GPU on power up.
- * Usually this would be dangerous, but if the GPU is working correctly it should
- * be completely safe as the GPU should not be active at this point.
- * However this is disabled normally because it will most likely interfere with
- * bus logging etc.
- */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
- kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
-#endif
-}
-
-struct kbase_pm_callback_conf pm_callbacks = {
- .power_on_callback = pm_callback_power_on,
- .power_off_callback = pm_callback_power_off,
- .power_suspend_callback = NULL,
- .power_resume_callback = NULL
-};
-
-/*
- * Juno Protected Mode integration
- */
-
-/* SMC Function Numbers */
-#define JUNO_SMC_PROTECTED_ENTER_FUNC 0xff06
-#define JUNO_SMC_PROTECTED_RESET_FUNC 0xff07
-
-static int juno_protected_mode_enter(struct kbase_device *kbdev)
-{
- /* T62X in SoC detected */
- u64 ret = kbase_invoke_smc(SMC_OEN_SIP,
- JUNO_SMC_PROTECTED_ENTER_FUNC, false,
- 0, 0, 0);
- return ret;
-}
-
-/* TODO: Remove these externs, reset should should be done by the firmware */
-extern void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value,
- struct kbase_context *kctx);
-
-extern u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset,
- struct kbase_context *kctx);
-
-static int juno_protected_mode_reset(struct kbase_device *kbdev)
-{
-
- /* T62X in SoC detected */
- u64 ret = kbase_invoke_smc(SMC_OEN_SIP,
- JUNO_SMC_PROTECTED_RESET_FUNC, false,
- 0, 0, 0);
-
- /* The GPU should now be reset */
-
- return ret;
-}
-
-static bool juno_protected_mode_supported(struct kbase_device *kbdev)
-{
- u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
-
- /*
- * Protected mode is only supported for the built in GPU
- * _and_ only if the right firmware is running.
- *
- * Given that at init time the GPU is not powered up the
- * juno_protected_mode_reset function can't be used as
- * is needs to access GPU registers.
- * However, although we don't want the GPU to boot into
- * protected mode we know a GPU reset will be done after
- * this function is called so although we set the GPU to
- * protected mode it will exit protected mode before the
- * driver is ready to run work.
- */
- if (gpu_id == GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0) &&
- (kbdev->reg_start == 0x2d000000))
- return juno_protected_mode_enter(kbdev) == 0;
-
- return false;
-}
-
-struct kbase_protected_ops juno_protected_ops = {
- .protected_mode_enter = juno_protected_mode_enter,
- .protected_mode_reset = juno_protected_mode_reset,
- .protected_mode_supported = juno_protected_mode_supported,
-};
-
-static struct kbase_platform_config versatile_platform_config = {
-#ifndef CONFIG_OF
- .io_resources = &io_resources
-#endif
-};
-
-struct kbase_platform_config *kbase_get_platform_config(void)
-{
- return &versatile_platform_config;
-}
-
-int kbase_platform_early_init(void)
-{
- /* Nothing needed at this stage */
- return 0;
-}
diff --git a/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h b/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h
deleted file mode 100644
index ab29e9d..0000000
--- a/mali_kbase/platform/juno_soc/mali_kbase_config_platform.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * Maximum frequency GPU will be clocked at. Given in kHz.
- * This must be specified as there is no default value.
- *
- * Attached value: number in kHz
- * Default value: NA
- */
-#define GPU_FREQ_KHZ_MAX 600000
-/**
- * Minimum frequency GPU will be clocked at. Given in kHz.
- * This must be specified as there is no default value.
- *
- * Attached value: number in kHz
- * Default value: NA
- */
-#define GPU_FREQ_KHZ_MIN 600000
-
-/**
- * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
- *
- * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
- * for the function prototype.
- *
- * Attached value: A kbase_cpu_clk_speed_func.
- * Default Value: NA
- */
-#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed)
-
-/**
- * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
- *
- * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
- * for the function prototype.
- *
- * Attached value: A kbase_gpu_clk_speed_func.
- * Default Value: NA
- */
-#define GPU_SPEED_FUNC (NULL)
-
-/**
- * Power management configuration
- *
- * Attached value: pointer to @ref kbase_pm_callback_conf
- * Default value: See @ref kbase_pm_callback_conf
- */
-#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
-
-/**
- * Platform specific configuration functions
- *
- * Attached value: pointer to @ref kbase_platform_funcs_conf
- * Default value: See @ref kbase_platform_funcs_conf
- */
-#define PLATFORM_FUNCS (NULL)
-
-/**
- * Protected mode switch
- *
- * Attached value: pointer to @ref kbase_protected_ops
- */
-#define PROTECTED_CALLBACKS (&juno_protected_ops)
-
-extern struct kbase_pm_callback_conf pm_callbacks;
-#ifdef CONFIG_DEVFREQ_THERMAL
-extern struct devfreq_cooling_ops juno_model_ops;
-#endif
-extern struct kbase_protected_ops juno_protected_ops;
diff --git a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h
index dc4471b..02835f1 100644
--- a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h
+++ b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -72,11 +72,4 @@
*/
#define PLATFORM_FUNCS (NULL)
-/**
- * Protected mode switch
- *
- * Attached value: pointer to @ref kbase_protected_ops
- */
-#define PROTECTED_CALLBACKS (NULL)
-
extern struct kbase_pm_callback_conf pm_callbacks;
diff --git a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h
index b0490ca..0efbf39 100644
--- a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h
+++ b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -70,11 +70,4 @@
*/
#define PLATFORM_FUNCS (NULL)
-/**
- * Protected mode switch
- *
- * Attached value: pointer to @ref kbase_protected_ops
- */
-#define PROTECTED_CALLBACKS (NULL)
-
extern struct kbase_pm_callback_conf pm_callbacks;
diff --git a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h
index 22ffccb..dbdf21e 100644
--- a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h
+++ b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -72,11 +72,4 @@
*/
#define PLATFORM_FUNCS (NULL)
-/**
- * Protected mode switch
- *
- * Attached value: pointer to @ref kbase_protected_ops
- */
-#define PROTECTED_CALLBACKS (NULL)
-
extern struct kbase_pm_callback_conf pm_callbacks;
diff --git a/mali_kbase/protected_mode_switcher.h b/mali_kbase/protected_mode_switcher.h
new file mode 100644
index 0000000..5dc2f3b
--- /dev/null
+++ b/mali_kbase/protected_mode_switcher.h
@@ -0,0 +1,64 @@
+/*
+ *
+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _PROTECTED_MODE_SWITCH_H_
+#define _PROTECTED_MODE_SWITCH_H_
+
+struct protected_mode_device;
+
+/**
+ * struct protected_mode_ops - Callbacks for protected mode switch operations
+ *
+ * @protected_mode_enable: Callback to enable protected mode for device
+ * @protected_mode_disable: Callback to disable protected mode for device
+ */
+struct protected_mode_ops {
+ /**
+ * protected_mode_enable() - Enable protected mode on device
+ * @dev: The struct device
+ *
+ * Return: 0 on success, non-zero on error
+ */
+ int (*protected_mode_enable)(
+ struct protected_mode_device *protected_dev);
+
+ /**
+ * protected_mode_disable() - Disable protected mode on device, and
+ * reset device
+ * @dev: The struct device
+ *
+ * Return: 0 on success, non-zero on error
+ */
+ int (*protected_mode_disable)(
+ struct protected_mode_device *protected_dev);
+};
+
+/**
+ * struct protected_mode_device - Device structure for protected mode devices
+ *
+ * @ops - Callbacks associated with this device
+ * @data - Pointer to device private data
+ *
+ * This structure should be registered with the platform device using
+ * platform_set_drvdata().
+ */
+struct protected_mode_device {
+ struct protected_mode_ops ops;
+ void *data;
+};
+
+#endif /* _PROTECTED_MODE_SWITCH_H_ */
diff --git a/mali_kbase/sconscript b/mali_kbase/sconscript
index 137d532..ff23d7a 100644
--- a/mali_kbase/sconscript
+++ b/mali_kbase/sconscript
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -16,14 +16,13 @@
import sys
Import('env')
-if Glob('tests/sconscript'):
- SConscript( 'tests/sconscript' )
+SConscript( 'tests/sconscript' )
mock_test = 0
# Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data.
# For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0.
-if env['platform_config']=='devicetree':
+if env['platform_config']=='devicetree' or env['platform_config']=='juno_soc':
fake_platform_device = 0
else:
fake_platform_device = 1
@@ -33,10 +32,14 @@ kbase_src = [
Glob('*.c'),
Glob('backend/*/*.c'),
Glob('internal/*/*.c'),
- Glob('ipa/*.c'),
- Glob('platform/%s/*.c' % env['platform_config']),
+ Glob('ipa/*.c')
]
+if env['platform_config']=='juno_soc':
+ kbase_src += [Glob('platform/devicetree/*.c')]
+else:
+ kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])]
+
if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1':
kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')]
mock_test = 1
@@ -57,14 +60,14 @@ make_args = env.kernel_get_config_defines(ret_list = True,
fake = fake_platform_device) + [
'PLATFORM=%s' % env['platform'],
'MALI_ERROR_INJECT_ON=%s' % env['error_inject'],
- 'MALI_KERNEL_TEST_API=%s' % env['unit'],
+ 'MALI_KERNEL_TEST_API=%s' % env['debug'],
'MALI_UNIT_TEST=%s' % env['unit'],
'MALI_RELEASE_NAME=%s' % env['mali_release_name'],
'MALI_MOCK_TEST=%s' % mock_test,
'MALI_CUSTOMER_RELEASE=%s' % env['release'],
'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'],
'MALI_COVERAGE=%s' % env['coverage'],
- 'MALI_BUS_LOG=%s' % env['buslog'],
+ 'MALI_BUS_LOG=%s' % env['buslog']
]
kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src,
@@ -81,6 +84,9 @@ if env['os'] != 'android':
if int(env['ump']) == 1:
env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko')
+if 'smc_protected_mode_switcher' in env:
+ env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko')
+
env.KernelObjTarget('kbase', kbase)
env.AppendUnique(BASE=['cutils_linked_list'])
diff --git a/mali_kbase/platform/juno_soc/Kbuild b/mali_kbase/tests/Kbuild
index 5e1ce63..b4bed04 100644
--- a/mali_kbase/platform/juno_soc/Kbuild
+++ b/mali_kbase/tests/Kbuild
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -13,4 +13,5 @@
#
-mali_kbase-y += $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_juno_soc.o
+obj-$(CONFIG_MALI_KUTF) += kutf/
+obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/
diff --git a/mali_kbase/tests/Kconfig b/mali_kbase/tests/Kconfig
new file mode 100644
index 0000000..da0515c
--- /dev/null
+++ b/mali_kbase/tests/Kconfig
@@ -0,0 +1,17 @@
+#
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+source "drivers/gpu/arm/midgard/tests/kutf/Kconfig"
+source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig"
diff --git a/mali_kbase/tests/include/kutf/kutf_mem.h b/mali_kbase/tests/include/kutf/kutf_mem.h
new file mode 100644
index 0000000..0d145e4
--- /dev/null
+++ b/mali_kbase/tests/include/kutf/kutf_mem.h
@@ -0,0 +1,65 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KERNEL_UTF_MEM_H_
+#define _KERNEL_UTF_MEM_H_
+
+/* kutf_mem.h
+ * Functions for management of memory pools in the kernel.
+ *
+ * This module implements a memory pool allocator, allowing a test
+ * implementation to allocate linked allocations which can then be freed by a
+ * single free which releases all of the resources held by the entire pool.
+ *
+ * Note that it is not possible to free single resources within the pool once
+ * allocated.
+ */
+
+#include <linux/list.h>
+
+/**
+ * struct kutf_mempool - the memory pool context management structure
+ * @head: list head on which the allocations in this context are added to
+ *
+ */
+struct kutf_mempool {
+ struct list_head head;
+};
+
+/**
+ * kutf_mempool_init() - Initialize a memory pool.
+ * @pool: Memory pool structure to initialize, provided by the user
+ *
+ * Return: zero on success
+ */
+int kutf_mempool_init(struct kutf_mempool *pool);
+
+/**
+ * kutf_mempool_alloc() - Allocate memory from a pool
+ * @pool: Memory pool to allocate from
+ * @size: Size of memory wanted in number of bytes
+ *
+ * Return: Pointer to memory on success, NULL on failure.
+ */
+void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size);
+
+/**
+ * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it.
+ * @pool: The memory pool to free
+ */
+void kutf_mempool_destroy(struct kutf_mempool *pool);
+#endif /* _KERNEL_UTF_MEM_H_ */
diff --git a/mali_kbase/tests/include/kutf/kutf_resultset.h b/mali_kbase/tests/include/kutf/kutf_resultset.h
new file mode 100644
index 0000000..1cc85f1
--- /dev/null
+++ b/mali_kbase/tests/include/kutf/kutf_resultset.h
@@ -0,0 +1,121 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KERNEL_UTF_RESULTSET_H_
+#define _KERNEL_UTF_RESULTSET_H_
+
+/* kutf_resultset.h
+ * Functions and structures for handling test results and result sets.
+ *
+ * This section of the kernel UTF contains structures and functions used for the
+ * management of Results and Result Sets.
+ */
+
+/**
+ * enum kutf_result_status - Status values for a single Test error.
+ * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark
+ * results.
+ * @KUTF_RESULT_SKIP: The test was skipped.
+ * @KUTF_RESULT_UNKNOWN: The test has an unknown result.
+ * @KUTF_RESULT_PASS: The test result passed.
+ * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug
+ * message.
+ * @KUTF_RESULT_INFO: The test result passed, but raised
+ * an informative message.
+ * @KUTF_RESULT_WARN: The test result passed, but raised a warning
+ * message.
+ * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error.
+ * @KUTF_RESULT_FATAL: The test result failed with a fatal error.
+ * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF
+ * assertion failure.
+ * @KUTF_RESULT_COUNT: The current number of possible status messages.
+ */
+enum kutf_result_status {
+ KUTF_RESULT_BENCHMARK = -3,
+ KUTF_RESULT_SKIP = -2,
+ KUTF_RESULT_UNKNOWN = -1,
+
+ KUTF_RESULT_PASS = 0,
+ KUTF_RESULT_DEBUG = 1,
+ KUTF_RESULT_INFO = 2,
+ KUTF_RESULT_WARN = 3,
+ KUTF_RESULT_FAIL = 4,
+ KUTF_RESULT_FATAL = 5,
+ KUTF_RESULT_ABORT = 6,
+
+ KUTF_RESULT_COUNT
+};
+
+/* The maximum size of a kutf_result_status result when
+ * converted to a string
+ */
+#define KUTF_ERROR_MAX_NAME_SIZE 21
+
+#ifdef __KERNEL__
+
+#include <kutf/kutf_mem.h>
+
+/**
+ * struct kutf_result - Represents a single test result.
+ * @node: Next result in the list of results.
+ * @status: The status summary (pass / warn / fail / etc).
+ * @message: A more verbose status message.
+ */
+struct kutf_result {
+ struct list_head node;
+ enum kutf_result_status status;
+ const char *message;
+};
+
+/**
+ * kutf_create_result_set() - Create a new result set
+ * to which results can be added.
+ *
+ * Return: The created resultset.
+ */
+struct kutf_result_set *kutf_create_result_set(void);
+
+/**
+ * kutf_add_result() - Add a result to the end of an existing resultset.
+ *
+ * @mempool: The memory pool to allocate the result storage from.
+ * @set: The resultset to add the result to.
+ * @status: The result status to add.
+ * @message: The result message to add.
+ */
+void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set,
+ enum kutf_result_status status, const char *message);
+
+/**
+ * kutf_remove_result() - Remove a result from the head of a resultset.
+ * @set: The resultset.
+ *
+ * Return: result or NULL if there are no further results in the resultset.
+ */
+struct kutf_result *kutf_remove_result(
+ struct kutf_result_set *set);
+
+/**
+ * kutf_destroy_result_set() - Free a previously created resultset.
+ *
+ * @results: The result set whose resources to free.
+ */
+void kutf_destroy_result_set(struct kutf_result_set *results);
+
+#endif /* __KERNEL__ */
+
+#endif /* _KERNEL_UTF_RESULTSET_H_ */
diff --git a/mali_kbase/tests/include/kutf/kutf_suite.h b/mali_kbase/tests/include/kutf/kutf_suite.h
new file mode 100644
index 0000000..754c3ad
--- /dev/null
+++ b/mali_kbase/tests/include/kutf/kutf_suite.h
@@ -0,0 +1,508 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KERNEL_UTF_SUITE_H_
+#define _KERNEL_UTF_SUITE_H_
+
+/* kutf_suite.h
+ * Functions for management of test suites.
+ *
+ * This collection of data structures, macros, and functions are used to
+ * create Test Suites, Tests within those Test Suites, and Fixture variants
+ * of each test.
+ */
+
+#include <kutf/kutf_mem.h>
+#include <kutf/kutf_resultset.h>
+
+/**
+ * Pseudo-flag indicating an absence of any specified test class. Note that
+ * tests should not be annotated with this constant as it is simply a zero
+ * value; tests without a more specific class must be marked with the flag
+ * KUTF_F_TEST_GENERIC.
+ */
+#define KUTF_F_TEST_NONE ((unsigned int)(0))
+
+/**
+ * Class indicating this test is a smoke test.
+ * A given set of smoke tests should be quick to run, enabling rapid turn-around
+ * of "regress-on-commit" test runs.
+ */
+#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1))
+
+/**
+ * Class indicating this test is a performance test.
+ * These tests typically produce a performance metric, such as "time to run" or
+ * "frames per second",
+ */
+#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2))
+
+/**
+ * Class indicating that this test is a deprecated test.
+ * These tests have typically been replaced by an alternative test which is
+ * more efficient, or has better coverage.
+ */
+#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3))
+
+/**
+ * Class indicating that this test is a known failure.
+ * These tests have typically been run and failed, but marking them as a known
+ * failure means it is easier to triage results.
+ *
+ * It is typically more convenient to triage known failures using the
+ * results database and web UI, as this means there is no need to modify the
+ * test code.
+ */
+#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4))
+
+/**
+ * Class indicating that this test is a generic test, which is not a member of
+ * a more specific test class. Tests which are not created with a specific set
+ * of filter flags by the user are assigned this test class by default.
+ */
+#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5))
+
+/**
+ * Class indicating this test is a resource allocation failure test.
+ * A resource allocation failure test will test that an error code is
+ * correctly propagated when an allocation fails.
+ */
+#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6))
+
+/**
+ * Additional flag indicating that this test is an expected failure when
+ * run in resource failure mode. These tests are never run when running
+ * the low resource mode.
+ */
+#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7))
+
+/**
+ * Flag reserved for user-defined filter zero.
+ */
+#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24))
+
+/**
+ * Flag reserved for user-defined filter one.
+ */
+#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25))
+
+/**
+ * Flag reserved for user-defined filter two.
+ */
+#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26))
+
+/**
+ * Flag reserved for user-defined filter three.
+ */
+#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27))
+
+/**
+ * Flag reserved for user-defined filter four.
+ */
+#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28))
+
+/**
+ * Flag reserved for user-defined filter five.
+ */
+#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29))
+
+/**
+ * Flag reserved for user-defined filter six.
+ */
+#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30))
+
+/**
+ * Flag reserved for user-defined filter seven.
+ */
+#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31))
+
+/**
+ * Pseudo-flag indicating that all test classes should be executed.
+ */
+#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU))
+
+/**
+ * union kutf_callback_data - Union used to store test callback data
+ * @ptr_value: pointer to the location where test callback data
+ * are stored
+ * @u32_value: a number which represents test callback data
+ */
+union kutf_callback_data {
+ void *ptr_value;
+ u32 u32_value;
+};
+
+/**
+ * struct kutf_context - Structure representing a kernel test context
+ * @suite: Convenience pointer to the suite this context
+ * is running
+ * @test_fix: The fixture that is being run in this context
+ * @fixture_pool: The memory pool used for the duration of
+ * the fixture/text context.
+ * @fixture: The user provided fixture structure.
+ * @fixture_index: The index (id) of the current fixture.
+ * @fixture_name: The name of the current fixture (or NULL if unnamed).
+ * @test_data: Any user private data associated with this test
+ * @result_set: All the results logged by this test context
+ * @status: The status of the currently running fixture.
+ * @expected_status: The expected status on exist of the currently
+ * running fixture.
+ */
+struct kutf_context {
+ struct kutf_suite *suite;
+ struct kutf_test_fixture *test_fix;
+ struct kutf_mempool fixture_pool;
+ void *fixture;
+ unsigned int fixture_index;
+ const char *fixture_name;
+ union kutf_callback_data test_data;
+ struct kutf_result_set *result_set;
+ enum kutf_result_status status;
+ enum kutf_result_status expected_status;
+};
+
+/**
+ * struct kutf_suite - Structure representing a kernel test suite
+ * @app: The application this suite belongs to.
+ * @name: The name of this suite.
+ * @suite_data: Any user private data associated with this
+ * suite.
+ * @create_fixture: Function used to create a new fixture instance
+ * @remove_fixture: Function used to destroy a new fixture instance
+ * @fixture_variants: The number of variants (must be at least 1).
+ * @suite_default_flags: Suite global filter flags which are set on
+ * all tests.
+ * @node: List node for suite_list
+ * @dir: The debugfs directory for this suite
+ * @test_list: List head to store all the tests which are
+ * part of this suite
+ */
+struct kutf_suite {
+ struct kutf_application *app;
+ const char *name;
+ union kutf_callback_data suite_data;
+ void *(*create_fixture)(struct kutf_context *context);
+ void (*remove_fixture)(struct kutf_context *context);
+ unsigned int fixture_variants;
+ unsigned int suite_default_flags;
+ struct list_head node;
+ struct dentry *dir;
+ struct list_head test_list;
+};
+
+/* ============================================================================
+ Application functions
+============================================================================ */
+
+/**
+ * kutf_create_application() - Create an in kernel test application.
+ * @name: The name of the test application.
+ *
+ * Return: pointer to the kutf_application on success or NULL
+ * on failure
+ */
+struct kutf_application *kutf_create_application(const char *name);
+
+/**
+ * kutf_destroy_application() - Destroy an in kernel test application.
+ *
+ * @app: The test application to destroy.
+ */
+void kutf_destroy_application(struct kutf_application *app);
+
+/* ============================================================================
+ Suite functions
+============================================================================ */
+
+/**
+ * kutf_create_suite() - Create a kernel test suite.
+ * @app: The test application to create the suite in.
+ * @name: The name of the suite.
+ * @fixture_count: The number of fixtures to run over the test
+ * functions in this suite
+ * @create_fixture: Callback used to create a fixture. The returned value
+ * is stored in the fixture pointer in the context for
+ * use in the test functions.
+ * @remove_fixture: Callback used to remove a previously created fixture.
+ *
+ * Suite names must be unique. Should two suites with the same name be
+ * registered with the same application then this function will fail, if they
+ * are registered with different applications then the function will not detect
+ * this and the call will succeed.
+ *
+ * Return: pointer to the created kutf_suite on success or NULL
+ * on failure
+ */
+struct kutf_suite *kutf_create_suite(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context));
+
+/**
+ * kutf_create_suite_with_filters() - Create a kernel test suite with user
+ * defined default filters.
+ * @app: The test application to create the suite in.
+ * @name: The name of the suite.
+ * @fixture_count: The number of fixtures to run over the test
+ * functions in this suite
+ * @create_fixture: Callback used to create a fixture. The returned value
+ * is stored in the fixture pointer in the context for
+ * use in the test functions.
+ * @remove_fixture: Callback used to remove a previously created fixture.
+ * @filters: Filters to apply to a test if it doesn't provide its own
+ *
+ * Suite names must be unique. Should two suites with the same name be
+ * registered with the same application then this function will fail, if they
+ * are registered with different applications then the function will not detect
+ * this and the call will succeed.
+ *
+ * Return: pointer to the created kutf_suite on success or NULL on failure
+ */
+struct kutf_suite *kutf_create_suite_with_filters(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context),
+ unsigned int filters);
+
+/**
+ * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with
+ * user defined default filters.
+ * @app: The test application to create the suite in.
+ * @name: The name of the suite.
+ * @fixture_count: The number of fixtures to run over the test
+ * functions in this suite
+ * @create_fixture: Callback used to create a fixture. The returned value
+ * is stored in the fixture pointer in the context for
+ * use in the test functions.
+ * @remove_fixture: Callback used to remove a previously created fixture.
+ * @filters: Filters to apply to a test if it doesn't provide its own
+ * @suite_data: Suite specific callback data, provided during the
+ * running of the test in the kutf_context
+ *
+ * Return: pointer to the created kutf_suite on success or NULL
+ * on failure
+ */
+struct kutf_suite *kutf_create_suite_with_filters_and_data(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context),
+ unsigned int filters,
+ union kutf_callback_data suite_data);
+
+/**
+ * kutf_add_test() - Add a test to a kernel test suite.
+ * @suite: The suite to add the test to.
+ * @id: The ID of the test.
+ * @name: The name of the test.
+ * @execute: Callback to the test function to run.
+ *
+ * Note: As no filters are provided the test will use the suite filters instead
+ */
+void kutf_add_test(struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context));
+
+/**
+ * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters
+ * @suite: The suite to add the test to.
+ * @id: The ID of the test.
+ * @name: The name of the test.
+ * @execute: Callback to the test function to run.
+ * @filters: A set of filtering flags, assigning test categories.
+ */
+void kutf_add_test_with_filters(struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context),
+ unsigned int filters);
+
+/**
+ * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite
+ * with filters.
+ * @suite: The suite to add the test to.
+ * @id: The ID of the test.
+ * @name: The name of the test.
+ * @execute: Callback to the test function to run.
+ * @filters: A set of filtering flags, assigning test categories.
+ * @test_data: Test specific callback data, provoided during the
+ * running of the test in the kutf_context
+ */
+void kutf_add_test_with_filters_and_data(
+ struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context),
+ unsigned int filters,
+ union kutf_callback_data test_data);
+
+/* ============================================================================
+ Test functions
+============================================================================ */
+/**
+ * kutf_test_log_result_external() - Log a result which has been created
+ * externally into a in a standard form
+ * recognized by the log parser.
+ * @context: The test context the test is running in
+ * @message: The message for this result
+ * @new_status: The result status of this log message
+ */
+void kutf_test_log_result_external(
+ struct kutf_context *context,
+ const char *message,
+ enum kutf_result_status new_status);
+
+/**
+ * kutf_test_expect_abort() - Tell the kernel that you expect the current
+ * fixture to produce an abort.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_expect_abort(struct kutf_context *context);
+
+/**
+ * kutf_test_expect_fatal() - Tell the kernel that you expect the current
+ * fixture to produce a fatal error.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_expect_fatal(struct kutf_context *context);
+
+/**
+ * kutf_test_expect_fail() - Tell the kernel that you expect the current
+ * fixture to fail.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_expect_fail(struct kutf_context *context);
+
+/**
+ * kutf_test_expect_warn() - Tell the kernel that you expect the current
+ * fixture to produce a warning.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_expect_warn(struct kutf_context *context);
+
+/**
+ * kutf_test_expect_pass() - Tell the kernel that you expect the current
+ * fixture to pass.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_expect_pass(struct kutf_context *context);
+
+/**
+ * kutf_test_skip() - Tell the kernel that the test should be skipped.
+ * @context: The test context this test is running in.
+ */
+void kutf_test_skip(struct kutf_context *context);
+
+/**
+ * kutf_test_skip_msg() - Tell the kernel that this test has been skipped,
+ * supplying a reason string.
+ * @context: The test context this test is running in.
+ * @message: A message string containing the reason for the skip.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a prebaked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_skip_msg(struct kutf_context *context, const char *message);
+
+/**
+ * kutf_test_pass() - Tell the kernel that this test has passed.
+ * @context: The test context this test is running in.
+ * @message: A message string containing the reason for the pass.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_pass(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_debug() - Send a debug message
+ * @context: The test context this test is running in.
+ * @message: A message string containing the debug information.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_debug(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_info() - Send an information message
+ * @context: The test context this test is running in.
+ * @message: A message string containing the information message.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_info(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_warn() - Send a warning message
+ * @context: The test context this test is running in.
+ * @message: A message string containing the warning message.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_warn(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_fail() - Tell the kernel that a test has failed
+ * @context: The test context this test is running in.
+ * @message: A message string containing the failure message.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_fail(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error
+ * @context: The test context this test is running in.
+ * @message: A message string containing the fatal error message.
+ *
+ * Note: The message must not be freed during the lifetime of the test run.
+ * This means it should either be a pre-baked string, or if a dynamic string
+ * is required it must be created with kutf_dsprintf which will store
+ * the resultant string in a buffer who's lifetime is the same as the test run.
+ */
+void kutf_test_fatal(struct kutf_context *context, char const *message);
+
+/**
+ * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test
+ *
+ * @context: The test context this test is running in.
+ */
+void kutf_test_abort(struct kutf_context *context);
+
+#endif /* _KERNEL_UTF_SUITE_H_ */
diff --git a/mali_kbase/tests/include/kutf/kutf_utils.h b/mali_kbase/tests/include/kutf/kutf_utils.h
new file mode 100644
index 0000000..c458c1f
--- /dev/null
+++ b/mali_kbase/tests/include/kutf/kutf_utils.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KERNEL_UTF_UTILS_H_
+#define _KERNEL_UTF_UTILS_H_
+
+/* kutf_utils.h
+ * Utilities for the kernel UTF test infrastructure.
+ *
+ * This collection of library functions are provided for use by kernel UTF
+ * and users of kernel UTF which don't directly fit within the other
+ * code modules.
+ */
+
+#include <kutf/kutf_mem.h>
+
+/**
+ * Maximum size of the message strings within kernel UTF, messages longer then
+ * this will be truncated.
+ */
+#define KUTF_MAX_DSPRINTF_LEN 1024
+
+/**
+ * kutf_dsprintf() - dynamic sprintf
+ * @pool: memory pool to allocate from
+ * @fmt: The format string describing the string to document.
+ * @... The parameters to feed in to the format string.
+ *
+ * This function implements sprintf which dynamically allocates memory to store
+ * the string. The library will free the memory containing the string when the
+ * result set is cleared or destroyed.
+ *
+ * Note The returned string may be truncated to fit an internal temporary
+ * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length.
+ *
+ * Return: Returns pointer to allocated string, or NULL on error.
+ */
+const char *kutf_dsprintf(struct kutf_mempool *pool,
+ const char *fmt, ...);
+
+#endif /* _KERNEL_UTF_UTILS_H_ */
diff --git a/mali_kbase/tests/kutf/Kbuild b/mali_kbase/tests/kutf/Kbuild
new file mode 100644
index 0000000..6b840c2
--- /dev/null
+++ b/mali_kbase/tests/kutf/Kbuild
@@ -0,0 +1,20 @@
+#
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+ccflags-y += -I$(src)/../include
+
+obj-$(CONFIG_MALI_KUTF) += kutf.o
+
+kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o
diff --git a/mali_kbase/tests/kutf/Kconfig b/mali_kbase/tests/kutf/Kconfig
new file mode 100644
index 0000000..6a87bdb
--- /dev/null
+++ b/mali_kbase/tests/kutf/Kconfig
@@ -0,0 +1,22 @@
+#
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+
+config MALI_KUTF
+ tristate "Mali Kernel Unit Test Framework"
+ default m
+ help
+ Enables MALI testing framework. To compile it as a module,
+ choose M here - this will generate a single module called kutf.
diff --git a/mali_kbase/tests/kutf/Makefile b/mali_kbase/tests/kutf/Makefile
new file mode 100644
index 0000000..010c92c
--- /dev/null
+++ b/mali_kbase/tests/kutf/Makefile
@@ -0,0 +1,29 @@
+#
+# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+# linux build system bootstrap for out-of-tree module
+
+# default to building for the host
+ARCH ?= $(shell uname -m)
+
+ifeq ($(KDIR),)
+$(error Must specify KDIR to point to the kernel to target))
+endif
+
+all:
+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules
+
+clean:
+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
diff --git a/mali_kbase/tests/kutf/kutf_mem.c b/mali_kbase/tests/kutf/kutf_mem.c
new file mode 100644
index 0000000..5408e57
--- /dev/null
+++ b/mali_kbase/tests/kutf/kutf_mem.c
@@ -0,0 +1,94 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/* Kernel UTF memory management functions */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+
+#include <kutf/kutf_mem.h>
+
+
+/**
+ * struct kutf_alloc_entry - Structure representing an allocation.
+ * @node: List node for use with kutf_mempool.
+ * @data: Data area of the allocation
+ */
+struct kutf_alloc_entry {
+ struct list_head node;
+ u8 data[0];
+};
+
+int kutf_mempool_init(struct kutf_mempool *pool)
+{
+ if (!pool) {
+ pr_err("NULL pointer passed to %s\n", __func__);
+ return -1;
+ }
+
+ INIT_LIST_HEAD(&pool->head);
+
+ return 0;
+}
+EXPORT_SYMBOL(kutf_mempool_init);
+
+void kutf_mempool_destroy(struct kutf_mempool *pool)
+{
+ struct list_head *remove;
+ struct list_head *tmp;
+
+ if (!pool) {
+ pr_err("NULL pointer passed to %s\n", __func__);
+ return;
+ }
+
+ list_for_each_safe(remove, tmp, &pool->head) {
+ struct kutf_alloc_entry *remove_alloc;
+
+ remove_alloc = list_entry(remove, struct kutf_alloc_entry, node);
+ list_del(&remove_alloc->node);
+ kfree(remove_alloc);
+ }
+}
+EXPORT_SYMBOL(kutf_mempool_destroy);
+
+void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size)
+{
+ struct kutf_alloc_entry *ret;
+
+ if (!pool) {
+ pr_err("NULL pointer passed to %s\n", __func__);
+ goto fail_pool;
+ }
+
+ ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL);
+ if (!ret) {
+ pr_err("Failed to allocate memory\n");
+ goto fail_alloc;
+ }
+
+ INIT_LIST_HEAD(&ret->node);
+ list_add(&ret->node, &pool->head);
+
+ return &ret->data[0];
+
+fail_alloc:
+fail_pool:
+ return NULL;
+}
+EXPORT_SYMBOL(kutf_mempool_alloc);
diff --git a/mali_kbase/tests/kutf/kutf_resultset.c b/mali_kbase/tests/kutf/kutf_resultset.c
new file mode 100644
index 0000000..5bd0496
--- /dev/null
+++ b/mali_kbase/tests/kutf/kutf_resultset.c
@@ -0,0 +1,95 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/* Kernel UTF result management functions */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+
+#include <kutf/kutf_resultset.h>
+
+/**
+ * struct kutf_result_set - Represents a set of results.
+ * @results: Pointer to the linked list where the results are stored.
+ */
+struct kutf_result_set {
+ struct list_head results;
+};
+
+struct kutf_result_set *kutf_create_result_set(void)
+{
+ struct kutf_result_set *set;
+
+ set = kmalloc(sizeof(*set), GFP_KERNEL);
+ if (!set) {
+ pr_err("Failed to allocate resultset");
+ goto fail_alloc;
+ }
+
+ INIT_LIST_HEAD(&set->results);
+
+ return set;
+
+fail_alloc:
+ return NULL;
+}
+
+void kutf_add_result(struct kutf_mempool *mempool,
+ struct kutf_result_set *set,
+ enum kutf_result_status status,
+ const char *message)
+{
+ /* Create the new result */
+ struct kutf_result *new_result;
+
+ BUG_ON(set == NULL);
+
+ new_result = kutf_mempool_alloc(mempool, sizeof(*new_result));
+ if (!new_result) {
+ pr_err("Result allocation failed\n");
+ return;
+ }
+
+ INIT_LIST_HEAD(&new_result->node);
+ new_result->status = status;
+ new_result->message = message;
+
+ list_add_tail(&new_result->node, &set->results);
+}
+
+void kutf_destroy_result_set(struct kutf_result_set *set)
+{
+ if (!list_empty(&set->results))
+ pr_err("kutf_destroy_result_set: Unread results from test\n");
+
+ kfree(set);
+}
+
+struct kutf_result *kutf_remove_result(struct kutf_result_set *set)
+{
+ if (!list_empty(&set->results)) {
+ struct kutf_result *ret;
+
+ ret = list_first_entry(&set->results, struct kutf_result, node);
+ list_del(&ret->node);
+ return ret;
+ }
+
+ return NULL;
+}
+
diff --git a/mali_kbase/tests/kutf/kutf_suite.c b/mali_kbase/tests/kutf/kutf_suite.c
new file mode 100644
index 0000000..a7cfd3b
--- /dev/null
+++ b/mali_kbase/tests/kutf/kutf_suite.c
@@ -0,0 +1,1041 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/* Kernel UTF suite, test and fixture management including user to kernel
+ * interaction */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/version.h>
+
+#include <generated/autoconf.h>
+
+#include <kutf/kutf_suite.h>
+#include <kutf/kutf_resultset.h>
+#include <kutf/kutf_utils.h>
+
+#if defined(CONFIG_DEBUG_FS)
+
+/**
+ * struct kutf_application - Structure which represents kutf application
+ * @name: The name of this test application.
+ * @dir: The debugfs directory for this test
+ * @suite_list: List head to store all the suites which are part of this
+ * application
+ */
+struct kutf_application {
+ const char *name;
+ struct dentry *dir;
+ struct list_head suite_list;
+};
+
+/**
+ * struct kutf_test_function - Structure which represents kutf test function
+ * @suite: Back reference to the suite this test function
+ * belongs to
+ * @filters: Filters that apply to this test function
+ * @test_id: Test ID
+ * @execute: Function to run for this test
+ * @test_data: Static data for this test
+ * @node: List node for test_list
+ * @variant_list: List head to store all the variants which can run on
+ * this function
+ * @dir: debugfs directory for this test function
+ */
+struct kutf_test_function {
+ struct kutf_suite *suite;
+ unsigned int filters;
+ unsigned int test_id;
+ void (*execute)(struct kutf_context *context);
+ union kutf_callback_data test_data;
+ struct list_head node;
+ struct list_head variant_list;
+ struct dentry *dir;
+};
+
+/**
+ * struct kutf_test_fixture - Structure which holds information on the kutf
+ * test fixture
+ * @test_func: Test function this fixture belongs to
+ * @fixture_index: Index of this fixture
+ * @node: List node for variant_list
+ * @dir: debugfs directory for this test fixture
+ */
+struct kutf_test_fixture {
+ struct kutf_test_function *test_func;
+ unsigned int fixture_index;
+ struct list_head node;
+ struct dentry *dir;
+};
+
+struct dentry *base_dir;
+
+/**
+ * struct kutf_convert_table - Structure which keeps test results
+ * @result_name: Status of the test result
+ * @result: Status value for a single test
+ */
+struct kutf_convert_table {
+ char result_name[50];
+ enum kutf_result_status result;
+};
+
+struct kutf_convert_table kutf_convert[] = {
+#define ADD_UTF_RESULT(_name) \
+{ \
+ #_name, \
+ _name, \
+},
+ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK)
+ADD_UTF_RESULT(KUTF_RESULT_SKIP)
+ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN)
+ADD_UTF_RESULT(KUTF_RESULT_PASS)
+ADD_UTF_RESULT(KUTF_RESULT_DEBUG)
+ADD_UTF_RESULT(KUTF_RESULT_INFO)
+ADD_UTF_RESULT(KUTF_RESULT_WARN)
+ADD_UTF_RESULT(KUTF_RESULT_FAIL)
+ADD_UTF_RESULT(KUTF_RESULT_FATAL)
+ADD_UTF_RESULT(KUTF_RESULT_ABORT)
+};
+
+#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert))
+
+/**
+ * kutf_create_context() - Create a test context in which a specific fixture
+ * of an application will be run and its results
+ * reported back to the user
+ * @test_fix: Test fixture to be run.
+ *
+ * Return: Returns the created test context on success or NULL on failure
+ */
+static struct kutf_context *kutf_create_context(
+ struct kutf_test_fixture *test_fix);
+
+/**
+ * kutf_destroy_context() - Destroy a previously created test context
+ * @context: Test context to destroy
+ */
+static void kutf_destroy_context(struct kutf_context *context);
+
+/**
+ * kutf_set_result() - Set the test result against the specified test context
+ * @context: Test context
+ * @status: Result status
+ */
+static void kutf_set_result(struct kutf_context *context,
+ enum kutf_result_status status);
+
+/**
+ * kutf_set_expected_result() - Set the expected test result for the specified
+ * test context
+ * @context: Test context
+ * @expected_status: Expected result status
+ */
+static void kutf_set_expected_result(struct kutf_context *context,
+ enum kutf_result_status expected_status);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0))
+/* Pre 3.4.0 kernels don't have the simple_open helper */
+
+/**
+ * simple_open() - Helper for file opening which stores the inode private data
+ * into the file private data
+ * @inode: File entry representation
+ * @file: A specific opening of the file
+ *
+ * Return: always 0; if inode private data do not exist, the file will not
+ * be assigned private data
+ */
+static int simple_open(struct inode *inode, struct file *file)
+{
+ if (inode->i_private)
+ file->private_data = inode->i_private;
+ return 0;
+}
+#endif
+
+/**
+ * kutf_result_to_string() - Converts a KUTF result into a string
+ * @result_str: Output result string
+ * @result: Result status to convert
+ *
+ * Return: 1 if test result was successfully converted to string, 0 otherwise
+ */
+static int kutf_result_to_string(char **result_str,
+ enum kutf_result_status result)
+{
+ int i;
+ int ret = 0;
+
+ for (i = 0; i < UTF_CONVERT_SIZE; i++) {
+ if (result == kutf_convert[i].result) {
+ *result_str = kutf_convert[i].result_name;
+ ret = 1;
+ }
+ }
+ return ret;
+}
+
+/**
+ * kutf_debugfs_const_string_read() - Simple debugfs read callback which
+ * returns a constant string
+ * @file: Opened file to read from
+ * @buf: User buffer to write the data into
+ * @len: Amount of data to read
+ * @ppos: Offset into file to read from
+ *
+ * Return: On success, the number of bytes read and offset @ppos advanced by
+ * this number; on error, negative value
+ */
+static ssize_t kutf_debugfs_const_string_read(struct file *file,
+ char __user *buf, size_t len, loff_t *ppos)
+{
+ char *str = file->private_data;
+
+ return simple_read_from_buffer(buf, len, ppos, str, strlen(str));
+}
+
+static const struct file_operations kutf_debugfs_const_string_ops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = kutf_debugfs_const_string_read,
+ .llseek = default_llseek,
+};
+
+/**
+ * kutf_add_explicit_result() - Check if an explicit result needs to be added
+ * @context: KUTF test context
+ */
+static void kutf_add_explicit_result(struct kutf_context *context)
+{
+ switch (context->expected_status) {
+ case KUTF_RESULT_UNKNOWN:
+ if (context->status == KUTF_RESULT_UNKNOWN)
+ kutf_test_pass(context, "(implicit pass)");
+ break;
+
+ case KUTF_RESULT_WARN:
+ if (context->status == KUTF_RESULT_WARN)
+ kutf_test_pass(context,
+ "Pass (expected warn occurred)");
+ else if (context->status != KUTF_RESULT_SKIP)
+ kutf_test_fail(context,
+ "Fail (expected warn missing)");
+ break;
+
+ case KUTF_RESULT_FAIL:
+ if (context->status == KUTF_RESULT_FAIL)
+ kutf_test_pass(context,
+ "Pass (expected fail occurred)");
+ else if (context->status != KUTF_RESULT_SKIP) {
+ /* Force the expected status so the fail gets logged */
+ context->expected_status = KUTF_RESULT_PASS;
+ kutf_test_fail(context,
+ "Fail (expected fail missing)");
+ }
+ break;
+
+ case KUTF_RESULT_FATAL:
+ if (context->status == KUTF_RESULT_FATAL)
+ kutf_test_pass(context,
+ "Pass (expected fatal occurred)");
+ else if (context->status != KUTF_RESULT_SKIP)
+ kutf_test_fail(context,
+ "Fail (expected fatal missing)");
+ break;
+
+ case KUTF_RESULT_ABORT:
+ if (context->status == KUTF_RESULT_ABORT)
+ kutf_test_pass(context,
+ "Pass (expected abort occurred)");
+ else if (context->status != KUTF_RESULT_SKIP)
+ kutf_test_fail(context,
+ "Fail (expected abort missing)");
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * kutf_debugfs_run_open() Debugfs open callback for the "run" entry.
+ * @inode: inode of the opened file
+ * @file: Opened file to read from
+ *
+ * This function retrieves the test fixture data that is associated with the
+ * opened file and works back to get the test, suite and application so
+ * it can then run the test that is associated with the file entry.
+ *
+ * Return: 0 on success
+ */
+static int kutf_debugfs_run_open(struct inode *inode, struct file *file)
+{
+ struct kutf_test_fixture *test_fix = inode->i_private;
+ struct kutf_test_function *test_func = test_fix->test_func;
+ struct kutf_suite *suite = test_func->suite;
+ struct kutf_context *test_context;
+
+ test_context = kutf_create_context(test_fix);
+ if (!test_context)
+ return -ENODEV;
+
+ file->private_data = test_context;
+
+ /*
+ * Call the create fixture function if required before the
+ * fixture is run
+ */
+ if (suite->create_fixture)
+ test_context->fixture = suite->create_fixture(test_context);
+
+ /* Only run the test if the fixture was created (if required) */
+ if ((suite->create_fixture && test_context->fixture) ||
+ (!suite->create_fixture)) {
+ /* Run this fixture */
+ test_func->execute(test_context);
+
+ if (suite->remove_fixture)
+ suite->remove_fixture(test_context);
+
+ kutf_add_explicit_result(test_context);
+ }
+ return 0;
+}
+
+/**
+ * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry.
+ * @file: Opened file to read from
+ * @buf: User buffer to write the data into
+ * @len: Amount of data to read
+ * @ppos: Offset into file to read from
+ *
+ * This function emits the results which where logged during the opening of
+ * the file kutf_debugfs_run_open.
+ * Results will be emitted one at a time, once all the results have been read
+ * 0 will be returned to indicate there is no more data.
+ *
+ * Return: Number of bytes read.
+ */
+static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf,
+ size_t len, loff_t *ppos)
+{
+ struct kutf_context *test_context = file->private_data;
+ struct kutf_result *res;
+ unsigned long bytes_not_copied;
+ ssize_t bytes_copied = 0;
+
+ /* Note: This code assumes a result is read completely */
+ res = kutf_remove_result(test_context->result_set);
+ if (res) {
+ char *kutf_str_ptr = NULL;
+ unsigned int kutf_str_len = 0;
+ unsigned int message_len = 0;
+ char separator = ':';
+ char terminator = '\n';
+
+ kutf_result_to_string(&kutf_str_ptr, res->status);
+ if (kutf_str_ptr)
+ kutf_str_len = strlen(kutf_str_ptr);
+
+ if (res->message)
+ message_len = strlen(res->message);
+
+ if ((kutf_str_len + 1 + message_len + 1) > len) {
+ pr_err("Not enough space in user buffer for a single result");
+ return 0;
+ }
+
+ /* First copy the result string */
+ if (kutf_str_ptr) {
+ bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr,
+ kutf_str_len);
+ bytes_copied += kutf_str_len - bytes_not_copied;
+ if (bytes_not_copied)
+ goto exit;
+ }
+
+ /* Then the separator */
+ bytes_not_copied = copy_to_user(&buf[bytes_copied],
+ &separator, 1);
+ bytes_copied += 1 - bytes_not_copied;
+ if (bytes_not_copied)
+ goto exit;
+
+ /* Finally Next copy the result string */
+ if (res->message) {
+ bytes_not_copied = copy_to_user(&buf[bytes_copied],
+ res->message, message_len);
+ bytes_copied += message_len - bytes_not_copied;
+ if (bytes_not_copied)
+ goto exit;
+ }
+
+ /* Finally the terminator */
+ bytes_not_copied = copy_to_user(&buf[bytes_copied],
+ &terminator, 1);
+ bytes_copied += 1 - bytes_not_copied;
+ }
+exit:
+ return bytes_copied;
+}
+
+/**
+ * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry.
+ * @inode: File entry representation
+ * @file: A specific opening of the file
+ *
+ * Release any resources that where created during the opening of the file
+ *
+ * Return: 0 on success
+ */
+static int kutf_debugfs_run_release(struct inode *inode, struct file *file)
+{
+ struct kutf_context *test_context = file->private_data;
+
+ kutf_destroy_context(test_context);
+ return 0;
+}
+
+static const struct file_operations kutf_debugfs_run_ops = {
+ .owner = THIS_MODULE,
+ .open = kutf_debugfs_run_open,
+ .read = kutf_debugfs_run_read,
+ .release = kutf_debugfs_run_release,
+ .llseek = default_llseek,
+};
+
+/**
+ * create_fixture_variant() - Creates a fixture variant for the specified
+ * test function and index and the debugfs entries
+ * that represent it.
+ * @test_func: Test function
+ * @fixture_index: Fixture index
+ *
+ * Return: 0 on success, negative value corresponding to error code in failure
+ */
+static int create_fixture_variant(struct kutf_test_function *test_func,
+ unsigned int fixture_index)
+{
+ struct kutf_test_fixture *test_fix;
+ char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */
+ struct dentry *tmp;
+ int err;
+
+ test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL);
+ if (!test_fix) {
+ pr_err("Failed to create debugfs directory when adding fixture\n");
+ err = -ENOMEM;
+ goto fail_alloc;
+ }
+
+ test_fix->test_func = test_func;
+ test_fix->fixture_index = fixture_index;
+
+ snprintf(name, sizeof(name), "%d", fixture_index);
+ test_fix->dir = debugfs_create_dir(name, test_func->dir);
+ if (!test_func->dir) {
+ pr_err("Failed to create debugfs directory when adding fixture\n");
+ /* Might not be the right error, we don't get it passed back to us */
+ err = -EEXIST;
+ goto fail_dir;
+ }
+
+ tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n",
+ &kutf_debugfs_const_string_ops);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"type\" when adding fixture\n");
+ /* Might not be the right error, we don't get it passed back to us */
+ err = -EEXIST;
+ goto fail_file;
+ }
+
+ tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix,
+ &kutf_debugfs_run_ops);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"run\" when adding fixture\n");
+ /* Might not be the right error, we don't get it passed back to us */
+ err = -EEXIST;
+ goto fail_file;
+ }
+
+ list_add(&test_fix->node, &test_func->variant_list);
+ return 0;
+
+fail_file:
+ debugfs_remove_recursive(test_fix->dir);
+fail_dir:
+ kfree(test_fix);
+fail_alloc:
+ return err;
+}
+
+/**
+ * kutf_remove_test_variant() - Destroy a previously created fixture variant.
+ * @test_fix: Test fixture
+ */
+static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix)
+{
+ debugfs_remove_recursive(test_fix->dir);
+ kfree(test_fix);
+}
+
+void kutf_add_test_with_filters_and_data(
+ struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context),
+ unsigned int filters,
+ union kutf_callback_data test_data)
+{
+ struct kutf_test_function *test_func;
+ struct dentry *tmp;
+ unsigned int i;
+
+ test_func = kmalloc(sizeof(*test_func), GFP_KERNEL);
+ if (!test_func) {
+ pr_err("Failed to allocate memory when adding test %s\n", name);
+ goto fail_alloc;
+ }
+
+ INIT_LIST_HEAD(&test_func->variant_list);
+
+ test_func->dir = debugfs_create_dir(name, suite->dir);
+ if (!test_func->dir) {
+ pr_err("Failed to create debugfs directory when adding test %s\n", name);
+ goto fail_dir;
+ }
+
+ tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n",
+ &kutf_debugfs_const_string_ops);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name);
+ goto fail_file;
+ }
+
+ test_func->filters = filters;
+ tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir,
+ &test_func->filters);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name);
+ goto fail_file;
+ }
+
+ test_func->test_id = id;
+ tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir,
+ &test_func->test_id);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name);
+ goto fail_file;
+ }
+
+ for (i = 0; i < suite->fixture_variants; i++) {
+ if (create_fixture_variant(test_func, i)) {
+ pr_err("Failed to create fixture %d when adding test %s\n", i, name);
+ goto fail_file;
+ }
+ }
+
+ test_func->suite = suite;
+ test_func->execute = execute;
+ test_func->test_data = test_data;
+
+ list_add(&test_func->node, &suite->test_list);
+ return;
+
+fail_file:
+ debugfs_remove_recursive(test_func->dir);
+fail_dir:
+ kfree(test_func);
+fail_alloc:
+ return;
+}
+EXPORT_SYMBOL(kutf_add_test_with_filters_and_data);
+
+void kutf_add_test_with_filters(
+ struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context),
+ unsigned int filters)
+{
+ union kutf_callback_data data;
+
+ data.ptr_value = NULL;
+
+ kutf_add_test_with_filters_and_data(suite,
+ id,
+ name,
+ execute,
+ suite->suite_default_flags,
+ data);
+}
+EXPORT_SYMBOL(kutf_add_test_with_filters);
+
+void kutf_add_test(struct kutf_suite *suite,
+ unsigned int id,
+ const char *name,
+ void (*execute)(struct kutf_context *context))
+{
+ union kutf_callback_data data;
+
+ data.ptr_value = NULL;
+
+ kutf_add_test_with_filters_and_data(suite,
+ id,
+ name,
+ execute,
+ suite->suite_default_flags,
+ data);
+}
+EXPORT_SYMBOL(kutf_add_test);
+
+/**
+ * kutf_remove_test(): Remove a previously added test function.
+ * @test_func: Test function
+ */
+static void kutf_remove_test(struct kutf_test_function *test_func)
+{
+ struct list_head *pos;
+ struct list_head *tmp;
+
+ list_for_each_safe(pos, tmp, &test_func->variant_list) {
+ struct kutf_test_fixture *test_fix;
+
+ test_fix = list_entry(pos, struct kutf_test_fixture, node);
+ kutf_remove_test_variant(test_fix);
+ }
+
+ list_del(&test_func->node);
+ debugfs_remove_recursive(test_func->dir);
+ kfree(test_func);
+}
+
+struct kutf_suite *kutf_create_suite_with_filters_and_data(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context),
+ unsigned int filters,
+ union kutf_callback_data suite_data)
+{
+ struct kutf_suite *suite;
+ struct dentry *tmp;
+
+ suite = kmalloc(sizeof(*suite), GFP_KERNEL);
+ if (!suite) {
+ pr_err("Failed to allocate memory when creating suite %s\n", name);
+ goto fail_kmalloc;
+ }
+
+ suite->dir = debugfs_create_dir(name, app->dir);
+ if (!suite->dir) {
+ pr_err("Failed to create debugfs directory when adding test %s\n", name);
+ goto fail_debugfs;
+ }
+
+ tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n",
+ &kutf_debugfs_const_string_ops);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name);
+ goto fail_file;
+ }
+
+ INIT_LIST_HEAD(&suite->test_list);
+ suite->app = app;
+ suite->name = name;
+ suite->fixture_variants = fixture_count;
+ suite->create_fixture = create_fixture;
+ suite->remove_fixture = remove_fixture;
+ suite->suite_default_flags = filters;
+ suite->suite_data = suite_data;
+
+ list_add(&suite->node, &app->suite_list);
+
+ return suite;
+
+fail_file:
+ debugfs_remove_recursive(suite->dir);
+fail_debugfs:
+ kfree(suite);
+fail_kmalloc:
+ return NULL;
+}
+EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data);
+
+struct kutf_suite *kutf_create_suite_with_filters(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context),
+ unsigned int filters)
+{
+ union kutf_callback_data data;
+
+ data.ptr_value = NULL;
+ return kutf_create_suite_with_filters_and_data(app,
+ name,
+ fixture_count,
+ create_fixture,
+ remove_fixture,
+ filters,
+ data);
+}
+EXPORT_SYMBOL(kutf_create_suite_with_filters);
+
+struct kutf_suite *kutf_create_suite(
+ struct kutf_application *app,
+ const char *name,
+ unsigned int fixture_count,
+ void *(*create_fixture)(struct kutf_context *context),
+ void (*remove_fixture)(struct kutf_context *context))
+{
+ union kutf_callback_data data;
+
+ data.ptr_value = NULL;
+ return kutf_create_suite_with_filters_and_data(app,
+ name,
+ fixture_count,
+ create_fixture,
+ remove_fixture,
+ KUTF_F_TEST_GENERIC,
+ data);
+}
+EXPORT_SYMBOL(kutf_create_suite);
+
+/**
+ * kutf_destroy_suite() - Destroy a previously added test suite.
+ * @suite: Test suite
+ */
+static void kutf_destroy_suite(struct kutf_suite *suite)
+{
+ struct list_head *pos;
+ struct list_head *tmp;
+
+ list_for_each_safe(pos, tmp, &suite->test_list) {
+ struct kutf_test_function *test_func;
+
+ test_func = list_entry(pos, struct kutf_test_function, node);
+ kutf_remove_test(test_func);
+ }
+
+ list_del(&suite->node);
+ debugfs_remove_recursive(suite->dir);
+ kfree(suite);
+}
+
+struct kutf_application *kutf_create_application(const char *name)
+{
+ struct kutf_application *app;
+ struct dentry *tmp;
+
+ app = kmalloc(sizeof(*app), GFP_KERNEL);
+ if (!app) {
+ pr_err("Failed to create allocate memory when creating application %s\n", name);
+ goto fail_kmalloc;
+ }
+
+ app->dir = debugfs_create_dir(name, base_dir);
+ if (!app->dir) {
+ pr_err("Failed to create debugfs direcotry when creating application %s\n", name);
+ goto fail_debugfs;
+ }
+
+ tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n",
+ &kutf_debugfs_const_string_ops);
+ if (!tmp) {
+ pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name);
+ goto fail_file;
+ }
+
+ INIT_LIST_HEAD(&app->suite_list);
+ app->name = name;
+
+ return app;
+
+fail_file:
+ debugfs_remove_recursive(app->dir);
+fail_debugfs:
+ kfree(app);
+fail_kmalloc:
+ return NULL;
+}
+EXPORT_SYMBOL(kutf_create_application);
+
+void kutf_destroy_application(struct kutf_application *app)
+{
+ struct list_head *pos;
+ struct list_head *tmp;
+
+ list_for_each_safe(pos, tmp, &app->suite_list) {
+ struct kutf_suite *suite;
+
+ suite = list_entry(pos, struct kutf_suite, node);
+ kutf_destroy_suite(suite);
+ }
+
+ debugfs_remove_recursive(app->dir);
+ kfree(app);
+}
+EXPORT_SYMBOL(kutf_destroy_application);
+
+static struct kutf_context *kutf_create_context(
+ struct kutf_test_fixture *test_fix)
+{
+ struct kutf_context *new_context;
+
+ new_context = kmalloc(sizeof(*new_context), GFP_KERNEL);
+ if (!new_context) {
+ pr_err("Failed to allocate test context");
+ goto fail_alloc;
+ }
+
+ new_context->result_set = kutf_create_result_set();
+ if (!new_context->result_set) {
+ pr_err("Failed to create resultset");
+ goto fail_result_set;
+ }
+
+ new_context->test_fix = test_fix;
+ /* Save the pointer to the suite as the callbacks will require it */
+ new_context->suite = test_fix->test_func->suite;
+ new_context->status = KUTF_RESULT_UNKNOWN;
+ new_context->expected_status = KUTF_RESULT_UNKNOWN;
+
+ kutf_mempool_init(&new_context->fixture_pool);
+ new_context->fixture = NULL;
+ new_context->fixture_index = test_fix->fixture_index;
+ new_context->fixture_name = NULL;
+ new_context->test_data = test_fix->test_func->test_data;
+
+ return new_context;
+
+fail_result_set:
+ kfree(new_context);
+fail_alloc:
+ return NULL;
+}
+
+static void kutf_destroy_context(struct kutf_context *context)
+{
+ kutf_destroy_result_set(context->result_set);
+ kutf_mempool_destroy(&context->fixture_pool);
+ kfree(context);
+}
+
+static void kutf_set_result(struct kutf_context *context,
+ enum kutf_result_status status)
+{
+ context->status = status;
+}
+
+static void kutf_set_expected_result(struct kutf_context *context,
+ enum kutf_result_status expected_status)
+{
+ context->expected_status = expected_status;
+}
+
+/**
+ * kutf_test_log_result() - Log a result for the specified test context
+ * @context: Test context
+ * @message: Result string
+ * @new_status: Result status
+ */
+static void kutf_test_log_result(
+ struct kutf_context *context,
+ const char *message,
+ enum kutf_result_status new_status)
+{
+ if (context->status < new_status)
+ context->status = new_status;
+
+ if (context->expected_status != new_status)
+ kutf_add_result(&context->fixture_pool, context->result_set,
+ new_status, message);
+}
+
+void kutf_test_log_result_external(
+ struct kutf_context *context,
+ const char *message,
+ enum kutf_result_status new_status)
+{
+ kutf_test_log_result(context, message, new_status);
+}
+EXPORT_SYMBOL(kutf_test_log_result_external);
+
+void kutf_test_expect_abort(struct kutf_context *context)
+{
+ kutf_set_expected_result(context, KUTF_RESULT_ABORT);
+}
+EXPORT_SYMBOL(kutf_test_expect_abort);
+
+void kutf_test_expect_fatal(struct kutf_context *context)
+{
+ kutf_set_expected_result(context, KUTF_RESULT_FATAL);
+}
+EXPORT_SYMBOL(kutf_test_expect_fatal);
+
+void kutf_test_expect_fail(struct kutf_context *context)
+{
+ kutf_set_expected_result(context, KUTF_RESULT_FAIL);
+}
+EXPORT_SYMBOL(kutf_test_expect_fail);
+
+void kutf_test_expect_warn(struct kutf_context *context)
+{
+ kutf_set_expected_result(context, KUTF_RESULT_WARN);
+}
+EXPORT_SYMBOL(kutf_test_expect_warn);
+
+void kutf_test_expect_pass(struct kutf_context *context)
+{
+ kutf_set_expected_result(context, KUTF_RESULT_PASS);
+}
+EXPORT_SYMBOL(kutf_test_expect_pass);
+
+void kutf_test_skip(struct kutf_context *context)
+{
+ kutf_set_result(context, KUTF_RESULT_SKIP);
+ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN);
+
+ kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP);
+}
+EXPORT_SYMBOL(kutf_test_skip);
+
+void kutf_test_skip_msg(struct kutf_context *context, const char *message)
+{
+ kutf_set_result(context, KUTF_RESULT_SKIP);
+ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN);
+
+ kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool,
+ "Test skipped: %s", message), KUTF_RESULT_SKIP);
+ kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP);
+}
+EXPORT_SYMBOL(kutf_test_skip_msg);
+
+void kutf_test_debug(struct kutf_context *context, char const *message)
+{
+ kutf_test_log_result(context, message, KUTF_RESULT_DEBUG);
+}
+EXPORT_SYMBOL(kutf_test_debug);
+
+void kutf_test_pass(struct kutf_context *context, char const *message)
+{
+ static const char explicit_message[] = "(explicit pass)";
+
+ if (!message)
+ message = explicit_message;
+
+ kutf_test_log_result(context, message, KUTF_RESULT_PASS);
+}
+EXPORT_SYMBOL(kutf_test_pass);
+
+void kutf_test_info(struct kutf_context *context, char const *message)
+{
+ kutf_test_log_result(context, message, KUTF_RESULT_INFO);
+}
+EXPORT_SYMBOL(kutf_test_info);
+
+void kutf_test_warn(struct kutf_context *context, char const *message)
+{
+ kutf_test_log_result(context, message, KUTF_RESULT_WARN);
+}
+EXPORT_SYMBOL(kutf_test_warn);
+
+void kutf_test_fail(struct kutf_context *context, char const *message)
+{
+ kutf_test_log_result(context, message, KUTF_RESULT_FAIL);
+}
+EXPORT_SYMBOL(kutf_test_fail);
+
+void kutf_test_fatal(struct kutf_context *context, char const *message)
+{
+ kutf_test_log_result(context, message, KUTF_RESULT_FATAL);
+}
+EXPORT_SYMBOL(kutf_test_fatal);
+
+void kutf_test_abort(struct kutf_context *context)
+{
+ kutf_test_log_result(context, "", KUTF_RESULT_ABORT);
+}
+EXPORT_SYMBOL(kutf_test_abort);
+
+/**
+ * init_kutf_core() - Module entry point.
+ *
+ * Create the base entry point in debugfs.
+ */
+static int __init init_kutf_core(void)
+{
+ int ret;
+
+ base_dir = debugfs_create_dir("kutf_tests", NULL);
+ if (!base_dir) {
+ ret = -ENODEV;
+ goto exit_dir;
+ }
+
+ return 0;
+
+exit_dir:
+ return ret;
+}
+
+/**
+ * exit_kutf_core() - Module exit point.
+ *
+ * Remove the base entry point in debugfs.
+ */
+static void __exit exit_kutf_core(void)
+{
+ debugfs_remove_recursive(base_dir);
+}
+
+#else /* defined(CONFIG_DEBUG_FS) */
+
+/**
+ * init_kutf_core() - Module entry point.
+ *
+ * Stub for when build against a kernel without debugfs support
+ */
+static int __init init_kutf_core(void)
+{
+ pr_debug("KUTF requires a kernel with debug fs support");
+
+ return -ENODEV;
+}
+
+/**
+ * exit_kutf_core() - Module exit point.
+ *
+ * Stub for when build against a kernel without debugfs support
+ */
+static void __exit exit_kutf_core(void)
+{
+}
+#endif /* defined(CONFIG_DEBUG_FS) */
+
+MODULE_LICENSE("GPL");
+
+module_init(init_kutf_core);
+module_exit(exit_kutf_core);
diff --git a/mali_kbase/tests/kutf/kutf_utils.c b/mali_kbase/tests/kutf/kutf_utils.c
new file mode 100644
index 0000000..a429a2d
--- /dev/null
+++ b/mali_kbase/tests/kutf/kutf_utils.c
@@ -0,0 +1,71 @@
+/*
+ *
+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/* Kernel UTF utility functions */
+
+#include <linux/mutex.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+
+#include <kutf/kutf_utils.h>
+#include <kutf/kutf_mem.h>
+
+static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN];
+
+DEFINE_MUTEX(buffer_lock);
+
+const char *kutf_dsprintf(struct kutf_mempool *pool,
+ const char *fmt, ...)
+{
+ va_list args;
+ int len;
+ int size;
+ void *buffer;
+
+ mutex_lock(&buffer_lock);
+ va_start(args, fmt);
+ len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args);
+ va_end(args);
+
+ if (len < 0) {
+ pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt);
+ goto fail_format;
+ }
+
+ if (len >= sizeof(tmp_buffer)) {
+ pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt);
+ size = sizeof(tmp_buffer);
+ } else {
+ size = len + 1;
+ }
+
+ buffer = kutf_mempool_alloc(pool, size);
+ if (!buffer)
+ goto fail_alloc;
+
+ memcpy(buffer, tmp_buffer, size);
+ mutex_unlock(&buffer_lock);
+
+ return buffer;
+
+fail_alloc:
+fail_format:
+ mutex_unlock(&buffer_lock);
+ return NULL;
+}
+EXPORT_SYMBOL(kutf_dsprintf);
diff --git a/mali_kbase/tests/kutf/sconscript b/mali_kbase/tests/kutf/sconscript
new file mode 100644
index 0000000..d7f1124
--- /dev/null
+++ b/mali_kbase/tests/kutf/sconscript
@@ -0,0 +1,21 @@
+#
+# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+Import('kutf_env')
+
+make_args = kutf_env.kernel_get_config_defines(ret_list = True)
+
+mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args)
+kutf_env.KernelObjTarget('kutf', mod)
diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kbuild b/mali_kbase/tests/mali_kutf_irq_test/Kbuild
new file mode 100644
index 0000000..0cd9ceb
--- /dev/null
+++ b/mali_kbase/tests/mali_kutf_irq_test/Kbuild
@@ -0,0 +1,20 @@
+#
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android
+
+obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o
+
+mali_kutf_irq_test-y := mali_kutf_irq_test_main.o
diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kconfig b/mali_kbase/tests/mali_kutf_irq_test/Kconfig
new file mode 100644
index 0000000..f4553d3
--- /dev/null
+++ b/mali_kbase/tests/mali_kutf_irq_test/Kconfig
@@ -0,0 +1,23 @@
+#
+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+config MALI_IRQ_LATENCY
+ tristate "Mali GPU IRQ latency measurement"
+ depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF
+ default m
+ help
+ This option will build a test module mali_kutf_irq_test that
+ can determine the latency of the Mali GPU IRQ on your system.
+ Choosing M here will generate a single module called mali_kutf_irq_test.
diff --git a/mali_kbase/tests/mali_kutf_irq_test/Makefile b/mali_kbase/tests/mali_kutf_irq_test/Makefile
new file mode 100644
index 0000000..4e94876
--- /dev/null
+++ b/mali_kbase/tests/mali_kutf_irq_test/Makefile
@@ -0,0 +1,51 @@
+#
+# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+# linux build system bootstrap for out-of-tree module
+
+# default to building for the host
+ARCH ?= $(shell uname -m)
+
+ifeq ($(KDIR),)
+$(error Must specify KDIR to point to the kernel to target))
+endif
+
+TEST_CCFLAGS := \
+ -DMALI_DEBUG=$(MALI_DEBUG) \
+ -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \
+ -DMALI_MODEL=$(MALI_MODEL) \
+ -DMALI_NO_MALI=$(MALI_NO_MALI) \
+ -DMALI_BASE_QA_LEAK=$(MALI_BASE_QA_LEAK) \
+ -DMALI_BASE_QA_RESFAIL=$(MALI_BASE_QA_RESFAIL) \
+ -DMALI_BASE_QA_USE_AFTER_FREE=$(MALI_BASE_QA_USE_AFTER_FREE) \
+ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \
+ -DMALI_USE_UMP=$(MALI_USE_UMP) \
+ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \
+ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \
+ $(SCONS_CFLAGS) \
+ -I$(CURDIR)/../include \
+ -I$(CURDIR)/../../../../../../include \
+ -I$(CURDIR)/../../../ \
+ -I$(CURDIR)/../../ \
+ -I$(CURDIR)/../../backend/gpu \
+ -I$(CURDIR)/ \
+ -I$(srctree)/drivers/staging/android \
+ -I$(srctree)/include/linux
+
+all:
+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules
+
+clean:
+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
diff --git a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c
new file mode 100644
index 0000000..ffd802f
--- /dev/null
+++ b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c
@@ -0,0 +1,257 @@
+/*
+ *
+ * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+
+#include "mali_kbase.h"
+#include <midgard/backend/gpu/mali_kbase_device_internal.h>
+
+#include <kutf/kutf_suite.h>
+#include <kutf/kutf_utils.h>
+
+/*
+ * This file contains the code which is used for measuring interrupt latency
+ * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is
+ * used with this purpose and it is called within KUTF framework - a kernel
+ * unit test framework. The measured latency provided by this test should
+ * be representative for the latency of the Mali JOB/MMU IRQs as well.
+ */
+
+/* KUTF test application pointer for this test */
+struct kutf_application *irq_app;
+
+/**
+ * struct kutf_irq_fixture data - test fixture used by the test functions.
+ * @kbdev: kbase device for the GPU.
+ *
+ */
+struct kutf_irq_fixture_data {
+ struct kbase_device *kbdev;
+};
+
+#define SEC_TO_NANO(s) ((s)*1000000000LL)
+
+/* ID for the GPU IRQ */
+#define GPU_IRQ_HANDLER 2
+
+#define NR_TEST_IRQS 1000000
+
+/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not
+ * expect to see this in normal use (e.g., when Android is running). */
+#define TEST_IRQ MULTIPLE_GPU_FAULTS
+
+#define IRQ_TIMEOUT HZ
+
+/* Kernel API for setting irq throttle hook callback and irq time in us*/
+extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev,
+ irq_handler_t custom_handler,
+ int irq_type);
+extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data);
+
+static DECLARE_WAIT_QUEUE_HEAD(wait);
+static bool triggered;
+static u64 irq_time;
+
+static void *kbase_untag(void *ptr)
+{
+ return (void *)(((uintptr_t) ptr) & ~3);
+}
+
+/**
+ * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler
+ * @irq: IRQ number
+ * @data: Data associated with this IRQ
+ *
+ * Return: state of the IRQ
+ */
+static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data)
+{
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL);
+ if (val & TEST_IRQ) {
+ struct timespec tval;
+
+ getnstimeofday(&tval);
+ irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val,
+ NULL);
+
+ triggered = true;
+ wake_up(&wait);
+
+ return IRQ_HANDLED;
+ }
+
+ /* Trigger main irq handler */
+ return kbase_gpu_irq_handler(irq, data);
+}
+
+/**
+ * mali_kutf_irq_default_create_fixture() - Creates the fixture data required
+ * for all the tests in the irq suite.
+ * @context: KUTF context.
+ *
+ * Return: Fixture data created on success or NULL on failure
+ */
+static void *mali_kutf_irq_default_create_fixture(
+ struct kutf_context *context)
+{
+ struct kutf_irq_fixture_data *data;
+
+ data = kutf_mempool_alloc(&context->fixture_pool,
+ sizeof(struct kutf_irq_fixture_data));
+
+ if (!data)
+ goto fail;
+
+ /* Acquire the kbase device */
+ data->kbdev = kbase_find_device(-1);
+ if (data->kbdev == NULL) {
+ kutf_test_fail(context, "Failed to find kbase device");
+ goto fail;
+ }
+
+ return data;
+
+fail:
+ return NULL;
+}
+
+/**
+ * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously
+ * created by mali_kutf_irq_default_create_fixture.
+ *
+ * @context: KUTF context.
+ */
+static void mali_kutf_irq_default_remove_fixture(
+ struct kutf_context *context)
+{
+ struct kutf_irq_fixture_data *data = context->fixture;
+ struct kbase_device *kbdev = data->kbdev;
+
+ kbase_release_device(kbdev);
+}
+
+/**
+ * mali_kutf_irq_latency() - measure GPU IRQ latency
+ * @context: kutf context within which to perform the test
+ *
+ * The test triggers IRQs manually, and measures the
+ * time between triggering the IRQ and the IRQ handler being executed.
+ *
+ * This is not a traditional test, in that the pass/fail status has little
+ * meaning (other than indicating that the IRQ handler executed at all). Instead
+ * the results are in the latencies provided with the test result. There is no
+ * meaningful pass/fail result that can be obtained here, instead the latencies
+ * are provided for manual analysis only.
+ */
+static void mali_kutf_irq_latency(struct kutf_context *context)
+{
+ struct kutf_irq_fixture_data *data = context->fixture;
+ struct kbase_device *kbdev = data->kbdev;
+ u64 min_time = U64_MAX, max_time = 0, average_time = 0;
+ int i;
+ bool test_failed = false;
+
+ /* Force GPU to be powered */
+ kbase_pm_context_active(kbdev);
+
+ kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler,
+ GPU_IRQ_HANDLER);
+
+ for (i = 0; i < NR_TEST_IRQS; i++) {
+ struct timespec tval;
+ u64 start_time;
+ int ret;
+
+ triggered = false;
+ getnstimeofday(&tval);
+ start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec);
+
+ /* Trigger fake IRQ */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
+ TEST_IRQ, NULL);
+
+ ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT);
+
+ if (ret == 0) {
+ kutf_test_fail(context, "Timed out waiting for IRQ\n");
+ test_failed = true;
+ break;
+ }
+
+ if ((irq_time - start_time) < min_time)
+ min_time = irq_time - start_time;
+ if ((irq_time - start_time) > max_time)
+ max_time = irq_time - start_time;
+ average_time += irq_time - start_time;
+
+ udelay(10);
+ }
+
+ /* Go back to default handler */
+ kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER);
+
+ kbase_pm_context_idle(kbdev);
+
+ if (!test_failed) {
+ const char *results;
+
+ do_div(average_time, NR_TEST_IRQS);
+ results = kutf_dsprintf(&context->fixture_pool,
+ "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n",
+ min_time, max_time, average_time);
+ kutf_test_pass(context, results);
+ }
+}
+
+/**
+ * Module entry point for this test.
+ */
+int mali_kutf_irq_test_main_init(void)
+{
+ struct kutf_suite *suite;
+
+ irq_app = kutf_create_application("irq");
+ suite = kutf_create_suite(irq_app, "irq_default",
+ 1, mali_kutf_irq_default_create_fixture,
+ mali_kutf_irq_default_remove_fixture);
+
+ kutf_add_test(suite, 0x0, "irq_latency",
+ mali_kutf_irq_latency);
+ return 0;
+}
+
+/**
+ * Module exit point for this test.
+ */
+void mali_kutf_irq_test_main_exit(void)
+{
+ kutf_destroy_application(irq_app);
+}
+
+module_init(mali_kutf_irq_test_main_init);
+module_exit(mali_kutf_irq_test_main_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("ARM Ltd.");
+MODULE_VERSION("1.0");
diff --git a/mali_kbase/tests/mali_kutf_irq_test/sconscript b/mali_kbase/tests/mali_kutf_irq_test/sconscript
new file mode 100644
index 0000000..ec837f1
--- /dev/null
+++ b/mali_kbase/tests/mali_kutf_irq_test/sconscript
@@ -0,0 +1,30 @@
+#
+# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+import os
+Import('env')
+
+src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')]
+
+if env.GetOption('clean') :
+ env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test'))
+ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [])
+ env.KernelObjTarget('mali_kutf_irq_test', cmd)
+else:
+ makeAction=Action("cd ${SOURCE.dir} && make MALI_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_MODEL=${mali_model} MALI_NO_MALI=${no_mali} MALI_HW_VERSION=${hwver} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % (env.base_get_qa_settings(), env.kernel_get_config_defines()), '$MAKECOMSTR')
+ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction])
+ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko')
+ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko')
+ env.KernelObjTarget('mali_kutf_irq_test', cmd)
diff --git a/mali_kbase/tests/sconscript b/mali_kbase/tests/sconscript
new file mode 100644
index 0000000..5337e10
--- /dev/null
+++ b/mali_kbase/tests/sconscript
@@ -0,0 +1,37 @@
+#
+# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+Import ('env')
+
+kutf_env = env.Clone()
+kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include')
+Export('kutf_env')
+
+if Glob('internal/sconscript'):
+ SConscript('internal/sconscript')
+
+if kutf_env['debug'] == '1':
+ SConscript('kutf/sconscript')
+ SConscript('mali_kutf_irq_test/sconscript')
+
+ if Glob('kutf_test/sconscript'):
+ SConscript('kutf_test/sconscript')
+
+ if Glob('kutf_test_runner/sconscript'):
+ SConscript('kutf_test_runner/sconscript')
+
+if env['unit'] == '1':
+ SConscript('mali_kutf_ipa_test/sconscript')
+ SConscript('mali_kutf_vinstr_test/sconscript')