diff options
| author | Sidath Senanayake <sidaths@google.com> | 2021-01-29 15:03:53 +0000 |
|---|---|---|
| committer | Sidath Senanayake <sidaths@google.com> | 2021-01-29 15:03:53 +0000 |
| commit | 9748305a584b9f1f7705303ce6e33a5e8b923e60 (patch) | |
| tree | a73788e1d912a3202db3a99018002e0858e9a948 /mali_kbase | |
| parent | 201c8bfb4637601363b6e9283f3bdc510711a226 (diff) | |
| download | gpu-9748305a584b9f1f7705303ce6e33a5e8b923e60.tar.gz | |
Mali Valhall DDK r29p0 KMD
Provenance:
afaca8da1 (collaborate/EAC/v_r29p0)
VX504X08X-BU-00000-r29p0-01eac0 - Valhall Android DDK
VX504X08X-BU-60000-r29p0-01eac0 - Valhall Android Document Bundle
VX504X08X-DC-11001-r29p0-01eac0 - Valhall Android DDK Software Errata
VX504X08X-SW-99006-r29p0-01eac0 - Valhall Android Renderscript AOSP parts
Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: Ie0904c9223b7ec9311b848a52d3159ac2b07530e
Diffstat (limited to 'mali_kbase')
340 files changed, 13346 insertions, 2864 deletions
diff --git a/mali_kbase/Kbuild b/mali_kbase/Kbuild index 5b495c9..c8cef84 100644 --- a/mali_kbase/Kbuild +++ b/mali_kbase/Kbuild @@ -21,7 +21,7 @@ # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= "r28p0-01eac0" +MALI_RELEASE_NAME ?= "r29p0-01eac0" # Paths required for build @@ -91,7 +91,6 @@ SRC := \ mali_kbase_config.c \ mali_kbase_vinstr.c \ mali_kbase_hwcnt.c \ - mali_kbase_hwcnt_backend_jm.c \ mali_kbase_hwcnt_gpu.c \ mali_kbase_hwcnt_legacy.c \ mali_kbase_hwcnt_types.c \ @@ -128,6 +127,8 @@ SRC := \ ifeq ($(MALI_USE_CSF),1) SRC += \ + mali_kbase_hwcnt_backend_csf.c \ + mali_kbase_hwcnt_backend_csf_if_fw.c \ debug/backend/mali_kbase_debug_ktrace_csf.c \ device/backend/mali_kbase_device_csf.c \ device/backend/mali_kbase_device_hw_csf.c \ @@ -137,6 +138,7 @@ ifeq ($(MALI_USE_CSF),1) context/backend/mali_kbase_context_csf.c else SRC += \ + mali_kbase_hwcnt_backend_jm.c \ mali_kbase_dummy_job_wa.c \ mali_kbase_debug_job_fault.c \ mali_kbase_event.c \ diff --git a/mali_kbase/Makefile b/mali_kbase/Makefile index 53a1209..45fb434 100644 --- a/mali_kbase/Makefile +++ b/mali_kbase/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/Makefile.kbase b/mali_kbase/Makefile.kbase index 6b0f81e..74a9763 100644 --- a/mali_kbase/Makefile.kbase +++ b/mali_kbase/Makefile.kbase @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010, 2013, 2018 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010, 2013, 2018-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,4 +20,3 @@ # EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(KBASE_PATH)/platform_$(PLATFORM) - diff --git a/mali_kbase/Mconfig b/mali_kbase/Mconfig index a70c76c..d7a5569 100644 --- a/mali_kbase/Mconfig +++ b/mali_kbase/Mconfig @@ -1,17 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 # # (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software # Foundation, and any use by you of this program is subject to the terms -# of such GNU licence. +# of such GNU license. # -# A copy of the licence is included with the program, and can also be obtained -# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -# Boston, MA 02110-1301, USA. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. # # - menuconfig MALI_MIDGARD bool "Mali Midgard series support" diff --git a/mali_kbase/arbiter/mali_kbase_arbif.c b/mali_kbase/arbiter/mali_kbase_arbif.c index 3bfc62d..8e03466 100644 --- a/mali_kbase/arbiter/mali_kbase_arbif.c +++ b/mali_kbase/arbiter/mali_kbase_arbif.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/arbiter/mali_kbase_arbif.h b/mali_kbase/arbiter/mali_kbase_arbif.h index e7e9de7..ff082ef 100644 --- a/mali_kbase/arbiter/mali_kbase_arbif.h +++ b/mali_kbase/arbiter/mali_kbase_arbif.h @@ -19,7 +19,6 @@ * SPDX-License-Identifier: GPL-2.0 * *//* SPDX-License-Identifier: GPL-2.0 */ - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -38,7 +37,6 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * */ /** diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_defs.h b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h index 1f53cbf..071d5ee 100644 --- a/mali_kbase/arbiter/mali_kbase_arbiter_defs.h +++ b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h @@ -19,7 +19,6 @@ * SPDX-License-Identifier: GPL-2.0 * *//* SPDX-License-Identifier: GPL-2.0 */ - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -38,7 +37,6 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * */ /** @@ -66,7 +64,6 @@ * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU * @vm_arb_starting: Work queue resume in progress * @vm_arb_stopping: Work queue suspend in progress - * @vm_arb_users_waiting: Count of users waiting for GPU */ struct kbase_arbiter_vm_state { struct kbase_device *kbdev; @@ -78,7 +75,6 @@ struct kbase_arbiter_vm_state { struct work_struct vm_resume_work; bool vm_arb_starting; bool vm_arb_stopping; - int vm_arb_users_waiting; }; /** diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_interface.h b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h index 5d5d8a7..59a51d9 100644 --- a/mali_kbase/arbiter/mali_kbase_arbiter_interface.h +++ b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h @@ -19,7 +19,6 @@ * SPDX-License-Identifier: GPL-2.0 * *//* SPDX-License-Identifier: GPL-2.0 */ - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -38,7 +37,6 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * */ /** diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.c b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c index 1fc432b..f3893b6 100644 --- a/mali_kbase/arbiter/mali_kbase_arbiter_pm.c +++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -29,10 +28,7 @@ #include <mali_kbase.h> #include <mali_kbase_pm.h> -#include <mali_kbase_hwaccess_jm.h> #include <mali_kbase_irq_internal.h> -#include <mali_kbase_hwcnt_context.h> -#include <mali_kbase_pm_internal.h> #include <tl/mali_kbase_tracepoints.h> static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); @@ -131,6 +127,9 @@ static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, lockdep_assert_held(&arb_vm_state->vm_state_lock); arb_vm_state->vm_state = new_state; + if (new_state != KBASE_VM_STATE_INITIALIZING_WITH_GPU && + new_state != KBASE_VM_STATE_INITIALIZING) + KBASE_KTRACE_ADD(kbdev, ARB_VM_STATE, NULL, new_state); wake_up(&arb_vm_state->vm_state_wait); } @@ -193,6 +192,7 @@ static void kbase_arbiter_pm_resume_wq(struct work_struct *data) } arb_vm_state->vm_arb_starting = false; mutex_unlock(&arb_vm_state->vm_state_lock); + KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); dev_dbg(kbdev->dev, "<%s\n", __func__); } @@ -231,7 +231,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq); INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq); arb_vm_state->vm_arb_starting = false; - arb_vm_state->vm_arb_users_waiting = 0; + atomic_set(&kbdev->pm.gpu_users_waiting, 0); kbdev->pm.arb_vm_state = arb_vm_state; err = kbase_arbif_init(kbdev); @@ -312,7 +312,7 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) lockdep_assert_held(&arb_vm_state->vm_state_lock); - if (arb_vm_state->vm_arb_users_waiting > 0 && + if (atomic_read(&kbdev->pm.gpu_users_waiting) > 0 && arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPING_ACTIVE); @@ -623,7 +623,9 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, mutex_lock(&arb_vm_state->vm_state_lock); dev_dbg(kbdev->dev, "%s %s\n", __func__, kbase_arbiter_pm_vm_event_str(evt)); - + if (arb_vm_state->vm_state != KBASE_VM_STATE_INITIALIZING_WITH_GPU && + arb_vm_state->vm_state != KBASE_VM_STATE_INITIALIZING) + KBASE_KTRACE_ADD(kbdev, ARB_VM_EVT, NULL, evt); switch (evt) { case KBASE_VM_GPU_GRANTED_EVT: kbase_arbiter_pm_vm_gpu_start(kbdev); @@ -656,8 +658,6 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, case KBASE_VM_REF_EVENT: switch (arb_vm_state->vm_state) { case KBASE_VM_STATE_STARTING: - KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); - /* FALL THROUGH */ case KBASE_VM_STATE_IDLE: kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_ACTIVE); @@ -799,7 +799,7 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, } /* Need to synchronously wait for GPU assignment */ - arb_vm_state->vm_arb_users_waiting++; + atomic_inc(&kbdev->pm.gpu_users_waiting); mutex_unlock(&arb_vm_state->vm_state_lock); mutex_unlock(&kbdev->pm.lock); mutex_unlock(&js_devdata->runpool_mutex); @@ -807,7 +807,7 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, mutex_lock(&js_devdata->runpool_mutex); mutex_lock(&kbdev->pm.lock); mutex_lock(&arb_vm_state->vm_state_lock); - arb_vm_state->vm_arb_users_waiting--; + atomic_dec(&kbdev->pm.gpu_users_waiting); } mutex_unlock(&arb_vm_state->vm_state_lock); } diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.h b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h index 3c49eb1..b029c62 100644 --- a/mali_kbase/arbiter/mali_kbase_arbiter_pm.h +++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h @@ -19,7 +19,6 @@ * SPDX-License-Identifier: GPL-2.0 * *//* SPDX-License-Identifier: GPL-2.0 */ - /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -38,8 +37,6 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * SPDX-License-Identifier: GPL-2.0 - * */ /** diff --git a/mali_kbase/backend/gpu/Kbuild b/mali_kbase/backend/gpu/Kbuild index 5dc8c01..b95e795 100644 --- a/mali_kbase/backend/gpu/Kbuild +++ b/mali_kbase/backend/gpu/Kbuild @@ -23,7 +23,6 @@ BACKEND += \ backend/gpu/mali_kbase_cache_policy_backend.c \ backend/gpu/mali_kbase_gpuprops_backend.c \ backend/gpu/mali_kbase_irq_linux.c \ - backend/gpu/mali_kbase_instr_backend.c \ backend/gpu/mali_kbase_js_backend.c \ backend/gpu/mali_kbase_pm_backend.c \ backend/gpu/mali_kbase_pm_driver.c \ @@ -40,6 +39,7 @@ ifeq ($(MALI_USE_CSF),1) # empty else BACKEND += \ + backend/gpu/mali_kbase_instr_backend.c \ backend/gpu/mali_kbase_jm_as.c \ backend/gpu/mali_kbase_debug_job_fault_backend.c \ backend/gpu/mali_kbase_jm_hw.c \ diff --git a/mali_kbase/backend/gpu/mali_kbase_backend_config.h b/mali_kbase/backend/gpu/mali_kbase_backend_config.h index 4a61f96..1e9fbcb 100644 --- a/mali_kbase/backend/gpu/mali_kbase_backend_config.h +++ b/mali_kbase/backend/gpu/mali_kbase_backend_config.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.c b/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.c index 4e07a3f..b58518a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2016, 2018, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.h b/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.h index f78ada7..1d63098 100644 --- a/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.h +++ b/mali_kbase/backend/gpu/mali_kbase_cache_policy_backend.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - #ifndef _KBASE_CACHE_POLICY_BACKEND_H_ #define _KBASE_CACHE_POLICY_BACKEND_H_ diff --git a/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.c index 187d7d6..8d82295 100644 --- a/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.c +++ b/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -127,13 +128,15 @@ int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev) unsigned int i; int ret = 0; - /* Return early if no callbacks provided for clock rate tracing */ - if (!callbacks) - return 0; - spin_lock_init(&clk_rtm->lock); INIT_LIST_HEAD(&clk_rtm->listeners); + /* Return early if no callbacks provided for clock rate tracing */ + if (!callbacks) { + WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); + return 0; + } + clk_rtm->gpu_idle = true; for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { @@ -151,10 +154,12 @@ int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev) /* Activate clock rate trace manager if at least one GPU clock was * enumerated. */ - if (i) + if (i) { WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks); - else + } else { dev_info(kbdev->dev, "No clock(s) available for rate tracing"); + WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); + } return 0; @@ -284,4 +289,3 @@ void kbase_clk_rate_trace_manager_notify_all( } } KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all); - diff --git a/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.h b/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.h index dcafb26..2e65b6a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.h +++ b/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CLK_RATE_TRACE_MGR_ diff --git a/mali_kbase/backend/gpu/mali_kbase_debug_job_fault_backend.c b/mali_kbase/backend/gpu/mali_kbase_debug_job_fault_backend.c index 3aadcb0..7a5bfb8 100644 --- a/mali_kbase/backend/gpu/mali_kbase_debug_job_fault_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_debug_job_fault_backend.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2012-2015, 2018-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/backend/gpu/mali_kbase_devfreq.c b/mali_kbase/backend/gpu/mali_kbase_devfreq.c index 2806f05..2f9015d 100644 --- a/mali_kbase/backend/gpu/mali_kbase_devfreq.c +++ b/mali_kbase/backend/gpu/mali_kbase_devfreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -32,35 +33,48 @@ #endif #include <linux/version.h> -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) #include <linux/pm_opp.h> -#else /* Linux >= 3.13 */ -/* In 3.13 the OPP include header file, types, and functions were all - * renamed. Use the old filename for the include, and define the new names to - * the old, when an old kernel is detected. - */ -#include <linux/opp.h> -#define dev_pm_opp opp -#define dev_pm_opp_get_voltage opp_get_voltage -#define dev_pm_opp_get_opp_count opp_get_opp_count -#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil -#define dev_pm_opp_find_freq_floor opp_find_freq_floor -#endif /* Linux >= 3.13 */ /** - * opp_translate - Translate nominal OPP frequency from devicetree into real - * frequency and core mask - * @kbdev: Device pointer - * @freq: Nominal frequency - * @core_mask: Pointer to u64 to store core mask to - * @freqs: Pointer to array of frequencies - * @volts: Pointer to array of voltages + * get_voltage() - Get the voltage value corresponding to the nominal frequency + * used by devfreq. + * @kbdev: Device pointer + * @freq: Nominal frequency in Hz passed by devfreq. + * + * This function will be called only when the opp table which is compatible with + * "operating-points-v2-mali", is not present in the devicetree for GPU device. * - * This function will only perform translation if an operating-points-v2-mali - * table is present in devicetree. If one is not present then it will return an - * untranslated frequency and all cores enabled. + * Return: Voltage value in milli volts, 0 in case of error. */ -static void opp_translate(struct kbase_device *kbdev, unsigned long freq, +static unsigned long get_voltage(struct kbase_device *kbdev, unsigned long freq) +{ + struct dev_pm_opp *opp; + unsigned long voltage = 0; + +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_lock(); +#endif + + opp = dev_pm_opp_find_freq_exact(kbdev->dev, freq, true); + + if (IS_ERR_OR_NULL(opp)) + dev_err(kbdev->dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); + else { + voltage = dev_pm_opp_get_voltage(opp); +#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE + dev_pm_opp_put(opp); +#endif + } + +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_unlock(); +#endif + + /* Return the voltage in milli volts */ + return voltage / 1000; +} + +void kbase_devfreq_opp_translate(struct kbase_device *kbdev, unsigned long freq, u64 *core_mask, unsigned long *freqs, unsigned long *volts) { unsigned int i; @@ -82,12 +96,17 @@ static void opp_translate(struct kbase_device *kbdev, unsigned long freq, } /* If failed to find OPP, return all cores enabled - * and nominal frequency + * and nominal frequency and the corresponding voltage. */ if (i == kbdev->num_opps) { + unsigned long voltage = get_voltage(kbdev, freq); + *core_mask = kbdev->gpu_props.props.raw_props.shader_present; - for (i = 0; i < kbdev->nr_clocks; i++) + + for (i = 0; i < kbdev->nr_clocks; i++) { freqs[i] = freq; + volts[i] = voltage; + } } } @@ -104,18 +123,18 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) nominal_freq = *target_freq; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_lock(); #endif opp = devfreq_recommended_opp(dev, &nominal_freq, flags); -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_unlock(); #endif if (IS_ERR_OR_NULL(opp)) { dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); return PTR_ERR(opp); } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE dev_pm_opp_put(opp); #endif @@ -127,7 +146,8 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) return 0; } - opp_translate(kbdev, nominal_freq, &core_mask, freqs, volts); + kbase_devfreq_opp_translate(kbdev, nominal_freq, &core_mask, + freqs, volts); #ifdef CONFIG_REGULATOR /* Regulators and clocks work in pairs: every clock has a regulator, @@ -238,6 +258,10 @@ kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) stat->current_frequency = kbdev->current_nominal_freq; stat->private_data = NULL; +#if MALI_USE_CSF && defined CONFIG_DEVFREQ_THERMAL + kbase_ipa_reset_data(kbdev); +#endif + return 0; } @@ -249,11 +273,11 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, unsigned long freq; struct dev_pm_opp *opp; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_lock(); #endif count = dev_pm_opp_get_opp_count(kbdev->dev); -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_unlock(); #endif if (count < 0) @@ -264,20 +288,20 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, if (!dp->freq_table) return -ENOMEM; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_lock(); #endif for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); if (IS_ERR(opp)) break; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE dev_pm_opp_put(opp); -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */ +#endif /* KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE */ dp->freq_table[i] = freq; } -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_unlock(); #endif @@ -359,7 +383,7 @@ static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev, static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) { -#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) +#ifndef CONFIG_OF /* OPP table initialization requires at least the capability to get * regulators and clocks from the device tree, as well as parsing * arrays of unsigned integer values. @@ -493,11 +517,9 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) kbdev->num_opps = i; return 0; -#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ +#endif /* CONFIG_OF */ } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) - static const char *kbase_devfreq_req_type_name(enum kbase_devfreq_work_type type) { const char *p; @@ -554,12 +576,9 @@ static void kbase_devfreq_suspend_resume_worker(struct work_struct *work) } } -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */ - void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, enum kbase_devfreq_work_type work_type) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) unsigned long flags; WARN_ON(work_type == DEVFREQ_WORK_NONE); @@ -569,12 +588,10 @@ void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n", kbase_devfreq_req_type_name(work_type)); -#endif } static int kbase_devfreq_work_init(struct kbase_device *kbdev) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) kbdev->devfreq_queue.req_type = DEVFREQ_WORK_NONE; kbdev->devfreq_queue.acted_type = DEVFREQ_WORK_RESUME; @@ -584,15 +601,12 @@ static int kbase_devfreq_work_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->devfreq_queue.work, kbase_devfreq_suspend_resume_worker); -#endif return 0; } static void kbase_devfreq_work_term(struct kbase_device *kbdev) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) destroy_workqueue(kbdev->devfreq_queue.workq); -#endif } int kbase_devfreq_init(struct kbase_device *kbdev) @@ -657,7 +671,8 @@ int kbase_devfreq_init(struct kbase_device *kbdev) } /* devfreq_add_device only copies a few of kbdev->dev's fields, so - * set drvdata explicitly so IPA models can access kbdev. */ + * set drvdata explicitly so IPA models can access kbdev. + */ dev_set_drvdata(&kbdev->devfreq->dev, kbdev); err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); diff --git a/mali_kbase/backend/gpu/mali_kbase_devfreq.h b/mali_kbase/backend/gpu/mali_kbase_devfreq.h index 8c976b2..ab731b9 100644 --- a/mali_kbase/backend/gpu/mali_kbase_devfreq.h +++ b/mali_kbase/backend/gpu/mali_kbase_devfreq.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _BASE_DEVFREQ_H_ @@ -43,4 +62,20 @@ void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq); void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, enum kbase_devfreq_work_type work_type); +/** + * kbase_devfreq_opp_translate - Translate nominal OPP frequency from devicetree + * into real frequency & voltage pair, along with + * core mask + * @kbdev: Device pointer + * @freq: Nominal frequency + * @core_mask: Pointer to u64 to store core mask to + * @freqs: Pointer to array of frequencies + * @volts: Pointer to array of voltages + * + * This function will only perform translation if an operating-points-v2-mali + * table is present in devicetree. If one is not present then it will return an + * untranslated frequency (and corresponding voltage) and all cores enabled. + */ +void kbase_devfreq_opp_translate(struct kbase_device *kbdev, unsigned long freq, + u64 *core_mask, unsigned long *freqs, unsigned long *volts); #endif /* _BASE_DEVFREQ_H_ */ diff --git a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c index 60ae020..7e92ffe 100644 --- a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c @@ -42,11 +42,19 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, registers.l2_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES)); #if !MALI_USE_CSF + /* TGOx */ registers.core_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(CORE_FEATURES)); #else /* !MALI_USE_CSF */ - registers.core_features = 0; -#endif /* !MALI_USE_CSF */ + if (((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) == + GPU_ID2_PRODUCT_TGRX) || + ((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) == + GPU_ID2_PRODUCT_TVAX)) + registers.core_features = + kbase_reg_read(kbdev, GPU_CONTROL_REG(CORE_FEATURES)); + else + registers.core_features = 0; +#endif /* MALI_USE_CSF */ registers.tiler_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_FEATURES)); registers.mem_features = kbase_reg_read(kbdev, @@ -105,6 +113,16 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, registers.stack_present_hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(STACK_PRESENT_HI)); + if (registers.gpu_id >= GPU_ID2_PRODUCT_MAKE(11, 8, 5, 2)) { + registers.gpu_features_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FEATURES_LO)); + registers.gpu_features_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FEATURES_HI)); + } else { + registers.gpu_features_lo = 0; + registers.gpu_features_hi = 0; + } + if (!kbase_is_gpu_removed(kbdev)) { *regdump = registers; return 0; diff --git a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c index 7c5001e..f366265 100644 --- a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /* * GPU backend instrumentation APIs. */ @@ -39,9 +38,7 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, { unsigned long flags; int err = -EINVAL; -#if !MALI_USE_CSF u32 irq_mask; -#endif u32 prfcnt_config; lockdep_assert_held(&kbdev->hwaccess_lock); @@ -58,12 +55,10 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, goto out_err; } -#if !MALI_USE_CSF /* Enable interrupt */ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED); -#endif /* In use, this context is the owner */ kbdev->hwcnt.kctx = kctx; @@ -82,29 +77,6 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; #endif -#if MALI_USE_CSF - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_OFF); - - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), - enable->dump_buffer & 0xFFFFFFFF); - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), - enable->dump_buffer >> 32); - - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CSHW_EN), - enable->fe_bm); - - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_SHADER_EN), - enable->shader_bm); - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_MMU_L2_EN), - enable->mmu_l2_bm); - - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_TILER_EN), - enable->tiler_bm); - - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); -#else kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), prfcnt_config | PRFCNT_CONFIG_MODE_OFF); @@ -126,7 +98,6 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); -#endif spin_lock_irqsave(&kbdev->hwcnt.lock, flags); @@ -148,9 +119,7 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) { unsigned long flags, pm_flags; int err = -EINVAL; -#if !MALI_USE_CSF u32 irq_mask; -#endif struct kbase_device *kbdev = kctx->kbdev; while (1) { @@ -185,10 +154,6 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; kbdev->hwcnt.backend.triggered = 0; -#if MALI_USE_CSF - /* Disable the counters */ - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), 0); -#else /* Disable interrupt */ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), @@ -196,7 +161,6 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) /* Disable the counters */ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0); -#endif kbdev->hwcnt.kctx = NULL; kbdev->hwcnt.addr = 0ULL; @@ -228,7 +192,8 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { /* HW counters are disabled or another dump is ongoing, or we're - * resetting */ + * resetting + */ goto unlock; } @@ -238,32 +203,18 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) */ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; - -#if MALI_USE_CSF - /* Reconfigure the dump address */ - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), - kbdev->hwcnt.addr & 0xFFFFFFFF); - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), - kbdev->hwcnt.addr >> 32); -#else /* Reconfigure the dump address */ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kbdev->hwcnt.addr & 0xFFFFFFFF); kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kbdev->hwcnt.addr >> 32); -#endif /* Start dumping */ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, kbdev->hwcnt.addr); -#if MALI_USE_CSF - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), - GPU_COMMAND_PRFCNT_SAMPLE); -#else kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE); -#endif dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); @@ -272,10 +223,6 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) unlock: spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -#if MALI_USE_CSF - tasklet_schedule(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); -#endif - return err; } KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); @@ -304,53 +251,6 @@ bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, } KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); -#if MALI_USE_CSF -/** - * kbasep_hwcnt_irq_poll_tasklet - tasklet to poll MCU IRQ status register - * - * @data: tasklet parameter which pointer to kbdev - * - * This tasklet poll GPU_IRQ_STATUS register in GPU_CONTROL_MCU page to check - * PRFCNT_SAMPLE_COMPLETED bit. - * - * Tasklet is needed here since work_queue is too slow and cuased some test - * cases timeout, the poll_count variable is introduced to avoid infinite - * loop in unexpected cases, the poll_count is 1 or 2 in normal case, 128 - * should be big enough to exit the tasklet in abnormal cases. - * - * Return: void - */ -static void kbasep_hwcnt_irq_poll_tasklet(unsigned long int data) -{ - struct kbase_device *kbdev = (struct kbase_device *)data; - unsigned long flags, pm_flags; - u32 mcu_gpu_irq_raw_status = 0; - u32 poll_count = 0; - - while (1) { - spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - mcu_gpu_irq_raw_status = kbase_reg_read(kbdev, - GPU_CONTROL_MCU_REG(GPU_IRQ_RAWSTAT)); - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - if (mcu_gpu_irq_raw_status & PRFCNT_SAMPLE_COMPLETED) { - kbase_reg_write(kbdev, - GPU_CONTROL_MCU_REG(GPU_IRQ_CLEAR), - PRFCNT_SAMPLE_COMPLETED); - kbase_instr_hwcnt_sample_done(kbdev); - break; - } else if (poll_count++ > 128) { - dev_err(kbdev->dev, - "Err: HWC dump timeout, count: %u", poll_count); - /* Still call sample_done to unblock waiting thread */ - kbase_instr_hwcnt_sample_done(kbdev); - break; - } - } -} -#endif - void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) { unsigned long flags; @@ -406,20 +306,16 @@ int kbase_instr_hwcnt_clear(struct kbase_context *kctx) spin_lock_irqsave(&kbdev->hwcnt.lock, flags); /* Check it's the context previously set up and we're not already - * dumping */ + * dumping + */ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) goto out; /* Clear the counters */ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); -#if MALI_USE_CSF - kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), - GPU_COMMAND_PRFCNT_CLEAR); -#else kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR); -#endif err = 0; @@ -437,11 +333,6 @@ int kbase_instr_backend_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->hwcnt.backend.wait); -#if MALI_USE_CSF - tasklet_init(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet, - kbasep_hwcnt_irq_poll_tasklet, (unsigned long int)kbdev); -#endif - kbdev->hwcnt.backend.triggered = 0; #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS @@ -460,9 +351,7 @@ int kbase_instr_backend_init(struct kbase_device *kbdev) void kbase_instr_backend_term(struct kbase_device *kbdev) { -#if MALI_USE_CSF - tasklet_kill(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); -#endif + (void)kbdev; } #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS diff --git a/mali_kbase/backend/gpu/mali_kbase_instr_defs.h b/mali_kbase/backend/gpu/mali_kbase_instr_defs.h index e4b6280..1344262 100644 --- a/mali_kbase/backend/gpu/mali_kbase_instr_defs.h +++ b/mali_kbase/backend/gpu/mali_kbase_instr_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -52,9 +71,6 @@ struct kbase_instr_backend { #endif enum kbase_instr_state state; -#if MALI_USE_CSF - struct tasklet_struct csf_hwc_irq_poll_tasklet; -#endif }; #endif /* _KBASE_INSTR_DEFS_H_ */ diff --git a/mali_kbase/backend/gpu/mali_kbase_instr_internal.h b/mali_kbase/backend/gpu/mali_kbase_instr_internal.h index 2254b9f..0716fd4 100644 --- a/mali_kbase/backend/gpu/mali_kbase_instr_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_instr_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Backend-specific HW access instrumentation APIs */ diff --git a/mali_kbase/backend/gpu/mali_kbase_irq_internal.h b/mali_kbase/backend/gpu/mali_kbase_irq_internal.h index ca3c048..f5ad27b 100644 --- a/mali_kbase/backend/gpu/mali_kbase_irq_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_irq_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_irq_linux.c b/mali_kbase/backend/gpu/mali_kbase_irq_linux.c index f6cf6a1..07ab5df 100644 --- a/mali_kbase/backend/gpu/mali_kbase_irq_linux.c +++ b/mali_kbase/backend/gpu/mali_kbase_irq_linux.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. @@ -215,20 +216,21 @@ int kbase_set_custom_irq_handler(struct kbase_device *kbdev, int result = 0; irq_handler_t requested_irq_handler = NULL; - KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && - (GPU_IRQ_HANDLER >= irq_type)); + KBASE_DEBUG_ASSERT((irq_type >= JOB_IRQ_HANDLER) && + (irq_type <= GPU_IRQ_HANDLER)); /* Release previous handler */ if (kbdev->irqs[irq_type].irq) free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); - requested_irq_handler = (NULL != custom_handler) ? custom_handler : - kbase_handler_table[irq_type]; + requested_irq_handler = (custom_handler != NULL) ? + custom_handler : + kbase_handler_table[irq_type]; - if (0 != request_irq(kbdev->irqs[irq_type].irq, - requested_irq_handler, + if (request_irq(kbdev->irqs[irq_type].irq, requested_irq_handler, kbdev->irqs[irq_type].flags | IRQF_SHARED, - dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { + dev_name(kbdev->dev), + kbase_tag(kbdev, irq_type)) != 0) { result = -EINVAL; dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", kbdev->irqs[irq_type].irq, irq_type); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_as.c b/mali_kbase/backend/gpu/mali_kbase_jm_as.c index 9b77589..87c4bd1 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_as.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_as.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -20,7 +21,6 @@ * */ - /* * Register backend context / address space management */ @@ -190,8 +190,8 @@ int kbase_backend_find_and_release_free_address_space( } /* Context was retained while locks were dropped, - * continue looking for free AS */ - + * continue looking for free AS + */ mutex_unlock(&js_devdata->runpool_mutex); mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_defs.h b/mali_kbase/backend/gpu/mali_kbase_jm_defs.h index 7d6c814..c6fad21 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_defs.h +++ b/mali_kbase/backend/gpu/mali_kbase_jm_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -92,13 +111,16 @@ struct kbase_backend_data { /* kbase_prepare_to_reset_gpu has been called */ #define KBASE_RESET_GPU_PREPARED 1 /* kbase_reset_gpu has been called - the reset will now definitely happen - * within the timeout period */ + * within the timeout period + */ #define KBASE_RESET_GPU_COMMITTED 2 /* The GPU reset process is currently occuring (timeout has expired or - * kbasep_try_reset_gpu_early was called) */ + * kbasep_try_reset_gpu_early was called) + */ #define KBASE_RESET_GPU_HAPPENING 3 /* Reset the GPU silently, used when resetting the GPU as part of normal - * behavior (e.g. when exiting protected mode). */ + * behavior (e.g. when exiting protected mode). + */ #define KBASE_RESET_GPU_SILENT 4 struct workqueue_struct *reset_workq; struct work_struct reset_work; diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c index 3658748..d1f2eef 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -205,7 +206,8 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, affinity = kbase_job_write_affinity(kbdev, katom->core_req, js); /* start MMU, medium priority, cache clean/flush on end, clean/flush on - * start */ + * start + */ cfg = kctx->as_nr; if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && @@ -331,7 +333,8 @@ static void kbasep_job_slot_update_head_start_timestamp( /* Only update the timestamp if it's a better estimate * than what's currently stored. This is because our * estimate that accounts for the throttle time may be - * too much of an overestimate */ + * too much of an overestimate + */ katom->start_timestamp = end_timestamp; } } @@ -374,9 +377,9 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) /* treat failed slots as finished slots */ u32 finished = (done & 0xFFFF) | failed; - /* Note: This is inherently unfair, as we always check - * for lower numbered interrupts before the higher - * numbered ones.*/ + /* Note: This is inherently unfair, as we always check for lower + * numbered interrupts before the higher numbered ones. + */ i = ffs(finished) - 1; KBASE_DEBUG_ASSERT(i >= 0); @@ -388,7 +391,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) if (failed & (1u << i)) { /* read out the job slot status code if the job - * slot reported failure */ + * slot reported failure + */ completion_code = kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS)); @@ -402,7 +406,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) /* Soft-stopped job - read the value of * JS<n>_TAIL so that the job chain can - * be resumed */ + * be resumed + */ job_tail = (u64)kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_TAIL_LO)) | ((u64)kbase_reg_read(kbdev, @@ -411,18 +416,21 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) } else if (completion_code == BASE_JD_EVENT_NOT_STARTED) { /* PRLAM-10673 can cause a TERMINATED - * job to come back as NOT_STARTED, but - * the error interrupt helps us detect - * it */ + * job to come back as NOT_STARTED, + * but the error interrupt helps us + * detect it + */ completion_code = BASE_JD_EVENT_TERMINATED; } kbase_gpu_irq_evict(kbdev, i, completion_code); - /* Some jobs that encounter a BUS FAULT may result in corrupted - * state causing future jobs to hang. Reset GPU before - * allowing any other jobs on the slot to continue. */ + /* Some jobs that encounter a BUS FAULT may + * result in corrupted state causing future + * jobs to hang. Reset GPU before allowing + * any other jobs on the slot to continue. + */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { if (kbase_prepare_to_reset_gpu_locked(kbdev)) @@ -483,7 +491,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) if ((rawstat >> (i + 16)) & 1) { /* There is a failed job that we've - * missed - add it back to active */ + * missed - add it back to active + */ active |= (1u << i); } } @@ -585,7 +594,8 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, } /* We are about to issue a soft stop, so mark the atom as having - * been soft stopped */ + * been soft stopped + */ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; /* Mark the point where we issue the soft-stop command */ @@ -1020,6 +1030,33 @@ void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, } } +int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) +{ + WARN(true, "%s Not implemented for JM GPUs", __func__); + return -EINVAL; +} + +int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) +{ + WARN(true, "%s Not implemented for JM GPUs", __func__); + return -EINVAL; +} + +void kbase_reset_gpu_allow(struct kbase_device *kbdev) +{ + WARN(true, "%s Not implemented for JM GPUs", __func__); +} + +void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) +{ + WARN(true, "%s Not implemented for JM GPUs", __func__); +} + +void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) +{ + WARN(true, "%s Not implemented for JM GPUs", __func__); +} + static void kbase_debug_dump_registers(struct kbase_device *kbdev) { int i; @@ -1086,13 +1123,15 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) /* Make sure the timer has completed - this cannot be done from * interrupt context, so this cannot be done within - * kbasep_try_reset_gpu_early. */ + * kbasep_try_reset_gpu_early. + */ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); if (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { /* This would re-activate the GPU. Since it's already idle, - * there's no need to reset it */ + * there's no need to reset it + */ atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); kbase_disjoint_state_down(kbdev); @@ -1113,14 +1152,16 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) kbdev->irq_reset_flush = true; /* Disable IRQ to avoid IRQ handlers to kick in after releasing the - * spinlock; this also clears any outstanding interrupts */ + * spinlock; this also clears any outstanding interrupts + */ kbase_pm_disable_interrupts_nolock(kbdev); spin_unlock(&kbdev->mmu_mask_change); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Ensure that any IRQ handlers have finished - * Must be done without any locks IRQ handlers will take */ + * Must be done without any locks IRQ handlers will take + */ kbase_synchronize_irqs(kbdev); /* Flush out any in-flight work items */ @@ -1131,7 +1172,8 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { /* Ensure that L2 is not transitioning when we send the reset - * command */ + * command + */ while (--max_loops && kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) ; @@ -1146,14 +1188,16 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) /* All slot have been soft-stopped and we've waited * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we * assume that anything that is still left on the GPU is stuck there and - * we'll kill it when we reset the GPU */ + * we'll kill it when we reset the GPU + */ if (!silent) dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", RESET_TIMEOUT); /* Output the state of some interesting registers to help in the - * debugging of GPU resets */ + * debugging of GPU resets + */ if (!silent) kbase_debug_dump_registers(kbdev); @@ -1192,7 +1236,8 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) kbase_pm_update_cores_state(kbdev); /* Synchronously request and wait for those cores, because if - * instrumentation is enabled it would need them immediately. */ + * instrumentation is enabled it would need them immediately. + */ kbase_pm_wait_for_desired_state(kbdev); mutex_unlock(&kbdev->pm.lock); @@ -1269,7 +1314,8 @@ static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) /* Check that the reset has been committed to (i.e. kbase_reset_gpu has * been called), and that no other thread beat this thread to starting - * the reset */ + * the reset + */ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != KBASE_RESET_GPU_COMMITTED) { @@ -1362,7 +1408,8 @@ void kbase_reset_gpu(struct kbase_device *kbdev) KBASE_DEBUG_ASSERT(kbdev); /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occuring here */ + * a race to be occurring here + */ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_PREPARED); atomic_set(&kbdev->hwaccess.backend.reset_gpu, @@ -1385,7 +1432,8 @@ void kbase_reset_gpu_locked(struct kbase_device *kbdev) KBASE_DEBUG_ASSERT(kbdev); /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occuring here */ + * a race to be occurring here + */ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_PREPARED); atomic_set(&kbdev->hwaccess.backend.reset_gpu, diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h index cd1f979..cfb26ee 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Job Manager backend-specific low-level APIs. */ diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c index afaaef2..cf96a86 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -20,7 +21,6 @@ * */ - /* * Register-based HW access backend specific APIs */ @@ -40,10 +40,12 @@ #include <backend/gpu/mali_kbase_pm_internal.h> /* Return whether the specified ringbuffer is empty. HW access lock must be - * held */ + * held + */ #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) /* Return number of atoms currently in the specified ringbuffer. HW access lock - * must be held */ + * must be held + */ #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) static void kbase_gpu_release_atom(struct kbase_device *kbdev, @@ -284,7 +286,8 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, kbase_kinstr_jm_atom_hw_release(katom); /* Inform power management at start/finish of atom so it can * update its GPU utilisation metrics. Mark atom as not - * submitted beforehand. */ + * submitted beforehand. + */ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; kbase_pm_metrics_update(kbdev, end_timestamp); @@ -544,7 +547,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. */ + * there are no atoms currently on the GPU. + */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); /* If hwcnt is disabled, it means we didn't clean up correctly @@ -570,19 +574,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, /* We couldn't disable atomically, so kick off a worker */ if (!kbdev->protected_mode_hwcnt_disabled) { -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, - &kbdev->protected_mode_hwcnt_disable_work); -#else - queue_work(system_highpri_wq, + kbase_hwcnt_context_queue_work( + kbdev->hwcnt_gpu_ctx, &kbdev->protected_mode_hwcnt_disable_work); -#endif return -EAGAIN; } - /* Once reaching this point GPU must be - * switched to protected mode or hwcnt - * re-enabled. */ + /* Once reaching this point GPU must be switched to protected + * mode or hwcnt re-enabled. + */ if (kbase_pm_protected_entry_override_enable(kbdev)) return -EAGAIN; @@ -722,7 +722,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. */ + * there are no atoms currently on the GPU. + */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); @@ -768,8 +769,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order */ + * already removed - as atoms must be returned in order + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -912,12 +913,14 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Set EVENT_DONE so this atom will be - completed, not unpulled. */ + * completed, not unpulled. + */ katom[idx]->event_code = BASE_JD_EVENT_DONE; /* Only return if head atom or previous * atom already removed - as atoms must - * be returned in order. */ + * be returned in order. + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -948,7 +951,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) if (idx == 1) { /* Only submit if head atom or previous - * atom already submitted */ + * atom already submitted + */ if ((katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED && katom[0]->gpu_rb_state != @@ -964,7 +968,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) } /* If inter-slot serialization in use then don't - * submit atom if any other slots are in use */ + * submit atom if any other slots are in use + */ if ((kbdev->serialize_jobs & KBASE_SERIALIZE_INTER_SLOT) && other_slots_busy(kbdev, js)) @@ -976,7 +981,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) break; #endif /* Check if this job needs the cycle counter - * enabled before submission */ + * enabled before submission + */ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) kbase_pm_request_gpu_cycle_counter_l2_is_on( kbdev); @@ -987,7 +993,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) /* Inform power management at start/finish of * atom so it can update its GPU utilisation - * metrics. */ + * metrics. + */ kbase_pm_metrics_update(kbdev, &katom[idx]->start_timestamp); @@ -1000,7 +1007,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) case KBASE_ATOM_GPU_RB_RETURN_TO_JS: /* Only return if head atom or previous atom * already removed - as atoms must be returned - * in order */ + * in order + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -1103,7 +1111,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not * flushed. To prevent future evictions causing possible memory * corruption we need to flush the cache manually before any - * affected memory gets reused. */ + * affected memory gets reused. + */ katom->need_cache_flush_cores_retained = true; } @@ -1184,7 +1193,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Can not dequeue this atom yet - will be - * dequeued when atom at idx0 completes */ + * dequeued when atom at idx0 completes + */ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; kbase_gpu_mark_atom_for_return(kbdev, katom_idx1); @@ -1271,7 +1281,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, kbase_pm_update_state(kbdev); /* Job completion may have unblocked other atoms. Try to update all job - * slots */ + * slots + */ kbase_backend_slot_update(kbdev); } @@ -1322,7 +1333,8 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; /* As the atom was not removed, increment the * index so that we read the correct atom in the - * next iteration. */ + * next iteration. + */ atom_idx++; continue; } @@ -1425,7 +1437,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, katom_idx0_valid = (katom_idx0 == katom); /* If idx0 is to be removed and idx1 is on the same context, * then idx1 must also be removed otherwise the atoms might be - * returned out of order */ + * returned out of order + */ if (katom_idx1) katom_idx1_valid = (katom_idx1 == katom) || (katom_idx0_valid && @@ -1472,7 +1485,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)) == 0) { /* idx0 has already completed - stop - * idx1 if needed*/ + * idx1 if needed + */ if (katom_idx1_valid) { kbase_gpu_stop_atom(kbdev, js, katom_idx1, @@ -1481,7 +1495,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, } } else { /* idx1 is in NEXT registers - attempt - * to remove */ + * to remove + */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), @@ -1496,7 +1511,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, JS_HEAD_NEXT_HI)) != 0) { /* idx1 removed successfully, - * will be handled in IRQ */ + * will be handled in IRQ + */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, true); @@ -1510,7 +1526,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ret = true; } else if (katom_idx1_valid) { /* idx0 has already completed, - * stop idx1 if needed */ + * stop idx1 if needed + */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); @@ -1529,7 +1546,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, * flow was also interrupted, and this function * might not enter disjoint state e.g. if we * don't actually do a hard stop on the head - * atom */ + * atom + */ kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); ret = true; @@ -1557,7 +1575,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ret = true; } else { /* idx1 is in NEXT registers - attempt to - * remove */ + * remove + */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP); @@ -1567,13 +1586,15 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI)) != 0) { /* idx1 removed successfully, will be - * handled in IRQ once idx0 completes */ + * handled in IRQ once idx0 completes + */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); } else { /* idx0 has already completed - stop - * idx1 */ + * idx1 + */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.h b/mali_kbase/backend/gpu/mali_kbase_jm_rb.h index c3b9f2d..1633d6e 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.h +++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /* * Register-based HW access backend specific APIs */ diff --git a/mali_kbase/backend/gpu/mali_kbase_js_backend.c b/mali_kbase/backend/gpu/mali_kbase_js_backend.c index 67c51c7..fb38510 100644 --- a/mali_kbase/backend/gpu/mali_kbase_js_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_js_backend.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -20,7 +21,6 @@ * */ - /* * Register-based HW access backend specific job scheduler APIs */ @@ -48,7 +48,8 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev) /* nr_contexts_pullable is updated with the runpool_mutex. However, the * locking in the caller gives us a barrier that ensures - * nr_contexts_pullable is up-to-date for reading */ + * nr_contexts_pullable is up-to-date for reading + */ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); #ifdef CONFIG_MALI_DEBUG @@ -114,7 +115,8 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) if (atom != NULL) { /* The current version of the model doesn't support - * Soft-Stop */ + * Soft-Stop + */ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { u32 ticks = atom->ticks++; @@ -142,7 +144,8 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) * new soft_stop timeout. This ensures that * atoms do not miss any of the timeouts due to * races between this worker and the thread - * changing the timeouts. */ + * changing the timeouts. + */ if (backend->timeouts_updated && ticks > soft_stop_ticks) ticks = atom->ticks = soft_stop_ticks; @@ -172,10 +175,11 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) * * Similarly, if it's about to be * decreased, the last job from another - * context has already finished, so it's - * not too bad that we observe the older - * value and register a disjoint event - * when we try soft-stopping */ + * context has already finished, so + * it's not too bad that we observe the + * older value and register a disjoint + * event when we try soft-stopping + */ if (js_devdata->nr_user_contexts_running >= disjoint_threshold) softstop_flags |= @@ -287,11 +291,12 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); backend->timer_running = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - /* From now on, return value of timer_callback_should_run() will - * also cause the timer to not requeue itself. Its return value - * cannot change, because it depends on variables updated with - * the runpool_mutex held, which the caller of this must also - * hold */ + /* From now on, return value of timer_callback_should_run() + * will also cause the timer to not requeue itself. Its return + * value cannot change, because it depends on variables updated + * with the runpool_mutex held, which the caller of this must + * also hold + */ hrtimer_cancel(&backend->scheduling_timer); } diff --git a/mali_kbase/backend/gpu/mali_kbase_js_internal.h b/mali_kbase/backend/gpu/mali_kbase_js_internal.h index 6576e55..a48b4ea 100644 --- a/mali_kbase/backend/gpu/mali_kbase_js_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_js_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /* * Register-based HW access backend specific job scheduler APIs */ diff --git a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h index 0c779ac..3d277ac 100644 --- a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h +++ b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h @@ -20,12 +20,13 @@ * *//* SPDX-License-Identifier: GPL-2.0 */ /* + * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_always_on.c b/mali_kbase/backend/gpu/mali_kbase_pm_always_on.c index e33fe0b..c694e9a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_always_on.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_always_on.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2015, 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - /* * "Always on" power management policy */ @@ -62,6 +61,9 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { always_on_shaders_needed, /* shaders_needed */ always_on_get_core_active, /* get_core_active */ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ +#if MALI_USE_CSF + ALWAYS_ON_PM_SCHED_FLAGS, /* pm_sched_flags */ +#endif }; KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_always_on.h b/mali_kbase/backend/gpu/mali_kbase_pm_always_on.h index e7927cf..332c09d 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_always_on.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_always_on.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2015,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * "Always on" power management policy */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c index bb11d68..1e90c6a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c @@ -1,4 +1,5 @@ - /* +// SPDX-License-Identifier: GPL-2.0 +/* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * @@ -20,7 +21,6 @@ * */ - /* * GPU backend implementation of base kernel power management APIs */ @@ -162,6 +162,8 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) if (ret) return ret; #else + mutex_init(&kbdev->pm.backend.policy_change_lock); + kbdev->pm.backend.policy_change_clamp_state_to_off = false; /* Due to dependency on kbase_ipa_control, the metrics subsystem can't * be initialized here. */ @@ -240,7 +242,8 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) lockdep_assert_held(&kbdev->pm.lock); /* Turn clocks and interrupts on - no-op if we haven't done a previous - * kbase_pm_clock_off() */ + * kbase_pm_clock_off() + */ kbase_pm_clock_on(kbdev, is_resume); if (!is_resume) { @@ -258,7 +261,8 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) kbase_pm_update_cores_state(kbdev); /* NOTE: We don't wait to reach the desired state, since running atoms - * will wait for that state to be reached anyway */ + * will wait for that state to be reached anyway + */ } static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) @@ -572,7 +576,8 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); /* Power up the GPU, don't enable IRQs as we are not ready to receive - * them. */ + * them + */ ret = kbase_pm_init_hw(kbdev, flags); if (ret) { kbase_pm_unlock(kbdev); @@ -598,7 +603,8 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, #endif /* Pretend the GPU is active to prevent a power policy turning the GPU - * cores off */ + * cores off + */ kbdev->pm.active_count = 1; spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, @@ -610,7 +616,8 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, irq_flags); /* We are ready to receive IRQ's now as power policy is set up, so - * enable them now. */ + * enable them now. + */ #ifdef CONFIG_MALI_DEBUG kbdev->pm.backend.driver_ready_for_irqs = true; #endif @@ -643,6 +650,8 @@ void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) mutex_lock(&kbdev->pm.lock); kbase_pm_do_poweroff(kbdev); mutex_unlock(&kbdev->pm.lock); + + kbase_pm_wait_for_poweroff_complete(kbdev); } KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); @@ -671,6 +680,12 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) #if !MALI_USE_CSF /* Shut down the metrics subsystem */ kbasep_pm_metrics_term(kbdev); +#else + if (WARN_ON(mutex_is_locked(&kbdev->pm.backend.policy_change_lock))) { + mutex_lock(&kbdev->pm.backend.policy_change_lock); + mutex_unlock(&kbdev->pm.backend.policy_change_lock); + } + mutex_destroy(&kbdev->pm.backend.policy_change_lock); #endif destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); @@ -737,7 +752,8 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) { /* Force power off the GPU and all cores (regardless of policy), only * after the PM active count reaches zero (otherwise, we risk turning it - * off prematurely) */ + * off prematurely) + */ kbase_pm_lock(kbdev); kbase_pm_do_poweroff(kbdev); @@ -783,6 +799,9 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) ktime_t end_timestamp = ktime_get(); struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + if (!kbdev->arb.arb_if) + return; + mutex_lock(&kbdev->pm.lock); mutex_lock(&arb_vm_state->vm_state_lock); if (kbdev->pm.backend.gpu_powered && diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c index 02dfdb0..975f42d 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_ca.h b/mali_kbase/backend/gpu/mali_kbase_pm_ca.h index 5423e96..af91624 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_ca.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_ca.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_ca_devfreq.h b/mali_kbase/backend/gpu/mali_kbase_pm_ca_devfreq.h index f67ec65..49a964c 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_ca_devfreq.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_ca_devfreq.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.c b/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.c index 9eef44a..5317a38 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2016, 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - /* * "Coarse Demand" power management policy */ @@ -61,6 +60,9 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { coarse_demand_shaders_needed, /* shaders_needed */ coarse_demand_get_core_active, /* get_core_active */ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ +#if MALI_USE_CSF + COARSE_ON_DEMAND_PM_SCHED_FLAGS, /* pm_sched_flags */ +#endif }; KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.h b/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.h index 304e5d7..fe17fd3 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_coarse_demand.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2015,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * "Coarse Demand" power management policy */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_defs.h b/mali_kbase/backend/gpu/mali_kbase_pm_defs.h index d7d8f75..2a4f3ac 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_defs.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -166,21 +185,29 @@ enum kbase_shader_core_state { * struct kbasep_pm_metrics - Metrics data collected for use by the power * management framework. * - * @time_busy: number of ns the GPU was busy executing jobs since the - * @time_period_start timestamp. - * @time_idle: number of ns since time_period_start the GPU was not executing - * jobs since the @time_period_start timestamp. - * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that - * if two CL jobs were active for 400ns, this value would be updated - * with 800. - * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that - * if two GL jobs were active for 400ns, this value would be updated - * with 800. + * @time_busy: the amount of time the GPU was busy executing jobs since the + * @time_period_start timestamp, in units of 256ns. This also includes + * time_in_protm, the time spent in protected mode, since it's assumed + * the GPU was busy 100% during this period. + * @time_idle: the amount of time the GPU was not executing jobs since the + * time_period_start timestamp, measured in units of 256ns. + * @busy_cl: the amount of time the GPU was busy executing CL jobs. Note that + * if two CL jobs were active for 256ns, this value would be updated + * with 2 (2x256ns). + * @busy_gl: the amount of time the GPU was busy executing GL jobs. Note that + * if two GL jobs were active for 256ns, this value would be updated + * with 2 (2x256ns). */ struct kbasep_pm_metrics { u32 time_busy; u32 time_idle; -#if !MALI_USE_CSF +#if MALI_USE_CSF + /* + * The amount of time the GPU has spent in protected mode since + * the time_period_start timestamp, measured in units of 256ns. + */ + u32 time_in_protm; +#else u32 busy_cl[2]; u32 busy_gl; #endif @@ -201,6 +228,7 @@ struct kbasep_pm_metrics { * @values: The current values of the power management metrics. The * kbase_pm_get_dvfs_metrics() function is used to compare these * current values with the saved values from a previous invocation. + * @initialized: tracks whether metrics_state has been initialized or not. * @timer: timer to regularly make DVFS decisions based on the power * management metrics. * @timer_active: boolean indicating @timer is running @@ -212,6 +240,10 @@ struct kbasep_pm_metrics_state { #if MALI_USE_CSF /* Handle returned on registering DVFS as a kbase_ipa_control client */ void *ipa_control_client; + /* Decide whether to skip GPU_ACTIVE sanity check in DVFS utilisation + * calculation + */ + bool skip_gpu_active_sanity_check; #else bool gpu_active; u32 active_cl_ctx[2]; @@ -225,6 +257,7 @@ struct kbasep_pm_metrics_state { struct kbasep_pm_metrics values; #ifdef CONFIG_MALI_MIDGARD_DVFS + bool initialized; struct hrtimer timer; bool timer_active; struct kbasep_pm_metrics dvfs_last; @@ -461,6 +494,21 @@ struct kbase_pm_backend_data { #if MALI_USE_CSF /* True if the micro-control unit should be powered on */ bool mcu_desired; + /* Signaling the backend is in PM policy change transition, needs the + * mcu/L2 to be brought back to the off state and remain in that state + * until the flag is cleared. + */ + bool policy_change_clamp_state_to_off; + /* CSF Dynamic PM control flags in accordance to the current active PM + * policy. This field is updated whenever a new policy is activated. + */ + unsigned int csf_pm_sched_flags; + /* Used to serialize the policy change calls. In CSF case, the change + * of policy may involve the scheduler to suspend running CSGs and + * then reconfigure the MCU. This mutex lock is to serialize such + * sequence. + */ + struct mutex policy_change_lock; #endif bool l2_desired; bool l2_always_on; @@ -485,6 +533,23 @@ struct kbase_pm_backend_data { struct work_struct gpu_clock_control_work; }; +#if MALI_USE_CSF +/* CSF PM flag, signaling that the MCU CORE should be kept on */ +#define CSF_DYNAMIC_PM_CORE_KEEP_ON (1 << 0) +/* CSF PM flag, signaling no scheduler suspension on idle groups */ +#define CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE (1 << 1) +/* CSF PM flag, signaling no scheduler suspension on no runnable groups */ +#define CSF_DYNAMIC_PM_SCHED_NO_SUSPEND (1 << 2) + +/* The following flags corresponds to existing defined PM policies */ +#define ALWAYS_ON_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_CORE_KEEP_ON | \ + CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE | \ + CSF_DYNAMIC_PM_SCHED_NO_SUSPEND) +#define COARSE_ON_DEMAND_PM_SCHED_FLAGS (0) +#if !MALI_CUSTOMER_RELEASE +#define ALWAYS_ON_DEMAND_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE) +#endif +#endif /* List of policy IDs */ enum kbase_pm_policy_id { @@ -564,6 +629,15 @@ struct kbase_pm_policy { bool (*get_core_active)(struct kbase_device *kbdev); enum kbase_pm_policy_id id; + +#if MALI_USE_CSF + /* Policy associated with CSF PM scheduling operational flags. + * There are pre-defined required flags exist for each of the + * ARM released policies, such as 'always_on', 'coarse_demand' + * and etc. + */ + unsigned int pm_sched_flags; +#endif }; #endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c index 420d5c5..f67e8b3 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c @@ -32,9 +32,13 @@ #include <mali_kbase_pm.h> #include <mali_kbase_config_defaults.h> #include <mali_kbase_smc.h> -#if !MALI_USE_CSF + +#if MALI_USE_CSF +#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> +#else #include <mali_kbase_hwaccess_jm.h> #endif /* !MALI_USE_CSF */ + #include <mali_kbase_reset_gpu.h> #include <mali_kbase_ctx_sched.h> #include <mali_kbase_hwcnt_context.h> @@ -106,13 +110,13 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) return true; /* MCU is supposed to be ON, only when scheduler.pm_active_count is - * non zero. But for always_on policy also MCU needs to be ON. - * GPUCORE-24926 will add the proper handling for always_on - * power policy. + * non zero. But for always_on policy, the MCU needs to be kept on, + * unless policy changing transition needs it off. */ + return (kbdev->pm.backend.mcu_desired && - (kbdev->pm.backend.pm_current_policy == - &kbase_pm_always_on_policy_ops)); + kbase_pm_no_mcu_core_pwroff(kbdev) && + !kbdev->pm.backend.policy_change_clamp_state_to_off); } #endif @@ -129,6 +133,11 @@ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) !kbdev->pm.backend.shaders_desired) return false; +#if MALI_USE_CSF + if (kbdev->pm.backend.policy_change_clamp_state_to_off) + return false; +#endif + return kbdev->pm.backend.l2_desired; } @@ -260,7 +269,8 @@ static void mali_cci_flush_l2(struct kbase_device *kbdev) GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); /* Wait for cache flush to complete before continuing, exit on - * gpu resets or loop expiry. */ + * gpu resets or loop expiry. + */ while (((raw & mask) == 0) && --loops) { raw = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); @@ -495,14 +505,10 @@ static void kbase_pm_trigger_hwcnt_disable(struct kbase_device *kbdev) */ if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) { backend->hwcnt_disabled = true; + } else { -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, - &backend->hwcnt_disable_work); -#else - queue_work(system_highpri_wq, - &backend->hwcnt_disable_work); -#endif + kbase_hwcnt_context_queue_work(kbdev->hwcnt_gpu_ctx, + &backend->hwcnt_disable_work); } } @@ -520,7 +526,8 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) * Skip if size and hash are not given explicitly, * which means default values are used. */ - if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0)) + if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0) && + (!kbdev->l2_hash_values_override)) return; val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG)); @@ -531,13 +538,25 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) } if (kbdev->l2_hash_override) { + WARN_ON(kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH)); val &= ~L2_CONFIG_HASH_MASK; val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); + } else if (kbdev->l2_hash_values_override) { + int i; + + WARN_ON(!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH)); + val &= ~L2_CONFIG_ASN_HASH_ENABLE_MASK; + val |= (0x1 << L2_CONFIG_ASN_HASH_ENABLE_SHIFT); + + for (i = 0; i < ASN_HASH_COUNT; i++) { + dev_dbg(kbdev->dev, "Program 0x%x to ASN_HASH[%d]\n", + kbdev->l2_hash_values[i], i); + kbase_reg_write(kbdev, GPU_CONTROL_REG(ASN_HASH(i)), + kbdev->l2_hash_values[i]); + } } dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); - - /* Write L2_CONFIG to override */ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG), val); } @@ -564,14 +583,33 @@ static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) return strings[state]; } -static inline void kbase_pm_mcu_update_core_mask(struct kbase_device *kbdev) +static inline bool kbase_pm_handle_mcu_core_attr_update(struct kbase_device *kbdev) { + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + bool timer_update; + bool core_mask_update; + lockdep_assert_held(&kbdev->hwaccess_lock); - WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_ON); - kbase_csf_firmware_update_core_mask(kbdev, - kbdev->pm.backend.shaders_desired_mask); - kbdev->pm.backend.mcu_state = KBASE_MCU_ON_CORE_MASK_UPDATE_PEND; + WARN_ON(backend->mcu_state != KBASE_MCU_ON); + + /* This function is only for cases where the MCU managing Cores, if + * the firmware mode is with host control, do nothing here. + */ + if (unlikely(kbdev->csf.firmware_hctl_core_pwr)) + return false; + + core_mask_update = + backend->shaders_avail != backend->shaders_desired_mask; + + timer_update = kbdev->csf.mcu_core_pwroff_dur_count != + kbdev->csf.mcu_core_pwroff_reg_shadow; + + if (core_mask_update || timer_update) + kbase_csf_firmware_update_core_attr(kbdev, timer_update, + core_mask_update, backend->shaders_desired_mask); + + return (core_mask_update || timer_update); } static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) @@ -591,11 +629,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) } do { + u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); + u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + + /* mask off ready from trans in case transitions finished + * between the register reads + */ + shaders_trans &= ~shaders_ready; + prev_state = backend->mcu_state; switch (backend->mcu_state) { case KBASE_MCU_OFF: if (kbase_pm_is_mcu_desired(kbdev) && + !backend->policy_change_clamp_state_to_off && backend->l2_state == KBASE_L2_ON) { kbase_csf_firmware_trigger_reload(kbdev); backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; @@ -607,7 +654,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); kbase_csf_firmware_global_reinit(kbdev, - backend->shaders_desired_mask); + backend->shaders_desired_mask); backend->mcu_state = KBASE_MCU_ON_GLB_REINIT_PEND; } @@ -616,11 +663,37 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_ON_GLB_REINIT_PEND: if (kbase_csf_firmware_global_reinit_complete(kbdev)) { backend->shaders_avail = - backend->shaders_desired_mask; - backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + backend->shaders_desired_mask; + backend->pm_shaders_core_mask = 0; + if (kbdev->csf.firmware_hctl_core_pwr) { + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + backend->shaders_avail, ACTION_PWRON); + backend->mcu_state = + KBASE_MCU_HCTL_SHADERS_PEND_ON; + } else + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + } + break; + + case KBASE_MCU_HCTL_SHADERS_PEND_ON: + if (!shaders_trans && + shaders_ready == backend->shaders_avail) { + /* Cores now stable, notify MCU the stable mask */ + kbase_csf_firmware_update_core_attr(kbdev, + false, true, shaders_ready); + + backend->pm_shaders_core_mask = shaders_ready; + backend->mcu_state = + KBASE_MCU_HCTL_CORES_NOTIFY_PEND; } break; + case KBASE_MCU_HCTL_CORES_NOTIFY_PEND: + /* Wait for the acknowledgement */ + if (kbase_csf_firmware_core_attr_updated(kbdev)) + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + break; + case KBASE_MCU_ON_HWCNT_ENABLE: backend->hwcnt_desired = true; if (backend->hwcnt_disabled) { @@ -632,19 +705,53 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) break; case KBASE_MCU_ON: + backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); + if (!kbase_pm_is_mcu_desired(kbdev)) backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; - else { - backend->shaders_desired_mask = - kbase_pm_ca_get_core_mask(kbdev); - if (unlikely(backend->shaders_avail != - backend->shaders_desired_mask)) - kbase_pm_mcu_update_core_mask(kbdev); + else if (kbdev->csf.firmware_hctl_core_pwr) { + /* Host control add additional Cores to be active */ + if (backend->shaders_desired_mask & ~shaders_ready) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) + kbase_pm_trigger_hwcnt_disable(kbdev); + backend->mcu_state = + KBASE_MCU_HCTL_MCU_ON_RECHECK; + } + } else if (kbase_pm_handle_mcu_core_attr_update(kbdev)) + kbdev->pm.backend.mcu_state = + KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND; + break; + + case KBASE_MCU_HCTL_MCU_ON_RECHECK: + backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); + + if (!backend->hwcnt_disabled) { + /* Wait for being disabled */ + ; + } else if (!kbase_pm_is_mcu_desired(kbdev)) { + /* Converging to MCU powering down flow */ + backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; + } else if (backend->shaders_desired_mask & ~shaders_ready) { + /* set cores ready but not available to + * meet SHADERS_PEND_ON check pass + */ + backend->shaders_avail = + (backend->shaders_desired_mask | shaders_ready); + + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + backend->shaders_avail & ~shaders_ready, + ACTION_PWRON); + backend->mcu_state = + KBASE_MCU_HCTL_SHADERS_PEND_ON; + } else { + backend->mcu_state = + KBASE_MCU_HCTL_SHADERS_PEND_ON; } break; - case KBASE_MCU_ON_CORE_MASK_UPDATE_PEND: - if (kbase_csf_firmware_core_mask_updated(kbdev)) { + case KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND: + if (kbase_csf_firmware_core_attr_updated(kbdev)) { backend->shaders_avail = backend->shaders_desired_mask; backend->mcu_state = KBASE_MCU_ON; @@ -669,14 +776,32 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (!kbase_pm_is_mcu_desired(kbdev)) { kbase_csf_firmware_trigger_mcu_halt(kbdev); backend->mcu_state = KBASE_MCU_ON_PEND_HALT; - } else if (kbase_pm_is_mcu_desired(kbdev)) { + } else backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; - } break; case KBASE_MCU_ON_PEND_HALT: - if (kbase_csf_firmware_mcu_halted(kbdev)) + if (kbase_csf_firmware_mcu_halted(kbdev)) { + if (kbdev->csf.firmware_hctl_core_pwr) + backend->mcu_state = + KBASE_MCU_HCTL_SHADERS_READY_OFF; + else + backend->mcu_state = KBASE_MCU_POWER_DOWN; + } + break; + + case KBASE_MCU_HCTL_SHADERS_READY_OFF: + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + shaders_ready, ACTION_PWROFF); + backend->mcu_state = + KBASE_MCU_HCTL_SHADERS_PEND_OFF; + break; + + case KBASE_MCU_HCTL_SHADERS_PEND_OFF: + if (!shaders_trans && !shaders_ready) { + backend->pm_shaders_core_mask = 0; backend->mcu_state = KBASE_MCU_POWER_DOWN; + } break; case KBASE_MCU_POWER_DOWN: @@ -1701,7 +1826,8 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) /* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has * aborted due to a fatal signal. If the time spent waiting has exceeded this - * threshold then there is most likely a hardware issue. */ + * threshold then there is most likely a hardware issue. + */ #define PM_TIMEOUT_MS (5000) /* 5s */ static void kbase_pm_timed_out(struct kbase_device *kbdev) @@ -1759,6 +1885,15 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev) kbase_reg_read(kbdev, GPU_CONTROL_REG( L2_PWRTRANS_LO))); +#if MALI_USE_CSF + /* PM timeout probably means hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); +#endif + dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); @@ -1828,7 +1963,7 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) { unsigned long flags; - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); /* * Clear all interrupts, * and unmask them all. @@ -1854,7 +1989,7 @@ KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) { - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); /* * Mask all interrupts, * and clear them all. @@ -1908,7 +2043,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) bool reset_required = is_resume; unsigned long flags; - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); #if !MALI_USE_CSF lockdep_assert_held(&kbdev->js_data.runpool_mutex); #endif /* !MALI_USE_CSF */ @@ -1953,21 +2088,24 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) if (reset_required) { /* GPU state was lost, reset GPU to ensure it is in a - * consistent state */ + * consistent state + */ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); } #ifdef CONFIG_MALI_ARBITER_SUPPORT else { - struct kbase_arbiter_vm_state *arb_vm_state = + if (kbdev->arb.arb_if) { + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - /* In the case that the GPU has just been granted by - * the Arbiter, a reset will have already been done. - * However, it is still necessary to initialize the GPU. - */ - if (arb_vm_state->vm_arb_starting) - kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | - PM_NO_RESET); + /* In the case that the GPU has just been granted by + * the Arbiter, a reset will have already been done. + * However, it is still necessary to initialize the GPU. + */ + if (arb_vm_state->vm_arb_starting) + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | + PM_NO_RESET); + } } #endif /* CONFIG_MALI_ARBITER_SUPPORT */ @@ -2014,7 +2152,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) { unsigned long flags; - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); lockdep_assert_held(&kbdev->pm.lock); /* ASSERT that the cores should now be unavailable. No lock needed. */ @@ -2038,7 +2176,8 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) if (atomic_read(&kbdev->faults_pending)) { /* Page/bus faults are still being processed. The GPU can not - * be powered off until they have completed */ + * be powered off until they have completed + */ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return false; } @@ -2119,7 +2258,8 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) rtdata->timed_out = 1; /* Set the wait queue to wake up kbase_pm_init_hw even though the reset - * hasn't completed */ + * hasn't completed + */ kbase_pm_reset_done(rtdata->kbdev); return HRTIMER_NORESTART; @@ -2386,11 +2526,13 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) } /* No interrupt has been received - check if the RAWSTAT register says - * the reset has completed */ + * the reset has completed + */ if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & RESET_COMPLETED)) { /* The interrupt is set in the RAWSTAT; this suggests that the - * interrupts are not getting to the CPU */ + * interrupts are not getting to the CPU + */ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); /* If interrupts aren't working we can't continue. */ destroy_hrtimer_on_stack(&rtdata.timer); @@ -2404,33 +2546,40 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) } /* The GPU doesn't seem to be responding to the reset so try a hard - * reset */ - dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", - RESET_TIMEOUT); - KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_HARD_RESET); + * reset, but only when NOT in arbitration mode. + */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (!kbdev->arb.arb_if) { +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", + RESET_TIMEOUT); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_HARD_RESET); - /* Restart the timer to wait for the hard reset to complete */ - rtdata.timed_out = 0; + /* Restart the timer to wait for the hard reset to complete */ + rtdata.timed_out = 0; - hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), - HRTIMER_MODE_REL); + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); - /* Wait for the RESET_COMPLETED interrupt to be raised */ - kbase_pm_wait_for_reset(kbdev); + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); - if (rtdata.timed_out == 0) { - /* GPU has been reset */ - hrtimer_cancel(&rtdata.timer); - destroy_hrtimer_on_stack(&rtdata.timer); - return 0; - } + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } - destroy_hrtimer_on_stack(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); - dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", - RESET_TIMEOUT); + dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", + RESET_TIMEOUT); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ return -EINVAL; } @@ -2454,7 +2603,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) unsigned long irq_flags; int err = 0; - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); lockdep_assert_held(&kbdev->pm.lock); /* Ensure the clock is on before attempting to access the hardware */ @@ -2466,7 +2615,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) } /* Ensure interrupts are off to begin with, this also clears any - * outstanding interrupts */ + * outstanding interrupts + */ kbase_pm_disable_interrupts(kbdev); /* Ensure cache snoops are disabled before reset. */ kbase_pm_cache_snoop_disable(kbdev); @@ -2487,6 +2637,12 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbdev->protected_dev); spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); +#if MALI_USE_CSF + if (kbdev->protected_mode) { + kbase_ipa_control_protm_exited(kbdev); + kbase_hwcnt_backend_csf_protm_exited(&kbdev->hwcnt_gpu_iface); + } +#endif kbdev->protected_mode = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); @@ -2507,7 +2663,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) GPU_STATUS_PROTECTED_MODE_ACTIVE); /* If cycle counter was in use re-enable it, enable_irqs will only be - * false when called from kbase_pm_powerup */ + * false when called from kbase_pm_powerup + */ if (kbdev->pm.backend.gpu_cycle_counter_requests && (flags & PM_ENABLE_IRQS)) { kbase_pm_enable_interrupts(kbdev); @@ -2565,7 +2722,7 @@ kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) ++kbdev->pm.backend.gpu_cycle_counter_requests; - if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) + if (kbdev->pm.backend.gpu_cycle_counter_requests == 1) kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START); @@ -2617,7 +2774,7 @@ void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) --kbdev->pm.backend.gpu_cycle_counter_requests; - if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) + if (kbdev->pm.backend.gpu_cycle_counter_requests == 0) kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h index 9f02dd4..304ecda 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Power management API definitions used internally by GPU backend */ @@ -492,7 +509,8 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev); void kbase_pm_register_access_disable(struct kbase_device *kbdev); /* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline - * function */ + * function + */ /** * kbase_pm_metrics_is_active - Check if the power management metrics @@ -724,6 +742,59 @@ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); * Return: true if MCU needs to be enabled. */ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); + +/** + * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be + * suspended to low power state when all + * the CSGs are idle + * + * @kbdev: Device pointer + * + * Return: true if allowed to enter the suspended state. + */ +static inline +bool kbase_pm_idle_groups_sched_suspendable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return !(kbdev->pm.backend.csf_pm_sched_flags & + CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE); +} + +/** + * kbase_pm_no_runnables_sched_suspendable - Check whether the scheduler can be + * suspended to low power state when + * there are no runnable CSGs. + * + * @kbdev: Device pointer + * + * Return: true if allowed to enter the suspended state. + */ +static inline +bool kbase_pm_no_runnables_sched_suspendable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return !(kbdev->pm.backend.csf_pm_sched_flags & + CSF_DYNAMIC_PM_SCHED_NO_SUSPEND); +} + +/** + * kbase_pm_no_mcu_core_pwroff - Check whether the PM is required to keep the + * MCU core powered in accordance to the active + * power management policy + * + * @kbdev: Device pointer + * + * Return: true if the MCU is to retain powered. + */ +static inline bool kbase_pm_no_mcu_core_pwroff(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbdev->pm.backend.csf_pm_sched_flags & + CSF_DYNAMIC_PM_CORE_KEEP_ON; +} #endif /** diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_l2_states.h b/mali_kbase/backend/gpu/mali_kbase_pm_l2_states.h index 12cb051..90a46fa 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_l2_states.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_l2_states.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_mcu_states.h b/mali_kbase/backend/gpu/mali_kbase_pm_mcu_states.h index bc3e6b1..95e68df 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_mcu_states.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_mcu_states.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -31,10 +50,16 @@ KBASEP_MCU_STATE(PEND_ON_RELOAD) KBASEP_MCU_STATE(ON_GLB_REINIT_PEND) KBASEP_MCU_STATE(ON_HWCNT_ENABLE) KBASEP_MCU_STATE(ON) -KBASEP_MCU_STATE(ON_CORE_MASK_UPDATE_PEND) +KBASEP_MCU_STATE(ON_CORE_ATTR_UPDATE_PEND) KBASEP_MCU_STATE(ON_HWCNT_DISABLE) KBASEP_MCU_STATE(ON_HALT) KBASEP_MCU_STATE(ON_PEND_HALT) KBASEP_MCU_STATE(POWER_DOWN) KBASEP_MCU_STATE(PEND_OFF) KBASEP_MCU_STATE(RESET_WAIT) +/* Additional MCU states with HOST_CONTROL_SHADERS */ +KBASEP_MCU_STATE(HCTL_SHADERS_PEND_ON) +KBASEP_MCU_STATE(HCTL_CORES_NOTIFY_PEND) +KBASEP_MCU_STATE(HCTL_MCU_ON_RECHECK) +KBASEP_MCU_STATE(HCTL_SHADERS_READY_OFF) +KBASEP_MCU_STATE(HCTL_SHADERS_PEND_OFF) diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c index dc07412..5c42c0a 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /* * Metrics for power management */ @@ -42,13 +41,11 @@ /* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly - * under 11s. Exceeding this will cause overflow */ + * under 11s. Exceeding this will cause overflow + */ #define KBASE_PM_TIME_SHIFT 8 #if MALI_USE_CSF -/* CSHW counter block offsets */ -#define GPU_ACTIVE (4) - /* To get the GPU_ACTIVE value in nano seconds unit */ #define GPU_ACTIVE_SCALING_FACTOR ((u64)1E9) #endif @@ -91,6 +88,7 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) kbdev->pm.backend.metrics.time_period_start = ktime_get(); kbdev->pm.backend.metrics.values.time_busy = 0; kbdev->pm.backend.metrics.values.time_idle = 0; + kbdev->pm.backend.metrics.values.time_in_protm = 0; perf_counter.scaling_factor = GPU_ACTIVE_SCALING_FACTOR; @@ -101,7 +99,7 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) perf_counter.type = KBASE_IPA_CORE_TYPE_CSHW; /* We need the GPU_ACTIVE counter */ - perf_counter.idx = GPU_ACTIVE; + perf_counter.idx = GPU_ACTIVE_CNT_IDX; err = kbase_ipa_control_register( kbdev, &perf_counter, NUM_PERF_COUNTERS, @@ -137,10 +135,18 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); kbdev->pm.backend.metrics.timer.function = dvfs_callback; - + kbdev->pm.backend.metrics.initialized = true; kbase_pm_metrics_start(kbdev); #endif /* CONFIG_MALI_MIDGARD_DVFS */ +#if MALI_USE_CSF + /* The sanity check on the GPU_ACTIVE performance counter + * is skipped for Juno platforms that have timing problems. + */ + kbdev->pm.backend.metrics.skip_gpu_active_sanity_check = + of_machine_is_compatible("arm,juno"); +#endif + return 0; } KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); @@ -157,6 +163,7 @@ void kbasep_pm_metrics_term(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); hrtimer_cancel(&kbdev->pm.backend.metrics.timer); + kbdev->pm.backend.metrics.initialized = false; #endif /* CONFIG_MALI_MIDGARD_DVFS */ #if MALI_USE_CSF @@ -172,42 +179,111 @@ KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); */ #if MALI_USE_CSF #if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) -static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, - ktime_t now) +static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) { - ktime_t diff; int err; u64 gpu_active_counter; + u64 protected_time; + ktime_t now; lockdep_assert_held(&kbdev->pm.backend.metrics.lock); - diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); - if (ktime_to_ns(diff) < 0) - return; - - /* TODO: The final parameter to this function is used to obtain the amount of - * protected-mode time the GPU has spent. This is the subject of a future - * ticket, and is not yet considered / implemented. + /* Query IPA_CONTROL for the latest GPU-active and protected-time + * info. */ err = kbase_ipa_control_query( kbdev, kbdev->pm.backend.metrics.ipa_control_client, - &gpu_active_counter, 1, NULL); + &gpu_active_counter, 1, &protected_time); + + /* Read the timestamp after reading the GPU_ACTIVE counter value. + * This ensures the time gap between the 2 reads is consistent for + * a meaningful comparison between the increment of GPU_ACTIVE and + * elapsed time. The lock taken inside kbase_ipa_control_query() + * function can cause lot of variation. + */ + now = ktime_get(); if (err) { dev_err(kbdev->dev, "Failed to query the increment of GPU_ACTIVE counter: err=%d", err); } else { - u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); + u64 diff_ns, margin_ns; + s64 diff_ns_signed; + u32 ns_time; + ktime_t diff = ktime_sub( + now, kbdev->pm.backend.metrics.time_period_start); + + diff_ns_signed = ktime_to_ns(diff); + + if (diff_ns_signed < 0) + return; + + diff_ns = (u64)diff_ns_signed; + + /* Use a margin value that is approximately 1% of the time + * difference. + */ + margin_ns = diff_ns >> 6; + + /* Calculate time difference in units of 256ns */ + ns_time = (u32)(diff_ns >> KBASE_PM_TIME_SHIFT); + +#ifndef CONFIG_MALI_NO_MALI + /* The GPU_ACTIVE counter shouldn't clock-up more time than has + * actually elapsed - but still some margin needs to be given + * when doing the comparison. There could be some drift between + * the CPU and GPU clock. + * + * Can do the check only in a real driver build, as an arbitrary + * value for GPU_ACTIVE can be fed into dummy model in no_mali + * configuration which may not correspond to the real elapsed + * time. + */ + if (!kbdev->pm.backend.metrics.skip_gpu_active_sanity_check) { + if (gpu_active_counter > (diff_ns + margin_ns)) { + dev_info( + kbdev->dev, + "GPU activity takes longer than time interval: %llu ns > %llu ns", + (unsigned long long)gpu_active_counter, + (unsigned long long)diff_ns); + } + } +#else + CSTD_UNUSED(margin_ns); +#endif + /* Add protected_time to gpu_active_counter so that time in + * protected mode is included in the apparent GPU active time, + * then convert it from units of 1ns to units of 256ns, to + * match what JM GPUs use. The assumption is made here that the + * GPU is 100% busy while in protected mode, so we should add + * this since the GPU can't (and thus won't) update these + * counters while it's actually in protected mode. + * + * Perform the add after dividing each value down, to reduce + * the chances of overflows. + */ + protected_time >>= KBASE_PM_TIME_SHIFT; gpu_active_counter >>= KBASE_PM_TIME_SHIFT; + gpu_active_counter += protected_time; - WARN_ON_ONCE(gpu_active_counter > ns_time); + /* Ensure the following equations don't go wrong if ns_time is + * slightly larger than gpu_active_counter somehow + */ + gpu_active_counter = MIN(gpu_active_counter, ns_time); kbdev->pm.backend.metrics.values.time_busy += gpu_active_counter; + kbdev->pm.backend.metrics.values.time_idle += ns_time - gpu_active_counter; + + /* Also make time in protected mode available explicitly, + * so users of this data have this info, too. + */ + kbdev->pm.backend.metrics.values.time_in_protm += + protected_time; } kbdev->pm.backend.metrics.time_period_start = now; @@ -240,8 +316,8 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, if (kbdev->pm.backend.metrics.active_gl_ctx[2]) kbdev->pm.backend.metrics.values.busy_gl += ns_time; } else { - kbdev->pm.backend.metrics.values.time_idle += (u32) (ktime_to_ns(diff) - >> KBASE_PM_TIME_SHIFT); + kbdev->pm.backend.metrics.values.time_idle += + (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); } kbdev->pm.backend.metrics.time_period_start = now; @@ -257,13 +333,19 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, unsigned long flags; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); +#if MALI_USE_CSF + kbase_pm_get_dvfs_utilisation_calc(kbdev); +#else kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get()); +#endif memset(diff, 0, sizeof(*diff)); diff->time_busy = cur->time_busy - last->time_busy; diff->time_idle = cur->time_idle - last->time_idle; -#if !MALI_USE_CSF +#if MALI_USE_CSF + diff->time_in_protm = cur->time_in_protm - last->time_in_protm; +#else diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; diff->busy_gl = cur->busy_gl - last->busy_gl; @@ -291,7 +373,8 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) diff = &kbdev->pm.backend.metrics.dvfs_diff; - kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, diff); + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, + diff); utilisation = (100 * diff->time_busy) / max(diff->time_busy + diff->time_idle, 1u); @@ -303,8 +386,15 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; - kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); + kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, + util_cl_share); #else + /* Note that, at present, we don't pass protected-mode time to the + * platform here. It's unlikely to be useful, however, as the platform + * probably just cares whether the GPU is busy or not; time in + * protected mode is already added to busy-time at this point, though, + * so we should be good. + */ kbase_platform_dvfs_event(kbdev, utilisation); #endif } @@ -327,11 +417,20 @@ KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); void kbase_pm_metrics_start(struct kbase_device *kbdev) { unsigned long flags; + bool update = true; + + if (unlikely(!kbdev->pm.backend.metrics.initialized)) + return; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - kbdev->pm.backend.metrics.timer_active = true; + if (!kbdev->pm.backend.metrics.timer_active) + kbdev->pm.backend.metrics.timer_active = true; + else + update = false; spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); - hrtimer_start(&kbdev->pm.backend.metrics.timer, + + if (update) + hrtimer_start(&kbdev->pm.backend.metrics.timer, HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), HRTIMER_MODE_REL); } @@ -339,11 +438,20 @@ void kbase_pm_metrics_start(struct kbase_device *kbdev) void kbase_pm_metrics_stop(struct kbase_device *kbdev) { unsigned long flags; + bool update = true; + + if (unlikely(!kbdev->pm.backend.metrics.initialized)) + return; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - kbdev->pm.backend.metrics.timer_active = false; + if (kbdev->pm.backend.metrics.timer_active) + kbdev->pm.backend.metrics.timer_active = false; + else + update = false; spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); - hrtimer_cancel(&kbdev->pm.backend.metrics.timer); + + if (update) + hrtimer_cancel(&kbdev->pm.backend.metrics.timer); } @@ -374,7 +482,8 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); /* Head atom may have just completed, so if it isn't running - * then try the next atom */ + * then try the next atom + */ if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) katom = kbase_gpu_inspect(kbdev, js, 1); @@ -413,8 +522,8 @@ void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) timestamp = &now; } - /* Track how much of time has been spent busy or idle. For JM GPUs, this also - * evaluates how long CL and/or GL jobs have been busy for + /* Track how much of time has been spent busy or idle. For JM GPUs, + * this also evaluates how long CL and/or GL jobs have been busy for. */ kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c index 426e8a4..60eae35 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -28,6 +29,11 @@ #include <gpu/mali_kbase_gpu_regmap.h> #include <mali_kbase_pm.h> #include <backend/gpu/mali_kbase_pm_internal.h> +#include <mali_kbase_reset_gpu.h> + +#if MALI_USE_CSF && defined CONFIG_MALI_DEBUG +#include <csf/mali_kbase_csf_firmware.h> +#endif static const struct kbase_pm_policy *const all_policy_list[] = { #ifdef CONFIG_MALI_NO_MALI @@ -45,11 +51,43 @@ static const struct kbase_pm_policy *const all_policy_list[] = { #endif /* CONFIG_MALI_NO_MALI */ }; +#if MALI_USE_CSF +void kbase_pm_policy_init(struct kbase_device *kbdev) +{ + unsigned long flags; + const struct kbase_pm_policy *default_policy = all_policy_list[0]; + +#if defined CONFIG_MALI_DEBUG + /* Use always_on policy if module param fw_debug=1 is + * passed, to aid firmware debugging. + */ + if (fw_debug) + default_policy = &kbase_pm_always_on_policy_ops; +#endif + default_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.pm_current_policy = default_policy; + kbdev->pm.backend.csf_pm_sched_flags = + default_policy->pm_sched_flags; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} +#else /* MALI_USE_CSF */ void kbase_pm_policy_init(struct kbase_device *kbdev) { kbdev->pm.backend.pm_current_policy = all_policy_list[0]; + +#if MALI_USE_CSF && defined CONFIG_MALI_DEBUG + /* Use always_on policy if module param fw_debug=1 is + * passed, to aid firmware debugging. + */ + if (fw_debug) + kbdev->pm.backend.pm_current_policy = + &kbase_pm_always_on_policy_ops; +#endif kbdev->pm.backend.pm_current_policy->init(kbdev); } +#endif /* MALI_USE_CSF */ void kbase_pm_policy_term(struct kbase_device *kbdev) { @@ -102,7 +140,8 @@ void kbase_pm_update_active(struct kbase_device *kbdev) } } else { /* It is an error for the power policy to power off the GPU - * when there are contexts active */ + * when there are contexts active + */ KBASE_DEBUG_ASSERT(pm->active_count == 0); pm->backend.poweron_required = false; @@ -167,7 +206,8 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) if (kbdev->pm.backend.protected_transition_override) /* We are trying to change in/out of protected mode - force all - * cores off so that the L2 powers down */ + * cores off so that the L2 powers down + */ shaders_desired = false; else shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); @@ -219,20 +259,106 @@ const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_pm_get_policy); +#if MALI_USE_CSF +static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) +{ +#define WAIT_DURATION_MS (3000) + long remaining; + long timeout = kbase_csf_timeout_in_jiffies(WAIT_DURATION_MS); + int err = 0; + + /* Wait for L2 becoming off, by which the MCU is also implicitly off + * since the L2 state machine would only start its power-down + * sequence when the MCU is in off state. The L2 off is required + * as the tiler may need to be power cycled for MCU reconfiguration + * for host control of shader cores. + */ +#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE + remaining = wait_event_killable_timeout( + kbdev->pm.backend.gpu_in_desired_state_wait, + kbdev->pm.backend.l2_state == KBASE_L2_OFF, timeout); +#else + remaining = wait_event_timeout( + kbdev->pm.backend.gpu_in_desired_state_wait, + kbdev->pm.backend.l2_state == KBASE_L2_OFF, timeout); +#endif + + if (!remaining) { + err = -ETIMEDOUT; + } else if (remaining < 0) { + dev_info(kbdev->dev, + "Wait for L2_off got interrupted"); + err = (int)remaining; + } + + dev_dbg(kbdev->dev, "%s: err=%d mcu_state=%d, L2_state=%d\n", __func__, + err, kbdev->pm.backend.mcu_state, kbdev->pm.backend.l2_state); + + return err; +} +#endif + void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *new_policy) { const struct kbase_pm_policy *old_policy; unsigned long flags; +#if MALI_USE_CSF + unsigned int new_policy_csf_pm_sched_flags; + bool sched_suspend; + bool reset_gpu = false; +#endif KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(new_policy != NULL); KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); +#if MALI_USE_CSF + /* Serialize calls on kbase_pm_set_policy() */ + mutex_lock(&kbdev->pm.backend.policy_change_lock); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /* policy_change_clamp_state_to_off, when needed, is set/cleared in + * this function, a very limited temporal scope for covering the + * change transition. + */ + WARN_ON(kbdev->pm.backend.policy_change_clamp_state_to_off); + new_policy_csf_pm_sched_flags = new_policy->pm_sched_flags; + + /* Requiring the scheduler PM suspend operation when changes involving + * the always_on policy, reflected by the CSF_DYNAMIC_PM_CORE_KEEP_ON + * flag bit. + */ + sched_suspend = kbdev->csf.firmware_inited && + (CSF_DYNAMIC_PM_CORE_KEEP_ON & + (new_policy_csf_pm_sched_flags | + kbdev->pm.backend.csf_pm_sched_flags)); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (sched_suspend) + kbase_csf_scheduler_pm_suspend(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /* If the current active policy is always_on, one needs to clamp the + * MCU/L2 for reaching off-state + */ + if (sched_suspend) + kbdev->pm.backend.policy_change_clamp_state_to_off = + CSF_DYNAMIC_PM_CORE_KEEP_ON & kbdev->pm.backend.csf_pm_sched_flags; + + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (sched_suspend) + reset_gpu = policy_change_wait_for_L2_off(kbdev); +#endif + /* During a policy change we pretend the GPU is active */ /* A suspend won't happen here, because we're in a syscall from a - * userspace thread */ + * userspace thread + */ kbase_pm_context_active(kbdev); kbase_pm_lock(kbdev); @@ -253,19 +379,40 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = new_policy; +#if MALI_USE_CSF + kbdev->pm.backend.csf_pm_sched_flags = new_policy_csf_pm_sched_flags; + /* New policy in place, release the clamping on mcu/L2 off state */ + kbdev->pm.backend.policy_change_clamp_state_to_off = false; + kbase_pm_update_state(kbdev); +#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* If any core power state changes were previously attempted, but * couldn't be made because the policy was changing (current_policy was - * NULL), then re-try them here. */ + * NULL), then re-try them here. + */ kbase_pm_update_active(kbdev); kbase_pm_update_cores_state(kbdev); kbase_pm_unlock(kbdev); /* Now the policy change is finished, we release our fake context active - * reference */ + * reference + */ kbase_pm_context_idle(kbdev); + +#if MALI_USE_CSF + /* Reverse the suspension done */ + if (reset_gpu) { + dev_warn(kbdev->dev, "Resorting to GPU reset for policy change\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + kbase_reset_gpu_wait(kbdev); + } else if (sched_suspend) + kbase_csf_scheduler_pm_resume(kbdev); + + mutex_unlock(&kbdev->pm.backend.policy_change_lock); +#endif } KBASE_EXPORT_TEST_API(kbase_pm_set_policy); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_policy.h b/mali_kbase/backend/gpu/mali_kbase_pm_policy.h index f103ef0..d566d9b 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_policy.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_policy.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_shader_states.h b/mali_kbase/backend/gpu/mali_kbase_pm_shader_states.h index 6cafaa1..dafb117 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_shader_states.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_shader_states.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/backend/gpu/mali_kbase_time.c b/mali_kbase/backend/gpu/mali_kbase_time.c index e19f53b..557b595 100644 --- a/mali_kbase/backend/gpu/mali_kbase_time.c +++ b/mali_kbase/backend/gpu/mali_kbase_time.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/build.bp b/mali_kbase/build.bp index b8d8310..3afd42b 100644 --- a/mali_kbase/build.bp +++ b/mali_kbase/build.bp @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. @@ -5,11 +6,16 @@ * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/mali_kbase/context/backend/mali_kbase_context_csf.c b/mali_kbase/context/backend/mali_kbase_context_csf.c index 6476921..da8cfc1 100644 --- a/mali_kbase/context/backend/mali_kbase_context_csf.c +++ b/mali_kbase/context/backend/mali_kbase_context_csf.c @@ -32,6 +32,7 @@ #include <mali_kbase_mem_linux.h> #include <mali_kbase_mem_pool_group.h> #include <mmu/mali_kbase_mmu.h> +#include <tl/mali_kbase_timeline.h> #ifdef CONFIG_DEBUG_FS #include <csf/mali_kbase_csf_csg_debugfs.h> @@ -72,24 +73,34 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx) KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); #endif /* CONFIG_DEBUG_FS */ +static void kbase_context_free(struct kbase_context *kctx) +{ + kbase_timeline_post_kbase_context_destroy(kctx); + + vfree(kctx); +} + static const struct kbase_context_init context_init[] = { - {kbase_context_common_init, kbase_context_common_term, NULL}, - {kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, - "Memory pool goup initialization failed"}, - {kbase_mem_evictable_init, kbase_mem_evictable_deinit, - "Memory evictable initialization failed"}, - {kbase_context_mmu_init, kbase_context_mmu_term, - "MMU initialization failed"}, - {kbase_context_mem_alloc_page, kbase_context_mem_pool_free, - "Memory alloc page failed"}, - {kbase_region_tracker_init, kbase_region_tracker_term, - "Region tracker initialization failed"}, - {kbase_sticky_resource_init, kbase_context_sticky_resource_term, - "Sticky resource initialization failed"}, - {kbase_jit_init, kbase_jit_term, - "JIT initialization failed"}, - {kbase_csf_ctx_init, kbase_csf_ctx_term, - "CSF context initialization failed"}, + { NULL, kbase_context_free, NULL }, + { kbase_context_common_init, kbase_context_common_term, + "Common context initialization failed" }, + { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, + "Memory pool group initialization failed" }, + { kbase_mem_evictable_init, kbase_mem_evictable_deinit, + "Memory evictable initialization failed" }, + { kbase_context_mmu_init, kbase_context_mmu_term, + "MMU initialization failed" }, + { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, + "Memory alloc page failed" }, + { kbase_region_tracker_init, kbase_region_tracker_term, + "Region tracker initialization failed" }, + { kbase_sticky_resource_init, kbase_context_sticky_resource_term, + "Sticky resource initialization failed" }, + { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, + { kbase_csf_ctx_init, kbase_csf_ctx_term, + "CSF context initialization failed" }, + { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, + "Adding kctx to device failed" }, }; static void kbase_context_term_partial( @@ -133,14 +144,23 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, #if defined(CONFIG_64BIT) else kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); -#endif /* !defined(CONFIG_64BIT) */ +#endif /* defined(CONFIG_64BIT) */ for (i = 0; i < ARRAY_SIZE(context_init); i++) { - int err = context_init[i].init(kctx); + int err = 0; + + if (context_init[i].init) + err = context_init[i].init(kctx); if (err) { dev_err(kbdev->dev, "%s error = %d\n", context_init[i].err_mes, err); + + /* kctx should be freed by kbase_context_free(). + * Otherwise it will result in memory leak. + */ + WARN_ON(i == 0); + kbase_context_term_partial(kctx, i); return NULL; } @@ -163,9 +183,8 @@ void kbase_destroy_context(struct kbase_context *kctx) /* Context termination could happen whilst the system suspend of * the GPU device is ongoing or has completed. It has been seen on - * Customer side for JM GPUs that a hang could occur if context - * termination is not blocked until the resume of GPU device. - * Similar issue can potentially occur on CSF GPUs also. + * Customer side that a hang could occur if context termination is + * not blocked until the resume of GPU device. */ while (kbase_pm_context_active_handle_suspend( kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { diff --git a/mali_kbase/context/backend/mali_kbase_context_jm.c b/mali_kbase/context/backend/mali_kbase_context_jm.c index a8cefb6..d0111be 100644 --- a/mali_kbase/context/backend/mali_kbase_context_jm.c +++ b/mali_kbase/context/backend/mali_kbase_context_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,7 +35,6 @@ #include <mali_kbase_mem_pool_group.h> #include <mmu/mali_kbase_mmu.h> #include <tl/mali_kbase_timeline.h> -#include <tl/mali_kbase_tracepoints.h> #ifdef CONFIG_DEBUG_FS #include <mali_kbase_debug_mem_view.h> @@ -47,14 +46,12 @@ void kbase_context_debugfs_init(struct kbase_context *const kctx) kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); kbase_jit_debugfs_init(kctx); kbasep_jd_debugfs_ctx_init(kctx); - kbase_debug_job_fault_context_init(kctx); } KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); void kbase_context_debugfs_term(struct kbase_context *const kctx) { debugfs_remove_recursive(kctx->kctx_dentry); - kbase_debug_job_fault_context_term(kctx); } KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); #else @@ -73,12 +70,7 @@ KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) { - int ret = kbase_kinstr_jm_init(&kctx->kinstr_jm); - - if (!ret) - return ret; - - return 0; + return kbase_kinstr_jm_init(&kctx->kinstr_jm); } static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) @@ -114,12 +106,27 @@ static int kbase_context_submit_check(struct kbase_context *kctx) return 0; } +static void kbase_context_flush_jobs(struct kbase_context *kctx) +{ + kbase_jd_zap_context(kctx); + flush_workqueue(kctx->jctx.job_done_wq); +} + +static void kbase_context_free(struct kbase_context *kctx) +{ + kbase_timeline_post_kbase_context_destroy(kctx); + + vfree(kctx); +} + static const struct kbase_context_init context_init[] = { - { kbase_context_common_init, kbase_context_common_term, NULL }, + { NULL, kbase_context_free, NULL }, + { kbase_context_common_init, kbase_context_common_term, + "Common context initialization failed" }, { kbase_dma_fence_init, kbase_dma_fence_term, "DMA fence initialization failed" }, { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, - "Memory pool goup initialization failed" }, + "Memory pool group initialization failed" }, { kbase_mem_evictable_init, kbase_mem_evictable_deinit, "Memory evictable initialization failed" }, { kbase_context_mmu_init, kbase_context_mmu_term, @@ -134,13 +141,22 @@ static const struct kbase_context_init context_init[] = { { kbase_context_kbase_kinstr_jm_init, kbase_context_kbase_kinstr_jm_term, "JM instrumentation initialization failed" }, - { kbase_context_kbase_timer_setup, NULL, NULL }, + { kbase_context_kbase_timer_setup, NULL, + "Timers initialization failed" }, { kbase_event_init, kbase_event_cleanup, "Event initialization failed" }, { kbasep_js_kctx_init, kbasep_js_kctx_term, "JS kctx initialization failed" }, { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, - { kbase_context_submit_check, NULL, NULL }, + { kbase_context_submit_check, NULL, "Enabling job submission failed" }, +#ifdef CONFIG_DEBUG_FS + { kbase_debug_job_fault_context_init, + kbase_debug_job_fault_context_term, + "Job fault context initialization failed" }, +#endif + { NULL, kbase_context_flush_jobs, NULL }, + { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, + "Adding kctx to device failed" }, }; static void kbase_context_term_partial( @@ -184,14 +200,23 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, #if defined(CONFIG_64BIT) else kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); -#endif /* !defined(CONFIG_64BIT) */ +#endif /* defined(CONFIG_64BIT) */ for (i = 0; i < ARRAY_SIZE(context_init); i++) { - int err = context_init[i].init(kctx); + int err = 0; + + if (context_init[i].init) + err = context_init[i].init(kctx); if (err) { dev_err(kbdev->dev, "%s error = %d\n", context_init[i].err_mes, err); + + /* kctx should be freed by kbase_context_free(). + * Otherwise it will result in memory leak. + */ + WARN_ON(i == 0); + kbase_context_term_partial(kctx, i); return NULL; } @@ -217,19 +242,22 @@ void kbase_destroy_context(struct kbase_context *kctx) * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + atomic_inc(&kbdev->pm.gpu_users_waiting); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ while (kbase_pm_context_active_handle_suspend( kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_info(kbdev->dev, + dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); } +#ifdef CONFIG_MALI_ARBITER_SUPPORT + atomic_dec(&kbdev->pm.gpu_users_waiting); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_mem_pool_group_mark_dying(&kctx->mem_pools); - kbase_jd_zap_context(kctx); - flush_workqueue(kctx->jctx.job_done_wq); - kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); kbase_pm_context_idle(kbdev); diff --git a/mali_kbase/context/mali_kbase_context.c b/mali_kbase/context/mali_kbase_context.c index 5de4c6b..9ab254c 100644 --- a/mali_kbase/context/mali_kbase_context.c +++ b/mali_kbase/context/mali_kbase_context.c @@ -30,7 +30,6 @@ #include <mali_kbase_mem_linux.h> #include <mali_kbase_ctx_sched.h> #include <mali_kbase_mem_pool_group.h> -#include <tl/mali_kbase_tracepoints.h> #include <tl/mali_kbase_timeline.h> #include <mmu/mali_kbase_mmu.h> #include <context/mali_kbase_context_internal.h> @@ -169,22 +168,49 @@ int kbase_context_common_init(struct kbase_context *kctx) mutex_init(&kctx->legacy_hwcnt_lock); mutex_lock(&kctx->kbdev->kctx_list_lock); - list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); err = kbase_insert_kctx_to_process(kctx); if (err) dev_err(kctx->kbdev->dev, "(err:%d) failed to insert kctx to kbase_process\n", err); - KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id, - kctx->kbdev->gpu_props.props.raw_props.gpu_id); - KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id, - (u32)(kctx->tgid)); mutex_unlock(&kctx->kbdev->kctx_list_lock); return err; } +int kbase_context_add_to_dev_list(struct kbase_context *kctx) +{ + if (WARN_ON(!kctx)) + return -EINVAL; + + if (WARN_ON(!kctx->kbdev)) + return -EINVAL; + + mutex_lock(&kctx->kbdev->kctx_list_lock); + list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); + mutex_unlock(&kctx->kbdev->kctx_list_lock); + + kbase_timeline_post_kbase_context_create(kctx); + + return 0; +} + +void kbase_context_remove_from_dev_list(struct kbase_context *kctx) +{ + if (WARN_ON(!kctx)) + return; + + if (WARN_ON(!kctx->kbdev)) + return; + + kbase_timeline_pre_kbase_context_destroy(kctx); + + mutex_lock(&kctx->kbdev->kctx_list_lock); + list_del_init(&kctx->kctx_list_link); + mutex_unlock(&kctx->kbdev->kctx_list_lock); +} + /** * kbase_remove_kctx_from_process - remove a terminating context from * the process list. @@ -237,24 +263,9 @@ void kbase_context_common_term(struct kbase_context *kctx) mutex_lock(&kctx->kbdev->kctx_list_lock); kbase_remove_kctx_from_process(kctx); - - KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id); - - KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx); - list_del(&kctx->kctx_list_link); mutex_unlock(&kctx->kbdev->kctx_list_lock); KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); - - /* Flush the timeline stream, so the user can see the termination - * tracepoints being fired. - * The "if" statement below is for optimization. It is safe to call - * kbase_timeline_streams_flush when timeline is disabled. - */ - if (atomic_read(&kctx->kbdev->timeline_flags) != 0) - kbase_timeline_streams_flush(kctx->kbdev->timeline); - - vfree(kctx); } int kbase_context_mem_pool_group_init(struct kbase_context *kctx) diff --git a/mali_kbase/context/mali_kbase_context.h b/mali_kbase/context/mali_kbase_context.h index e4ed894..895ca4b 100644 --- a/mali_kbase/context/mali_kbase_context.h +++ b/mali_kbase/context/mali_kbase_context.h @@ -21,12 +21,21 @@ *//* SPDX-License-Identifier: GPL-2.0 */ /* * - * (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017, 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ @@ -117,25 +126,7 @@ static inline bool kbase_ctx_flag(struct kbase_context *kctx, static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, enum kbase_context_flags flag) { -#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE - /* - * Earlier kernel versions doesn't have atomic_andnot() or - * atomic_and(). atomic_clear_mask() was only available on some - * architectures and removed on arm in v3.13 on arm and arm64. - * - * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, - * when atomic_andnot() becomes available. - */ - int old, new; - - do { - old = atomic_read(&kctx->flags); - new = old & ~flag; - - } while (atomic_cmpxchg(&kctx->flags, old, new) != old); -#else atomic_andnot(flag, &kctx->flags); -#endif } /** diff --git a/mali_kbase/context/mali_kbase_context_internal.h b/mali_kbase/context/mali_kbase_context_internal.h index 818cdbe..5a901d2 100644 --- a/mali_kbase/context/mali_kbase_context_internal.h +++ b/mali_kbase/context/mali_kbase_context_internal.h @@ -20,12 +20,23 @@ * *//* SPDX-License-Identifier: GPL-2.0 */ /* - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #include <mali_kbase.h> @@ -58,3 +69,6 @@ int kbase_context_mem_alloc_page(struct kbase_context *kctx); void kbase_context_mem_pool_free(struct kbase_context *kctx); void kbase_context_sticky_resource_term(struct kbase_context *kctx); + +int kbase_context_add_to_dev_list(struct kbase_context *kctx); +void kbase_context_remove_from_dev_list(struct kbase_context *kctx); diff --git a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c index 66ac377..ad14cb5 100644 --- a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <mali_kbase.h> #include "mali_kbase_clk_rate_trace_mgr.h" #include "mali_kbase_csf_ipa_control.h" @@ -45,14 +47,23 @@ /** * Default value for the TIMER register of the IPA Control interface, - * expressed as number of clock cycles. + * expressed in milliseconds. + * + * The chosen value is a trade off between two requirements: the IPA Control + * interface should sample counters with a resolution in the order of + * milliseconds, while keeping GPU overhead as limited as possible. */ -#define TIMER_DEFAULT_VALUE_CLK_CYCLES ((u32)1000) +#define TIMER_DEFAULT_VALUE_MS ((u32)10) /* 10 milliseconds */ + +/** + * Number of timer events per second. + */ +#define TIMER_EVENTS_PER_SECOND ((u32)1000 / TIMER_DEFAULT_VALUE_MS) /** * Maximum number of loops polling the GPU before we assume the GPU has hung. */ -#define IPA_INACTIVE_MAX_LOOPS ((unsigned int)100000000) +#define IPA_INACTIVE_MAX_LOOPS ((unsigned int)8000000) /** * Number of bits used to configure a performance counter in SELECT registers. @@ -76,14 +87,19 @@ struct kbase_ipa_control_listener_data { struct kbase_device *kbdev; }; +static u32 timer_value(u32 gpu_rate) +{ + return gpu_rate / TIMER_EVENTS_PER_SECOND; +} + static int wait_status(struct kbase_device *kbdev, u32 flags) { unsigned int max_loops = IPA_INACTIVE_MAX_LOOPS; u32 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); /* - * Wait for the STATUS register to indicate that flags have been cleared, - * in case a transition is pending. + * Wait for the STATUS register to indicate that flags have been + * cleared, in case a transition is pending. */ while (--max_loops && (status & flags)) status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); @@ -285,6 +301,15 @@ kbase_ipa_control_rate_change_notify(struct kbase_clk_rate_listener *listener, } ipa_ctrl->cur_gpu_rate = clk_rate_hz; + + /* Update the timer for automatic sampling if active sessions + * are present. Counters have already been manually sampled. + */ + if (ipa_ctrl->num_active_sessions > 0) { + kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), + timer_value(ipa_ctrl->cur_gpu_rate)); + } + spin_unlock(&ipa_ctrl->lock); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -499,6 +524,14 @@ int kbase_ipa_control_register( perf_counters[i].scaling_factor; session->prfcnts[i].gpu_norm = perf_counters[i].gpu_norm; + /* Reports to this client for GPU time spent in protected mode + * should begin from the point of registration. + */ + session->last_query_time = ktime_get_ns(); + + /* Initially, no time has been spent in protected mode */ + session->protm_time = 0; + prfcnt_config->refcount++; } @@ -528,8 +561,9 @@ int kbase_ipa_control_register( COMMAND_SAMPLE); ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE); if (!ret) { - kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), - TIMER_DEFAULT_VALUE_CLK_CYCLES); + kbase_reg_write( + kbdev, IPA_CONTROL_REG(TIMER), + timer_value(ipa_ctrl->cur_gpu_rate)); } else { dev_err(kbdev->dev, "%s: failed to sample new counters", @@ -680,8 +714,7 @@ int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client, gpu_ready = kbdev->pm.backend.gpu_ready; for (i = 0; i < session->num_prfcnts; i++) { - struct kbase_ipa_control_prfcnt *prfcnt = - &session->prfcnts[i]; + struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[i]; calc_prfcnt_delta(kbdev, prfcnt, gpu_ready); /* Return all the accumulated difference */ @@ -689,14 +722,28 @@ int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client, prfcnt->accumulated_diff = 0; } + if (protected_time) { + u64 time_now = ktime_get_ns(); + + /* This is the amount of protected-mode time spent prior to + * the current protm period. + */ + *protected_time = session->protm_time; + + if (kbdev->protected_mode) { + *protected_time += + time_now - MAX(session->last_query_time, + ipa_ctrl->protm_start); + } + session->last_query_time = time_now; + session->protm_time = 0; + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); for (i = session->num_prfcnts; i < num_values; i++) values[i] = 0; - if (protected_time) - *protected_time = 0; - return 0; } KBASE_EXPORT_TEST_API(kbase_ipa_control_query); @@ -774,7 +821,7 @@ void kbase_ipa_control_handle_gpu_power_on(struct kbase_device *kbdev) /* Re-enable the timer for periodic sampling */ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), - TIMER_DEFAULT_VALUE_CLK_CYCLES); + timer_value(ipa_ctrl->cur_gpu_rate)); spin_unlock(&ipa_ctrl->lock); } @@ -832,3 +879,49 @@ void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, } KBASE_EXPORT_TEST_API(kbase_ipa_control_rate_change_notify_test); #endif + +void kbase_ipa_control_protm_entered(struct kbase_device *kbdev) +{ + struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; + + lockdep_assert_held(&kbdev->hwaccess_lock); + ipa_ctrl->protm_start = ktime_get_ns(); +} + +void kbase_ipa_control_protm_exited(struct kbase_device *kbdev) +{ + struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; + size_t i; + u64 time_now = ktime_get_ns(); + u32 status; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < ipa_ctrl->num_active_sessions; i++) { + struct kbase_ipa_control_session *session = + &ipa_ctrl->sessions[i]; + u64 protm_time = time_now - MAX(session->last_query_time, + ipa_ctrl->protm_start); + + session->protm_time += protm_time; + } + + /* Acknowledge the protected_mode bit in the IPA_CONTROL STATUS + * register + */ + status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); + if (status & STATUS_PROTECTED_MODE) { + int ret; + + /* Acknowledge the protm command */ + kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), + COMMAND_PROTECTED_ACK); + ret = wait_status(kbdev, STATUS_PROTECTED_MODE); + if (ret) { + dev_err(kbdev->dev, + "Wait for the protm ack command failed: %d", + ret); + } + } +} + diff --git a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.h b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.h index ef93ff3..441c2bd 100644 --- a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.h +++ b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_IPA_CONTROL_H_ @@ -214,4 +233,32 @@ void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, u32 clk_index, u32 clk_rate_hz); #endif /* MALI_UNIT_TEST */ +/** + * kbase_ipa_control_protm_entered - Tell IPA_CONTROL that protected mode + * has been entered. + * + * @kbdev: Pointer to kbase device. + * + * This function provides a means through which IPA_CONTROL can be informed + * that the GPU has entered protected mode. Since the GPU cannot access + * performance counters while in this mode, this information is useful as + * it implies (a) the values of these registers cannot change, so theres no + * point trying to read them, and (b) IPA_CONTROL has a means through which + * to record the duration of time the GPU is in protected mode, which can + * then be forwarded on to clients, who may wish, for example, to assume + * that the GPU was busy 100% of the time while in this mode. + */ +void kbase_ipa_control_protm_entered(struct kbase_device *kbdev); + +/** + * kbase_ipa_control_protm_exited - Tell IPA_CONTROL that protected mode + * has been exited. + * + * @kbdev: Pointer to kbase device + * + * This function provides a means through which IPA_CONTROL can be informed + * that the GPU has exited from protected mode. + */ +void kbase_ipa_control_protm_exited(struct kbase_device *kbdev); + #endif /* _KBASE_CSF_IPA_CONTROL_H_ */ diff --git a/mali_kbase/csf/mali_base_csf_kernel.h b/mali_kbase/csf/mali_base_csf_kernel.h index bc356d1..d9ea65e 100644 --- a/mali_kbase/csf/mali_base_csf_kernel.h +++ b/mali_kbase/csf/mali_base_csf_kernel.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _BASE_CSF_KERNEL_H_ @@ -320,14 +339,16 @@ enum base_kcpu_command_type { /** * enum base_queue_group_priority - Priority of a GPU Command Queue Group. - * @BASE_QUEUE_GROUP_PRIORITY_HIGH: GPU Command Queue Group is of high - * priority. - * @BASE_QUEUE_GROUP_PRIORITY_MEDIUM: GPU Command Queue Group is of medium - * priority. - * @BASE_QUEUE_GROUP_PRIORITY_LOW: GPU Command Queue Group is of low - * priority. - * @BASE_QUEUE_GROUP_PRIORITY_COUNT: Number of GPU Command Queue Group - * priority levels. + * @BASE_QUEUE_GROUP_PRIORITY_HIGH: GPU Command Queue Group is of high + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_MEDIUM: GPU Command Queue Group is of medium + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_LOW: GPU Command Queue Group is of low + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_REALTIME: GPU Command Queue Group is of real-time + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_COUNT: Number of GPU Command Queue Group + * priority levels. * * Currently this is in order of highest to lowest, but if new levels are added * then those new levels may be out of order to preserve the ABI compatibility @@ -342,6 +363,7 @@ enum base_queue_group_priority { BASE_QUEUE_GROUP_PRIORITY_HIGH = 0, BASE_QUEUE_GROUP_PRIORITY_MEDIUM, BASE_QUEUE_GROUP_PRIORITY_LOW, + BASE_QUEUE_GROUP_PRIORITY_REALTIME, BASE_QUEUE_GROUP_PRIORITY_COUNT }; diff --git a/mali_kbase/csf/mali_gpu_csf_control_registers.h b/mali_kbase/csf/mali_gpu_csf_control_registers.h index 4fff80c..9b8a9bf 100644 --- a/mali_kbase/csf/mali_gpu_csf_control_registers.h +++ b/mali_kbase/csf/mali_gpu_csf_control_registers.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/csf/mali_gpu_csf_registers.h b/mali_kbase/csf/mali_gpu_csf_registers.h index 281c9c2..d06b38d 100644 --- a/mali_kbase/csf/mali_gpu_csf_registers.h +++ b/mali_kbase/csf/mali_gpu_csf_registers.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -115,6 +134,8 @@ #define CS_STATUS_WAIT_SYNC_POINTER_LO 0x0050 /* () Sync object pointer, low word */ #define CS_STATUS_WAIT_SYNC_POINTER_HI 0x0054 /* () Sync object pointer, high word */ #define CS_STATUS_WAIT_SYNC_VALUE 0x0058 /* () Sync object test value */ +#define CS_STATUS_SCOREBOARDS 0x005C /* () Scoreboard status */ +#define CS_STATUS_BLOCKED_REASON 0x0060 /* () Blocked reason */ #define CS_FAULT 0x0080 /* () Recoverable fault information */ #define CS_FATAL 0x0084 /* () Unrecoverable fault information */ #define CS_FAULT_INFO_LO 0x0088 /* () Additional information about a recoverable fault, low word */ @@ -207,6 +228,7 @@ #define GLB_PRFCNT_JASID 0x0024 /* () Performance counter address space */ #define GLB_PRFCNT_BASE_LO 0x0028 /* () Performance counter buffer address, low word */ #define GLB_PRFCNT_BASE_HI 0x002C /* () Performance counter buffer address, high word */ +#define GLB_PRFCNT_EXTRACT 0x0030 /* () Performance counter buffer extract index */ #define GLB_PRFCNT_CONFIG 0x0040 /* () Performance counter configuration */ #define GLB_PRFCNT_CSG_SELECT 0x0044 /* () CSG performance counting enable */ #define GLB_PRFCNT_FW_EN 0x0048 /* () Performance counter enable for firmware */ @@ -228,6 +250,7 @@ #define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */ #define GLB_HALT_STATUS 0x0010 /* () Global halt status */ #define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */ +#define GLB_PRFCNT_INSERT 0x0018 /* () Performance counter buffer insert index */ #define GLB_DEBUG_FWUTF_RESULT 0x0FE0 /* () Firmware debug test result */ #define GLB_DEBUG_ACK 0x0FFC /* () Global debug acknowledge */ @@ -237,6 +260,10 @@ /* End register offsets */ /* CS_KERNEL_INPUT_BLOCK register set definitions */ +/* GLB_VERSION register */ +#define GLB_VERSION_PATCH_SHIFT (0) +#define GLB_VERSION_MINOR_SHIFT (16) +#define GLB_VERSION_MAJOR_SHIFT (24) /* CS_REQ register */ #define CS_REQ_STATE_SHIFT 0 @@ -545,6 +572,39 @@ (((reg_val) & ~CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) | \ (((value) << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) & CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK)) +/* CS_STATUS_SCOREBOARDS register */ +#define CS_STATUS_SCOREBOARDS_NONZERO_SHIFT (0) +#define CS_STATUS_SCOREBOARDS_NONZERO_MASK \ + ((0xFFFF) << CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) +#define CS_STATUS_SCOREBOARDS_NONZERO_GET(reg_val) \ + (((reg_val)&CS_STATUS_SCOREBOARDS_NONZERO_MASK) >> \ + CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) +#define CS_STATUS_SCOREBOARDS_NONZERO_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_SCOREBOARDS_NONZERO_MASK) | \ + (((value) << CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) & \ + CS_STATUS_SCOREBOARDS_NONZERO_MASK)) + +/* CS_STATUS_BLOCKED_REASON register */ +#define CS_STATUS_BLOCKED_REASON_REASON_SHIFT (0) +#define CS_STATUS_BLOCKED_REASON_REASON_MASK \ + ((0xF) << CS_STATUS_BLOCKED_REASON_REASON_SHIFT) +#define CS_STATUS_BLOCKED_REASON_REASON_GET(reg_val) \ + (((reg_val)&CS_STATUS_BLOCKED_REASON_REASON_MASK) >> \ + CS_STATUS_BLOCKED_REASON_REASON_SHIFT) +#define CS_STATUS_BLOCKED_REASON_REASON_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_BLOCKED_REASON_REASON_MASK) | \ + (((value) << CS_STATUS_BLOCKED_REASON_REASON_SHIFT) & \ + CS_STATUS_BLOCKED_REASON_REASON_MASK)) +/* CS_STATUS_BLOCKED_REASON_reason values */ +#define CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED 0x0 +#define CS_STATUS_BLOCKED_REASON_REASON_WAIT 0x1 +#define CS_STATUS_BLOCKED_REASON_REASON_PROGRESS_WAIT 0x2 +#define CS_STATUS_BLOCKED_REASON_REASON_SYNC_WAIT 0x3 +#define CS_STATUS_BLOCKED_REASON_REASON_DEFERRED 0x4 +#define CS_STATUS_BLOCKED_REASON_REASON_RESOURCE 0x5 +#define CS_STATUS_BLOCKED_REASON_REASON_FLUSH 0x6 +/* End of CS_STATUS_BLOCKED_REASON_reason values */ + /* CS_FAULT register */ #define CS_FAULT_EXCEPTION_TYPE_SHIFT 0 #define CS_FAULT_EXCEPTION_TYPE_MASK (0xFF << CS_FAULT_EXCEPTION_TYPE_SHIFT) @@ -1079,6 +1139,19 @@ #define GLB_REQ_PROTM_EXIT_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_EXIT_MASK) >> GLB_REQ_PROTM_EXIT_SHIFT) #define GLB_REQ_PROTM_EXIT_SET(reg_val, value) \ (((reg_val) & ~GLB_REQ_PROTM_EXIT_MASK) | (((value) << GLB_REQ_PROTM_EXIT_SHIFT) & GLB_REQ_PROTM_EXIT_MASK)) +#define GLB_REQ_PRFCNT_THRESHOLD_SHIFT 24 +#define GLB_REQ_PRFCNT_THRESHOLD_MASK (0x1 << GLB_REQ_PRFCNT_THRESHOLD_SHIFT) +#define GLB_REQ_PRFCNT_THRESHOLD_GET(reg_val) \ + (((reg_val)&GLB_REQ_PRFCNT_THRESHOLD_MASK) >> GLB_REQ_PRFCNT_THRESHOLD_SHIFT) +#define GLB_REQ_PRFCNT_THRESHOLD_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PRFCNT_THRESHOLD_MASK) | \ + (((value) << GLB_REQ_PRFCNT_THRESHOLD_SHIFT) & GLB_REQ_PRFCNT_THRESHOLD_MASK)) +#define GLB_REQ_PRFCNT_OVERFLOW_SHIFT 25 +#define GLB_REQ_PRFCNT_OVERFLOW_MASK (0x1 << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) +#define GLB_REQ_PRFCNT_OVERFLOW_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_OVERFLOW_MASK) >> GLB_REQ_PRFCNT_OVERFLOW_SHIFT) +#define GLB_REQ_PRFCNT_OVERFLOW_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PRFCNT_OVERFLOW_MASK) | \ + (((value) << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) & GLB_REQ_PRFCNT_OVERFLOW_MASK)) #define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30 #define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT) #define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_CSF_REQ_MASK) >> GLB_REQ_DEBUG_CSF_REQ_SHIFT) @@ -1182,6 +1255,20 @@ #define GLB_ACK_IRQ_MASK_PROTM_EXIT_SET(reg_val, value) \ (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) | \ (((value) << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK)) +#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT 24 +#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK)) +#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT 25 +#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK)) #define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT 30 #define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) #define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_GET(reg_val) \ @@ -1301,4 +1388,14 @@ #define GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1 /* End of GLB_IDLE_TIMER_TIMER_SOURCE values */ +#define CSG_STATUS_STATE (0x0018) /* CSG state status register */ +/* CSG_STATUS_STATE register */ +#define CSG_STATUS_STATE_IDLE_SHIFT (0) +#define CSG_STATUS_STATE_IDLE_MASK ((0x1) << CSG_STATUS_STATE_IDLE_SHIFT) +#define CSG_STATUS_STATE_IDLE_GET(reg_val) \ + (((reg_val)&CSG_STATUS_STATE_IDLE_MASK) >> CSG_STATUS_STATE_IDLE_SHIFT) +#define CSG_STATUS_STATE_IDLE_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_STATE_IDLE_MASK) | \ + (((value) << CSG_STATUS_STATE_IDLE_SHIFT) & CSG_STATUS_STATE_IDLE_MASK)) + #endif /* _GPU_CSF_REGISTERS_H_ */ diff --git a/mali_kbase/csf/mali_kbase_csf.c b/mali_kbase/csf/mali_kbase_csf.c index b75cae3..4c6bb06 100644 --- a/mali_kbase/csf/mali_kbase_csf.c +++ b/mali_kbase/csf/mali_kbase_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -26,11 +27,13 @@ #include "mali_kbase_csf.h" #include "backend/gpu/mali_kbase_pm_internal.h" #include <linux/export.h> +#include <linux/priority_control_manager.h> #include <linux/shmem_fs.h> #include "mali_gpu_csf_registers.h" #include "mali_kbase_csf_tiler_heap.h" #include <mmu/mali_kbase_mmu.h> #include "mali_kbase_csf_timeout.h" +#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> #define CS_REQ_EXCEPTION_MASK (CS_REQ_FAULT_MASK | CS_REQ_FATAL_MASK) #define CS_ACK_EXCEPTION_MASK (CS_ACK_FAULT_MASK | CS_ACK_FATAL_MASK) @@ -55,6 +58,19 @@ struct kbase_csf_event { void *param; }; +const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT] = { + KBASE_QUEUE_GROUP_PRIORITY_HIGH, + KBASE_QUEUE_GROUP_PRIORITY_MEDIUM, + KBASE_QUEUE_GROUP_PRIORITY_LOW, + KBASE_QUEUE_GROUP_PRIORITY_REALTIME +}; +const u8 kbasep_csf_relative_to_queue_group_priority[KBASE_QUEUE_GROUP_PRIORITY_COUNT] = { + BASE_QUEUE_GROUP_PRIORITY_REALTIME, + BASE_QUEUE_GROUP_PRIORITY_HIGH, + BASE_QUEUE_GROUP_PRIORITY_MEDIUM, + BASE_QUEUE_GROUP_PRIORITY_LOW +}; + static void put_user_pages_mmap_handle(struct kbase_context *kctx, struct kbase_queue *queue) { @@ -502,6 +518,9 @@ int kbase_csf_queue_register(struct kbase_context *kctx, queue->sync_ptr = 0; queue->sync_value = 0; + queue->sb_status = 0; + queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED; + INIT_LIST_HEAD(&queue->link); INIT_LIST_HEAD(&queue->error.link); INIT_WORK(&queue->oom_event_work, oom_event_worker); @@ -523,10 +542,21 @@ static void unbind_queue(struct kbase_context *kctx, void kbase_csf_queue_terminate(struct kbase_context *kctx, struct kbase_ioctl_cs_queue_terminate *term) { + struct kbase_device *kbdev = kctx->kbdev; struct kbase_queue *queue; + int err; + bool reset_prevented = false; - mutex_lock(&kctx->csf.lock); + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when terminating queue (buffer_addr=0x%.16llx), attempting to terminate regardless", + term->buffer_gpu_addr); + else + reset_prevented = true; + mutex_lock(&kctx->csf.lock); queue = find_queue(kctx, term->buffer_gpu_addr); if (queue) { @@ -556,9 +586,9 @@ void kbase_csf_queue_terminate(struct kbase_context *kctx, kbase_gpu_vm_unlock(kctx); spin_lock_irqsave(&kctx->csf.event_lock, flags); - /* Remove any pending command queue fatal from - * the per-context list. - */ + dev_dbg(kctx->kbdev->dev, + "Remove any pending command queue fatal from context %p\n", + (void *)kctx); list_del_init(&queue->error.link); spin_unlock_irqrestore(&kctx->csf.event_lock, flags); @@ -566,6 +596,8 @@ void kbase_csf_queue_terminate(struct kbase_context *kctx, } mutex_unlock(&kctx->csf.lock); + if (reset_prevented) + kbase_reset_gpu_allow(kbdev); } int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_bind *bind) @@ -680,7 +712,8 @@ void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, } void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, - int csi_index, int csg_nr) + int csi_index, int csg_nr, + bool ring_csg_doorbell) { struct kbase_csf_cmd_stream_group_info *ginfo; u32 value; @@ -700,18 +733,28 @@ void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, 1 << csi_index); - kbase_csf_ring_csg_doorbell(kbdev, csg_nr); + if (likely(ring_csg_doorbell)) + kbase_csf_ring_csg_doorbell(kbdev, csg_nr); } int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue_kick *kick) { + struct kbase_device *kbdev = kctx->kbdev; struct kbase_queue_group *group; struct kbase_queue *queue; int err = 0; - mutex_lock(&kctx->csf.lock); + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) { + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when kicking queue (buffer_addr=0x%.16llx)", + kick->buffer_gpu_addr); + return err; + } + mutex_lock(&kctx->csf.lock); queue = find_queue(kctx, kick->buffer_gpu_addr); if (!queue) err = -EINVAL; @@ -726,8 +769,9 @@ int kbase_csf_queue_kick(struct kbase_context *kctx, if (!err) err = kbase_csf_scheduler_queue_start(queue); - mutex_unlock(&kctx->csf.lock); + kbase_reset_gpu_allow(kbdev); + return err; } @@ -777,6 +821,7 @@ static void unbind_stopped_queue(struct kbase_context *kctx, static void unbind_queue(struct kbase_context *kctx, struct kbase_queue *queue) { + kbase_reset_gpu_assert_failed_or_prevented(kctx->kbdev); lockdep_assert_held(&kctx->csf.lock); if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { @@ -800,6 +845,20 @@ void kbase_csf_queue_unbind(struct kbase_queue *queue) kbase_csf_free_command_stream_user_pages(kctx, queue); } +void kbase_csf_queue_unbind_stopped(struct kbase_queue *queue) +{ + struct kbase_context *kctx = queue->kctx; + + lockdep_assert_held(&kctx->csf.lock); + + WARN_ON(queue->bind_state == KBASE_CSF_QUEUE_BOUND); + unbind_stopped_queue(kctx, queue); + + /* Free the resources, if allocated for this queue. */ + if (queue->reg) + kbase_csf_free_command_stream_user_pages(kctx, queue); +} + /** * find_free_group_handle() - Find a free handle for a queue group * @@ -1095,7 +1154,8 @@ static int create_queue_group(struct kbase_context *const kctx, group->tiler_max = create->in.tiler_max; group->fragment_max = create->in.fragment_max; group->compute_max = create->in.compute_max; - group->priority = create->in.priority; + group->priority = kbase_csf_priority_queue_group_priority_to_relative( + kbase_csf_priority_check(kctx->kbdev, create->in.priority)); group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; group->faulted = false; @@ -1282,6 +1342,7 @@ static void term_queue_group(struct kbase_queue_group *group) { struct kbase_context *kctx = group->kctx; + kbase_reset_gpu_assert_failed_or_prevented(kctx->kbdev); lockdep_assert_held(&kctx->csf.lock); /* Stop the group and evict it from the scheduler */ @@ -1304,6 +1365,18 @@ void kbase_csf_queue_group_terminate(struct kbase_context *kctx, u8 group_handle) { struct kbase_queue_group *group; + int err; + bool reset_prevented = false; + struct kbase_device *const kbdev = kctx->kbdev; + + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when terminating group %d, attempting to terminate regardless", + group_handle); + else + reset_prevented = true; mutex_lock(&kctx->csf.lock); @@ -1313,7 +1386,11 @@ void kbase_csf_queue_group_terminate(struct kbase_context *kctx, unsigned long flags; spin_lock_irqsave(&kctx->csf.event_lock, flags); - /* Remove any pending group fatal error from the per-context list. */ + + dev_dbg(kbdev->dev, + "Remove any pending group fatal error from context %p\n", + (void *)group->kctx); + list_del_init(&group->error_tiler_oom.link); list_del_init(&group->error_timeout.link); list_del_init(&group->error_fatal.link); @@ -1324,6 +1401,8 @@ void kbase_csf_queue_group_terminate(struct kbase_context *kctx, } mutex_unlock(&kctx->csf.lock); + if (reset_prevented) + kbase_reset_gpu_allow(kbdev); if (!group) return; @@ -1341,26 +1420,80 @@ int kbase_csf_queue_group_suspend(struct kbase_context *kctx, struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle) { - int err = -EINVAL; + struct kbase_device *const kbdev = kctx->kbdev; + int err; struct kbase_queue_group *group; + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) { + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when suspending group %d", + group_handle); + return err; + } mutex_lock(&kctx->csf.lock); group = find_queue_group(kctx, group_handle); if (group) err = kbase_csf_scheduler_group_copy_suspend_buf(group, sus_buf); + else + err = -EINVAL; mutex_unlock(&kctx->csf.lock); + kbase_reset_gpu_allow(kbdev); + return err; } -void kbase_csf_add_fatal_error_to_kctx( +/** + * add_error() - Add an error to the list of errors to report to user space + * + * @kctx: Address of a base context associated with a GPU address space. + * @error: Address of the item to be added to the context's pending error list. + * @data: Error data to be returned to userspace. + * + * Does not wake up the event queue blocking a user thread in kbase_poll. This + * is to make it more efficient to add multiple errors. + * + * The added error must not already be on the context's list of errors waiting + * to be reported (e.g. because a previous error concerning the same object has + * not yet been reported). + */ +static void add_error(struct kbase_context *const kctx, + struct kbase_csf_notification *const error, + struct base_csf_notification const *const data) +{ + unsigned long flags; + + if (WARN_ON(!kctx)) + return; + + if (WARN_ON(!error)) + return; + + if (WARN_ON(!data)) + return; + + spin_lock_irqsave(&kctx->csf.event_lock, flags); + + if (!WARN_ON(!list_empty(&error->link))) { + error->data = *data; + list_add_tail(&error->link, &kctx->csf.error_list); + dev_dbg(kctx->kbdev->dev, + "Added error %p of type %d in context %p\n", + (void *)error, data->type, (void *)kctx); + } + + spin_unlock_irqrestore(&kctx->csf.event_lock, flags); +} + +void kbase_csf_add_group_fatal_error( struct kbase_queue_group *const group, struct base_gpu_queue_group_error const *const err_payload) { struct base_csf_notification error; - unsigned long flags; if (WARN_ON(!group)) return; @@ -1378,18 +1511,7 @@ void kbase_csf_add_fatal_error_to_kctx( } }; - spin_lock_irqsave(&group->kctx->csf.event_lock, flags); - - /* If this group has already been in fatal error status, - * subsequent fatal error on this group should never take place. - */ - if (!WARN_ON(!list_empty(&group->error_fatal.link))) { - group->error_fatal.data = error; - list_add_tail(&group->error_fatal.link, - &group->kctx->csf.error_list); - } - - spin_unlock_irqrestore(&group->kctx->csf.event_lock, flags); + add_error(group->kctx, &group->error_fatal, &error); } void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, @@ -1489,6 +1611,8 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, int gr; bool reported = false; struct base_gpu_queue_group_error err_payload; + int err; + struct kbase_device *kbdev; if (WARN_ON(!kctx)) return; @@ -1496,6 +1620,14 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, if (WARN_ON(!fault)) return; + kbdev = kctx->kbdev; + err = kbase_reset_gpu_try_prevent(kbdev); + /* Regardless of whether reset failed or is currently happening, exit + * early + */ + if (err) + return; + err_payload = (struct base_gpu_queue_group_error) { .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, .payload = { @@ -1514,7 +1646,7 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, if (group && group->run_state != KBASE_CSF_GROUP_TERMINATED) { term_queue_group(group); - kbase_csf_add_fatal_error_to_kctx(group, &err_payload); + kbase_csf_add_group_fatal_error(group, &err_payload); reported = true; } } @@ -1523,11 +1655,18 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, if (reported) kbase_event_wakeup(kctx); + + kbase_reset_gpu_allow(kbdev); } void kbase_csf_ctx_term(struct kbase_context *kctx) { + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_as *as = NULL; + unsigned long flags; u32 i; + int err; + bool reset_prevented = false; /* As the kbase context is terminating, its debugfs sub-directory would * have been removed already and so would be the debugfs file created @@ -1536,8 +1675,17 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) */ kbase_csf_event_wait_remove_all(kctx); - mutex_lock(&kctx->csf.lock); + /* Wait for a GPU reset if it is happening, prevent it if not happening */ + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when terminating csf context (%d_%d), attempting to terminate regardless", + kctx->tgid, kctx->id); + else + reset_prevented = true; + mutex_lock(&kctx->csf.lock); /* Iterate through the queue groups that were not terminated by * userspace and issue the term request to firmware for them. */ @@ -1545,19 +1693,39 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) if (kctx->csf.queue_groups[i]) term_queue_group(kctx->csf.queue_groups[i]); } - mutex_unlock(&kctx->csf.lock); + if (reset_prevented) + kbase_reset_gpu_allow(kbdev); + /* Now that all queue groups have been terminated, there can be no * more OoM or timer event interrupts but there can be inflight work * items. Destroying the wq will implicitly flush those work items. */ destroy_workqueue(kctx->csf.wq); + /* Wait for the firmware error work item to also finish as it could + * be affecting this outgoing context also. + */ + flush_work(&kctx->kbdev->csf.fw_error_work); + + /* A work item to handle page_fault/bus_fault/gpu_fault could be + * pending for the outgoing context. Flush the workqueue that will + * execute that work item. + */ + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + if (kctx->as_nr != KBASEP_AS_NR_INVALID) + as = &kctx->kbdev->as[kctx->as_nr]; + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + if (as) + flush_workqueue(as->pf_wq); + mutex_lock(&kctx->csf.lock); - for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) + for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { kfree(kctx->csf.queue_groups[i]); + kctx->csf.queue_groups[i] = NULL; + } /* Iterate through the queues that were not terminated by * userspace and do the required cleanup for them. @@ -1583,12 +1751,6 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) mutex_unlock(&kctx->csf.lock); - /* Wait for the firmware error work item to also finish as it could - * be affecting this outgoing context also. Proper handling would be - * added in GPUCORE-25209. - */ - flush_work(&kctx->kbdev->csf.fw_error_work); - kbase_csf_tiler_heap_context_term(kctx); kbase_csf_kcpu_queue_context_term(kctx); kbase_csf_scheduler_context_term(kctx); @@ -1612,6 +1774,9 @@ int kbase_csf_event_wait_add(struct kbase_context *kctx, spin_lock_irqsave(&kctx->csf.event_lock, flags); list_add_tail(&event->link, &kctx->csf.event_callback_list); + dev_dbg(kctx->kbdev->dev, + "Added event handler %p with param %p\n", event, + event->param); spin_unlock_irqrestore(&kctx->csf.event_lock, flags); err = 0; @@ -1631,6 +1796,9 @@ void kbase_csf_event_wait_remove(struct kbase_context *kctx, list_for_each_entry(event, &kctx->csf.event_callback_list, link) { if ((event->callback == callback) && (event->param == param)) { list_del(&event->link); + dev_dbg(kctx->kbdev->dev, + "Removed event handler %p with param %p\n", + event, event->param); kfree(event); break; } @@ -1652,6 +1820,8 @@ bool kbase_csf_read_error(struct kbase_context *kctx, struct kbase_csf_notification, link); list_del_init(&error_data->link); *event_data = error_data->data; + dev_dbg(kctx->kbdev->dev, "Dequeued error %p in context %p\n", + (void *)error_data, (void *)kctx); } else { got_event = false; } @@ -1668,6 +1838,8 @@ bool kbase_csf_error_pending(struct kbase_context *kctx) spin_lock_irqsave(&kctx->csf.event_lock, flags); event_pended = !list_empty(&kctx->csf.error_list); + dev_dbg(kctx->kbdev->dev, "%s error is pending in context %p\n", + event_pended ? "An" : "No", (void *)kctx); spin_unlock_irqrestore(&kctx->csf.event_lock, flags); return event_pended; @@ -1678,6 +1850,10 @@ void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu) struct kbase_csf_event *event, *next_event; unsigned long flags; + dev_dbg(kctx->kbdev->dev, + "Signal event (%s GPU notify) for context %p\n", + notify_gpu ? "with" : "without", (void *)kctx); + /* First increment the signal count and wake up event thread. */ atomic_set(&kctx->event_count, 1); @@ -1703,9 +1879,12 @@ void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu) list_for_each_entry_safe( event, next_event, &kctx->csf.event_callback_list, link) { - enum kbase_csf_event_callback_action action = - event->callback(event->param); + enum kbase_csf_event_callback_action action; + dev_dbg(kctx->kbdev->dev, + "Calling event handler %p with param %p\n", + (void *)event, event->param); + action = event->callback(event->param); if (action == KBASE_CSF_EVENT_CALLBACK_REMOVE) { list_del(&event->link); kfree(event); @@ -1725,6 +1904,9 @@ void kbase_csf_event_wait_remove_all(struct kbase_context *kctx) list_for_each_entry_safe( event, next_event, &kctx->csf.event_callback_list, link) { list_del(&event->link); + dev_dbg(kctx->kbdev->dev, + "Removed event handler %p with param %p\n", + (void *)event, event->param); kfree(event); } @@ -1816,20 +1998,9 @@ static void report_tiler_oom_error(struct kbase_queue_group *group) .error_type = BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, } } } }; - struct kbase_context *kctx = group->kctx; - unsigned long flags; - - spin_lock_irqsave(&kctx->csf.event_lock, flags); - - /* Ignore this error if the previous one hasn't been reported */ - if (!WARN_ON(!list_empty(&group->error_tiler_oom.link))) { - group->error_tiler_oom.data = error; - list_add_tail(&group->error_tiler_oom.link, - &kctx->csf.error_list); - kbase_event_wakeup(kctx); - } - spin_unlock_irqrestore(&kctx->csf.event_lock, flags); + add_error(group->kctx, &group->error_tiler_oom, &error); + kbase_event_wakeup(group->kctx); } /** @@ -1915,7 +2086,7 @@ static void kbase_queue_oom_event(struct kbase_queue *const queue) return; } - kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num); + kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num, true); unlock: kbase_csf_scheduler_unlock(kbdev); } @@ -1934,6 +2105,14 @@ static void oom_event_worker(struct work_struct *data) struct kbase_queue *queue = container_of(data, struct kbase_queue, oom_event_work); struct kbase_context *kctx = queue->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + + int err = kbase_reset_gpu_try_prevent(kbdev); + /* Regardless of whether reset failed or is currently happening, exit + * early + */ + if (err) + return; mutex_lock(&kctx->csf.lock); @@ -1941,6 +2120,7 @@ static void oom_event_worker(struct work_struct *data) release_queue(queue); mutex_unlock(&kctx->csf.lock); + kbase_reset_gpu_allow(kbdev); } /** @@ -1963,25 +2143,15 @@ static void handle_progress_timer_event(struct kbase_queue_group *const group) BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, } } } }; struct kbase_context *const kctx = group->kctx; - unsigned long flags; kbase_csf_scheduler_spin_lock_assert_held(kctx->kbdev); - spin_lock_irqsave(&kctx->csf.event_lock, flags); - dev_warn(kctx->kbdev->dev, "Notify the event notification thread, forward progress timeout (%llu cycles)\n", kbase_csf_timeout_get(kctx->kbdev)); - /* Ignore this error if the previous one hasn't been reported */ - if (!WARN_ON(!list_empty(&group->error_timeout.link))) { - group->error_timeout.data = error; - list_add_tail(&group->error_timeout.link, - &kctx->csf.error_list); - kbase_event_wakeup(kctx); - } - - spin_unlock_irqrestore(&kctx->csf.event_lock, flags); + add_error(kctx, &group->error_timeout, &error); + kbase_event_wakeup(kctx); } /** @@ -2011,7 +2181,7 @@ static void protm_event_worker(struct work_struct *data) * */ static void -handle_fault_event(struct kbase_queue const *const queue, +handle_fault_event(struct kbase_queue *const queue, struct kbase_csf_cmd_stream_info const *const stream) { const u32 cs_fault = kbase_csf_firmware_cs_output(stream, CS_FAULT); @@ -2036,20 +2206,21 @@ handle_fault_event(struct kbase_queue const *const queue, queue->csi_index, cs_fault_exception_type, kbase_gpu_exception_name(cs_fault_exception_type), cs_fault_exception_data, cs_fault_info_exception_data); + + /* TODO GPUCORE-26291: We've'identified an issue with faulted CSIs not + * making progress in some cases. Until the issue is resolved, + * RESOURCE_EVICTION_TIMEOUT error shall be treated as a fatal error + * to give userspace a chance to terminate the group. This is intended + * to be a temporary workaround. + */ + if (cs_fault_exception_type == + CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT) + kbase_csf_add_queue_fatal_error( + queue, GPU_EXCEPTION_TYPE_SW_FAULT_2, 0); } -/** - * report_queue_fatal_error - Report queue fatal error to user space - * - * @queue: Pointer to queue for which fatal event was received. - * @cs_fatal: Fault information - * @cs_fatal_info: Additional fault information - * - * If a queue has already been in fatal error status, - * subsequent fatal error on the queue should never take place. - */ -static void report_queue_fatal_error(struct kbase_queue *const queue, - u32 cs_fatal, u64 cs_fatal_info) +void kbase_csf_add_queue_fatal_error(struct kbase_queue *const queue, + u32 cs_fatal, u64 cs_fatal_info) { struct base_csf_notification error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, @@ -2070,22 +2241,10 @@ static void report_queue_fatal_error(struct kbase_queue *const queue, } } }; - unsigned long flags; kbase_csf_scheduler_spin_lock_assert_held(queue->kctx->kbdev); - - spin_lock_irqsave(&queue->kctx->csf.event_lock, flags); - - /* If a queue has already been in fatal error status, - * subsequent fatal error on the queue should never take place. - */ - if (!WARN_ON(!list_empty(&queue->error.link))) { - queue->error.data = error; - list_add_tail(&queue->error.link, &queue->kctx->csf.error_list); - kbase_event_wakeup(queue->kctx); - } - - spin_unlock_irqrestore(&queue->kctx->csf.event_lock, flags); + add_error(queue->kctx, &queue->error, &error); + kbase_event_wakeup(queue->kctx); } /** @@ -2134,7 +2293,7 @@ handle_fatal_event(struct kbase_queue *const queue, CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR) *fw_error = true; else - report_queue_fatal_error(queue, cs_fatal, cs_fatal_info); + kbase_csf_add_queue_fatal_error(queue, cs_fatal, cs_fatal_info); } /** @@ -2171,7 +2330,7 @@ static void handle_queue_exception_event(struct kbase_queue *const queue, handle_fault_event(queue, stream); kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK); - kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num); + kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num, true); } if (internal_fw_error) @@ -2354,7 +2513,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, /* Check if the scheduling tick can be advanced */ if (kbase_csf_scheduler_all_csgs_idle(kbdev) && !scheduler->gpu_idle_fw_timer_enabled) { - mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); + kbase_csf_scheduler_advance_tick_nolock(kbdev); } } @@ -2372,6 +2531,90 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, process_cs_interrupts(group, ginfo, irqreq, irqack); } +static void process_prfcnt_interrupts(struct kbase_device *kbdev, u32 glb_req, + u32 glb_ack, unsigned long *flags) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + + lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); + + /* Process PRFCNT_SAMPLE interrupt. */ + if (kbdev->csf.hwcnt.request_pending && + ((glb_req & GLB_REQ_PRFCNT_SAMPLE_MASK) == + (glb_ack & GLB_REQ_PRFCNT_SAMPLE_MASK))) { + kbdev->csf.hwcnt.request_pending = false; + kbase_csf_scheduler_spin_unlock(kbdev, *flags); + + dev_dbg(kbdev->dev, "PRFCNT_SAMPLE done interrupt received."); + + kbase_hwcnt_backend_csf_on_prfcnt_sample( + &kbdev->hwcnt_gpu_iface); + + kbase_csf_scheduler_spin_lock(kbdev, flags); + } + + /* Process PRFCNT_ENABLE interrupt. */ + if (kbdev->csf.hwcnt.enable_pending && + ((glb_req & GLB_REQ_PRFCNT_ENABLE_MASK) == + (glb_ack & GLB_REQ_PRFCNT_ENABLE_MASK))) { + kbdev->csf.hwcnt.enable_pending = false; + kbase_csf_scheduler_spin_unlock(kbdev, *flags); + + dev_dbg(kbdev->dev, + "PRFCNT_ENABLE status changed interrupt received."); + + if (glb_ack & GLB_REQ_PRFCNT_ENABLE_MASK) { + kbase_hwcnt_backend_csf_on_prfcnt_enable( + &kbdev->hwcnt_gpu_iface); + } else { + kbase_hwcnt_backend_csf_on_prfcnt_disable( + &kbdev->hwcnt_gpu_iface); + } + + kbase_csf_scheduler_spin_lock(kbdev, flags); + } + + /* Process PRFCNT_THRESHOLD interrupt. */ + if ((glb_req ^ glb_ack) & GLB_REQ_PRFCNT_THRESHOLD_MASK) { + kbase_csf_scheduler_spin_unlock(kbdev, *flags); + dev_dbg(kbdev->dev, "PRFCNT_THRESHOLD interrupt received."); + + kbase_hwcnt_backend_csf_on_prfcnt_threshold( + &kbdev->hwcnt_gpu_iface); + + kbase_csf_scheduler_spin_lock(kbdev, flags); + + /* Set the GLB_REQ.PRFCNT_THRESHOLD flag back to + * the same value as GLB_ACK.PRFCNT_THRESHOLD + * flag in order to enable reporting of another + * PRFCNT_THRESHOLD event. + */ + kbase_csf_firmware_global_input_mask( + global_iface, GLB_REQ, glb_ack, + GLB_REQ_PRFCNT_THRESHOLD_MASK); + } + + /* Process PRFCNT_OVERFLOW interrupt. */ + if ((glb_req ^ glb_ack) & GLB_REQ_PRFCNT_OVERFLOW_MASK) { + kbase_csf_scheduler_spin_unlock(kbdev, *flags); + dev_dbg(kbdev->dev, "PRFCNT_OVERFLOW interrupt received."); + + kbase_hwcnt_backend_csf_on_prfcnt_overflow( + &kbdev->hwcnt_gpu_iface); + + kbase_csf_scheduler_spin_lock(kbdev, flags); + /* Set the GLB_REQ.PRFCNT_OVERFLOW flag back to + * the same value as GLB_ACK.PRFCNT_OVERFLOW + * flag in order to enable reporting of another + * PRFCNT_OVERFLOW event. + */ + kbase_csf_firmware_global_input_mask( + global_iface, GLB_REQ, glb_ack, + GLB_REQ_PRFCNT_OVERFLOW_MASK); + } +} + void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) { unsigned long flags; @@ -2409,6 +2652,9 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) scheduler->active_protm_grp = NULL; KBASE_KTRACE_ADD(kbdev, SCHEDULER_EXIT_PROTM, NULL, 0u); kbdev->protected_mode = false; + kbase_ipa_control_protm_exited(kbdev); + kbase_hwcnt_backend_csf_protm_exited( + &kbdev->hwcnt_gpu_iface); } /* Handle IDLE Hysteresis notification event */ @@ -2419,18 +2665,22 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) GLB_REQ_IDLE_EVENT_MASK); if (!atomic_read(&scheduler->non_idle_offslot_grps)) { - mod_delayed_work(system_highpri_wq, - &scheduler->gpu_idle_work, 0); + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + queue_work(system_highpri_wq, + &scheduler->gpu_idle_work); } else { /* Advance the scheduling tick to get * the non-idle suspended groups loaded * soon. */ - mod_delayed_work(scheduler->wq, - &scheduler->tick_work, 0); + kbase_csf_scheduler_advance_tick_nolock( + kbdev); } } + process_prfcnt_interrupts(kbdev, glb_req, glb_ack, + &flags); + kbase_csf_scheduler_spin_unlock(kbdev, flags); /* Invoke the MCU state machine as a state transition @@ -2539,3 +2789,18 @@ int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev) return 0; } + +u8 kbase_csf_priority_check(struct kbase_device *kbdev, u8 req_priority) +{ + struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; + u8 out_priority = req_priority; + + if (pcm_device) { + req_priority = kbase_csf_priority_queue_group_priority_to_relative(req_priority); + out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, req_priority); + out_priority = kbase_csf_priority_relative_to_queue_group_priority(out_priority); + } + + return out_priority; +} + diff --git a/mali_kbase/csf/mali_kbase_csf.h b/mali_kbase/csf/mali_kbase_csf.h index 44bc131..7eed90d 100644 --- a/mali_kbase/csf/mali_kbase_csf.h +++ b/mali_kbase/csf/mali_kbase_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_H_ @@ -40,9 +59,6 @@ */ #define KBASEP_USER_DB_NR_INVALID ((s8)-1) -/* Waiting timeout for global request completion acknowledgment */ -#define GLB_REQ_WAIT_TIMEOUT_MS (300) /* 300 milliseconds */ - #define FIRMWARE_PING_INTERVAL_MS (2000) /* 2 seconds */ #define FIRMWARE_IDLE_HYSTERESIS_TIME_MS (10) /* Default 10 milliseconds */ @@ -267,6 +283,16 @@ int kbase_csf_queue_bind(struct kbase_context *kctx, void kbase_csf_queue_unbind(struct kbase_queue *queue); /** + * kbase_csf_queue_unbind_stopped - Unbind a GPU command queue in the case + * where it was never started. + * @queue: Pointer to queue to be unbound. + * + * Variant of kbase_csf_queue_unbind() for use on error paths for cleaning up + * queues that failed to fully bind. + */ +void kbase_csf_queue_unbind_stopped(struct kbase_queue *queue); + +/** * kbase_csf_queue_kick - Schedule a GPU command queue on the firmware * * @kctx: The kbase context. @@ -350,16 +376,29 @@ int kbase_csf_queue_group_suspend(struct kbase_context *kctx, struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle); /** - * kbase_csf_add_fatal_error_to_kctx - Add a fatal error to per-ctx error list. + * kbase_csf_add_group_fatal_error - Report a fatal group error to userspace * * @group: GPU command queue group. * @err_payload: Error payload to report. */ -void kbase_csf_add_fatal_error_to_kctx( +void kbase_csf_add_group_fatal_error( struct kbase_queue_group *const group, struct base_gpu_queue_group_error const *const err_payload); /** + * kbase_csf_add_queue_fatal_error - Report a fatal queue error to userspace + * + * @queue: Pointer to queue for which fatal event was received. + * @cs_fatal: Fault information + * @cs_fatal_info: Additional fault information + * + * If a queue has already been in fatal error status, + * subsequent fatal error on the queue should never take place. + */ +void kbase_csf_add_queue_fatal_error(struct kbase_queue *const queue, + u32 cs_fatal, u64 cs_fatal_info); + +/** * kbase_csf_interrupt - Handle interrupts issued by CSF firmware. * * @kbdev: The kbase device to handle an IRQ for @@ -448,9 +487,16 @@ void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, * @csi_index: ID of the CSI assigned to the GPU queue. * @csg_nr: Index of the CSG slot assigned to the queue * group to which the GPU queue is bound. + * @ring_csg_doorbell: Flag to indicate if the CSG doorbell needs to be rung + * after updating the CSG_DB_REQ. So if this flag is false + * the doorbell interrupt will not be sent to FW. + * The flag is supposed be false only when the input page + * for bound GPU queues is programmed at the time of + * starting/resuming the group on a CSG slot. */ void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, - int csi_index, int csg_nr); + int csi_index, int csg_nr, + bool ring_csg_doorbell); /** * kbase_csf_ring_cs_user_doorbell - ring the user doorbell allocated for a @@ -482,4 +528,54 @@ void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, struct kbase_context *kctx); +/** + * kbase_csf_priority_check - Check the priority requested + * + * @kbdev: Device pointer + * @req_priority: Requested priority + * + * This will determine whether the requested priority can be satisfied. + * + * Return: The same or lower priority than requested. + */ +u8 kbase_csf_priority_check(struct kbase_device *kbdev, u8 req_priority); + +extern const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT]; +extern const u8 kbasep_csf_relative_to_queue_group_priority[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; + +/** + * kbase_csf_priority_relative_to_queue_group_priority - Convert relative to base priority + * + * @priority: kbase relative priority + * + * This will convert the monotonically increasing realtive priority to the + * fixed base priority list. + * + * Return: base_queue_group_priority priority. + */ +static inline u8 kbase_csf_priority_relative_to_queue_group_priority(u8 priority) +{ + if (priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT) + priority = KBASE_QUEUE_GROUP_PRIORITY_LOW; + return kbasep_csf_relative_to_queue_group_priority[priority]; +} + +/** + * kbase_csf_priority_queue_group_priority_to_relative - Convert base priority to relative + * + * @priority: base_queue_group_priority priority + * + * This will convert the fixed base priority list to monotonically increasing realtive priority. + * + * Return: kbase relative priority. + */ +static inline u8 kbase_csf_priority_queue_group_priority_to_relative(u8 priority) +{ + /* Apply low priority in case of invalid priority */ + if (priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT) + priority = BASE_QUEUE_GROUP_PRIORITY_LOW; + return kbasep_csf_queue_group_priority_to_relative[priority]; +} + + #endif /* _KBASE_CSF_H_ */ diff --git a/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.c b/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.c index 3acceeb..166c1b4 100644 --- a/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.c +++ b/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.h b/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.h index 71309bf..4ba0b8c 100644 --- a/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.h +++ b/mali_kbase/csf/mali_kbase_csf_cpu_queue_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_CPU_QUEUE_DEBUGFS_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_csg_debugfs.c b/mali_kbase/csf/mali_kbase_csf_csg_debugfs.c index e54c724..86f359a 100644 --- a/mali_kbase/csf/mali_kbase_csf_csg_debugfs.c +++ b/mali_kbase/csf/mali_kbase_csf_csg_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -29,12 +30,37 @@ #ifdef CONFIG_DEBUG_FS #include "mali_kbase_csf_tl_reader.h" +/** + * blocked_reason_to_string() - Convert blocking reason id to a string + * + * @reason_id: blocked_reason + * + * Return: Suitable string + */ +static const char *blocked_reason_to_string(u32 reason_id) +{ + /* possible blocking reasons of a cs */ + static const char *const cs_blocked_reason[] = { + [CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED] = "UNBLOCKED", + [CS_STATUS_BLOCKED_REASON_REASON_WAIT] = "WAIT", + [CS_STATUS_BLOCKED_REASON_REASON_PROGRESS_WAIT] = + "PROGRESS_WAIT", + [CS_STATUS_BLOCKED_REASON_REASON_SYNC_WAIT] = "SYNC_WAIT", + [CS_STATUS_BLOCKED_REASON_REASON_DEFERRED] = "DEFERRED", + [CS_STATUS_BLOCKED_REASON_REASON_RESOURCE] = "RESOURCE", + [CS_STATUS_BLOCKED_REASON_REASON_FLUSH] = "FLUSH" + }; + + if (WARN_ON(reason_id >= ARRAY_SIZE(cs_blocked_reason))) + return "UNKNOWN_BLOCKED_REASON_ID"; + + return cs_blocked_reason[reason_id]; +} + static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait( - struct seq_file *file, - u32 wait_status, - u32 wait_sync_value, - u64 wait_sync_live_value, - u64 wait_sync_pointer) + struct seq_file *file, u32 wait_status, u32 wait_sync_value, + u64 wait_sync_live_value, u64 wait_sync_pointer, u32 sb_status, + u32 blocked_reason) { #define WAITING "Waiting" #define NOT_WAITING "Not waiting" @@ -56,6 +82,11 @@ static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait( seq_printf(file, "SYNC_POINTER: 0x%llx\n", wait_sync_pointer); seq_printf(file, "SYNC_VALUE: %d\n", wait_sync_value); seq_printf(file, "SYNC_LIVE_VALUE: 0x%016llx\n", wait_sync_live_value); + seq_printf(file, "SB_STATUS: %u\n", + CS_STATUS_SCOREBOARDS_NONZERO_GET(sb_status)); + seq_printf(file, "BLOCKED_REASON: %s\n", + blocked_reason_to_string(CS_STATUS_BLOCKED_REASON_REASON_GET( + blocked_reason))); } /** @@ -74,6 +105,8 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, u32 cs_active; u64 wait_sync_pointer; u32 wait_status, wait_sync_value; + u32 sb_status; + u32 blocked_reason; struct kbase_vmap_struct *mapping; u64 *evt; u64 wait_sync_live_value; @@ -109,6 +142,8 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, wait_status = queue->status_wait; wait_sync_value = queue->sync_value; wait_sync_pointer = queue->sync_ptr; + sb_status = queue->sb_status; + blocked_reason = queue->blocked_reason; evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); if (evt) { @@ -120,7 +155,8 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, kbasep_csf_scheduler_dump_active_queue_cs_status_wait( file, wait_status, wait_sync_value, - wait_sync_live_value, wait_sync_pointer); + wait_sync_live_value, wait_sync_pointer, + sb_status, blocked_reason); } } else { struct kbase_device const *const kbdev = @@ -161,6 +197,11 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, wait_sync_pointer |= (u64)kbase_csf_firmware_cs_output(stream, CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; + sb_status = kbase_csf_firmware_cs_output(stream, + CS_STATUS_SCOREBOARDS); + blocked_reason = kbase_csf_firmware_cs_output( + stream, CS_STATUS_BLOCKED_REASON); + evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); if (evt) { wait_sync_live_value = evt[0]; @@ -171,7 +212,8 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, kbasep_csf_scheduler_dump_active_queue_cs_status_wait( file, wait_status, wait_sync_value, - wait_sync_live_value, wait_sync_pointer); + wait_sync_live_value, wait_sync_pointer, sb_status, + blocked_reason); } seq_puts(file, "\n"); diff --git a/mali_kbase/csf/mali_kbase_csf_csg_debugfs.h b/mali_kbase/csf/mali_kbase_csf_csg_debugfs.h index c2e99d3..68e9ce0 100644 --- a/mali_kbase/csf/mali_kbase_csf_csg_debugfs.h +++ b/mali_kbase/csf/mali_kbase_csf_csg_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_CSG_DEBUGFS_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_defs.h b/mali_kbase/csf/mali_kbase_csf_defs.h index 3939238..968895f 100644 --- a/mali_kbase/csf/mali_kbase_csf_defs.h +++ b/mali_kbase/csf/mali_kbase_csf_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* Definitions (types, defines, etcs) common to the CSF. @@ -66,18 +85,36 @@ enum kbase_csf_queue_bind_state { * enum kbase_csf_reset_gpu_state - state of the gpu reset * * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending + * + * @KBASE_CSF_RESET_GPU_PREPARED: Set when kbase_prepare_to_reset_gpu() has + * been called. This is just for debugging checks to encourage callers to call + * kbase_prepare_to_reset_gpu() before kbase_reset_gpu(). + * + * @KBASE_CSF_RESET_GPU_COMMITTED: Set when the GPU reset process has been + * committed and so will definitely happen, but the procedure to reset the GPU + * has not yet begun. Other threads must finish accessing the HW before we + * reach %KBASE_CSF_RESET_GPU_HAPPENING. + * * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring - * @KBASE_CSF_RESET_GPU_SILENT: Set when the GPU reset process is occurring, - * used when resetting the GPU as part of normal behavior (e.g. when exiting - * protected mode). + * (silent or otherwise), and is actively accessing the HW. Any changes to the + * HW in other threads might get lost, overridden, or corrupted. + * + * @KBASE_CSF_RESET_GPU_COMMITTED_SILENT: Set when the GPU reset process has + * been committed but has not started happening. This is used when resetting + * the GPU as part of normal behavior (e.g. when exiting protected mode). + * Other threads must finish accessing the HW before we reach + * %KBASE_CSF_RESET_GPU_HAPPENING. + * * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the * GPU reset process. No more work could then be executed on GPU, unloading * the Driver module is the only option. */ enum kbase_csf_reset_gpu_state { KBASE_CSF_RESET_GPU_NOT_PENDING, + KBASE_CSF_RESET_GPU_PREPARED, + KBASE_CSF_RESET_GPU_COMMITTED, KBASE_CSF_RESET_GPU_HAPPENING, - KBASE_CSF_RESET_GPU_SILENT, + KBASE_CSF_RESET_GPU_COMMITTED_SILENT, KBASE_CSF_RESET_GPU_FAILED, }; @@ -202,6 +239,24 @@ enum kbase_csf_scheduler_state { }; /** + * enum kbase_queue_group_priority - Kbase internal relative priority list. + * + * @KBASE_QUEUE_GROUP_PRIORITY_REALTIME: The realtime queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_HIGH: The high queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_MEDIUM: The medium queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_LOW: The low queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_COUNT: The number of priority levels. + */ +enum kbase_queue_group_priority { + KBASE_QUEUE_GROUP_PRIORITY_REALTIME = 0, + KBASE_QUEUE_GROUP_PRIORITY_HIGH, + KBASE_QUEUE_GROUP_PRIORITY_MEDIUM, + KBASE_QUEUE_GROUP_PRIORITY_LOW, + KBASE_QUEUE_GROUP_PRIORITY_COUNT +}; + + +/** * struct kbase_csf_notification - Event or error generated as part of command * queue execution * @@ -248,7 +303,7 @@ struct kbase_csf_notification { * @base_addr: Base address of the CS buffer. * @size: Size of the CS buffer. * @priority: Priority of this queue within the group. - * @bind_state: Bind state of the queue. + * @bind_state: Bind state of the queue as enum @kbase_csf_queue_bind_state * @csi_index: The ID of the assigned CS hardware interface. * @enabled: Indicating whether the CS is running, or not. * @status_wait: Value of CS_STATUS_WAIT register of the CS will @@ -267,6 +322,10 @@ struct kbase_csf_notification { * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value * tested against the synchronization object. * Valid only when @status_wait is set. + * @sb_status: Value indicates which of the scoreboard entries in the queue + * are non-zero + * @blocked_reason: Value shows if the queue is blocked, and if so, + * the reason why it is blocked * @error: GPU command queue fatal information to pass to user space. */ struct kbase_queue { @@ -285,12 +344,14 @@ struct kbase_queue { u64 base_addr; u32 size; u8 priority; - u8 bind_state; s8 csi_index; + enum kbase_csf_queue_bind_state bind_state; bool enabled; u32 status_wait; u64 sync_ptr; u32 sync_value; + u32 sb_status; + u32 blocked_reason; struct kbase_csf_notification error; }; @@ -484,14 +545,17 @@ struct kbase_csf_heap_context_allocator { * @kbase_context. It is not the same as a heap context structure allocated by * the kernel for use by the firmware. * - * @lock: Lock preventing concurrent access to the tiler heaps. - * @list: List of tiler heaps. - * @ctx_alloc: Allocator for heap context structures. + * @lock: Lock preventing concurrent access to the tiler heaps. + * @list: List of tiler heaps. + * @ctx_alloc: Allocator for heap context structures. + * @nr_of_heaps: Total number of tiler heaps that were added during the + * life time of the context. */ struct kbase_csf_tiler_heap_context { struct mutex lock; struct list_head list; struct kbase_csf_heap_context_allocator ctx_alloc; + u64 nr_of_heaps; }; /** @@ -499,7 +563,7 @@ struct kbase_csf_tiler_heap_context { * context for a GPU address space. * * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, - * one per queue group priority level. + * one per queue group relative-priority level. * @num_runnable_grps: Total number of runnable groups across all priority * levels in @runnable_groups. * @idle_wait_groups: A list of GPU command queue groups in which all enabled @@ -517,7 +581,7 @@ struct kbase_csf_tiler_heap_context { * 'groups_to_schedule' list of scheduler instance. */ struct kbase_csf_scheduler_context { - struct list_head runnable_groups[BASE_QUEUE_GROUP_PRIORITY_COUNT]; + struct list_head runnable_groups[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; u32 num_runnable_grps; struct list_head idle_wait_groups; u32 num_idle_wait_grps; @@ -606,21 +670,26 @@ struct kbase_csf_context { * @workq: Workqueue to execute the GPU reset work item @work. * @work: Work item for performing the GPU reset. * @wait: Wait queue used to wait for the GPU reset completion. + * @sem: RW Semaphore to ensure no other thread attempts to use the + * GPU whilst a reset is in process. Unlike traditional + * semaphores and wait queues, this allows Linux's lockdep + * mechanism to check for deadlocks involving reset waits. * @state: Tracks if the GPU reset is in progress or not. + * The state is represented by enum @kbase_csf_reset_gpu_state. */ struct kbase_csf_reset_gpu { struct workqueue_struct *workq; struct work_struct work; wait_queue_head_t wait; + struct rw_semaphore sem; atomic_t state; }; /** * struct kbase_csf_csg_slot - Object containing members for tracking the state * of CSG slots. - * @resident_group: pointer to the queue group that is resident on the - * CSG slot. - * @state: state of the slot as per enum kbase_csf_csg_slot_state. + * @resident_group: pointer to the queue group that is resident on the CSG slot. + * @state: state of the slot as per enum @kbase_csf_csg_slot_state. * @trigger_jiffies: value of jiffies when change in slot state is recorded. * @priority: dynamic priority assigned to CSG slot. */ @@ -689,9 +758,12 @@ struct kbase_csf_csg_slot { * then it will only perform scheduling under the * influence of external factors e.g., IRQs, IOCTLs. * @wq: Dedicated workqueue to execute the @tick_work. - * @tick_work: Work item that would perform the schedule on tick - * operation to implement the time slice based - * scheduling. + * @tick_timer: High-resolution timer employed to schedule tick + * workqueue items (kernel-provided delayed_work + * items do not use hrtimer and for some reason do + * not provide sufficiently reliable periodicity). + * @tick_work: Work item that performs the "schedule on tick" + * operation to implement timeslice-based scheduling. * @tock_work: Work item that would perform the schedule on tock * operation to implement the asynchronous scheduling. * @ping_work: Work item that would ping the firmware at regular @@ -733,6 +805,14 @@ struct kbase_csf_csg_slot { * becomes 0. It is used to enable the power up of MCU * after GPU and L2 cache have been powered up. So when * this count is zero, MCU will not be powered up. + * @csg_scheduling_period_ms: Duration of Scheduling tick in milliseconds. + * @tick_timer_active: Indicates whether the @tick_timer is effectively + * active or not, as the callback function of + * @tick_timer will enqueue @tick_work only if this + * flag is true. This is mainly useful for the case + * when scheduling tick needs to be advanced from + * interrupt context, without actually deactivating + * the @tick_timer first and then enqueing @tick_work. */ struct kbase_csf_scheduler { struct mutex lock; @@ -754,7 +834,8 @@ struct kbase_csf_scheduler { unsigned long last_schedule; bool timer_enabled; struct workqueue_struct *wq; - struct delayed_work tick_work; + struct hrtimer tick_timer; + struct work_struct tick_work; struct delayed_work tock_work; struct delayed_work ping_work; struct kbase_context *top_ctx; @@ -763,11 +844,13 @@ struct kbase_csf_scheduler { bool tock_pending_request; struct kbase_queue_group *active_protm_grp; bool gpu_idle_fw_timer_enabled; - struct delayed_work gpu_idle_work; + struct work_struct gpu_idle_work; atomic_t non_idle_offslot_grps; u32 non_idle_scanout_grps; bool apply_async_protm; u32 pm_active_count; + unsigned int csg_scheduling_period_ms; + bool tick_timer_active; }; /** @@ -784,18 +867,20 @@ struct kbase_csf_scheduler { GLB_PROGRESS_TIMER_TIMEOUT_SCALE) /** - * Number of GPU cycles per unit of the global poweroff timeout. + * Default GLB_PWROFF_TIMER_TIMEOUT value in unit of micro-seconds. */ -#define GLB_PWROFF_TIMER_TIMEOUT_SCALE ((u64)1024) +#define DEFAULT_GLB_PWROFF_TIMEOUT_US (800) /** - * Minimum number of GPU cycles for which shader cores must be idle before they - * are powered off. - * Value chosen is equivalent to the hysteresis delay used in the shader cores - * state machine of JM GPUs, which is ~800 micro seconds. It is assumed the GPU - * is usually clocked at ~500 MHZ. + * In typical operations, the management of the shader core power transitions + * is delegated to the MCU/firmware. However, if the host driver is configured + * to take direct control, one needs to disable the MCU firmware GLB_PWROFF + * timer. */ -#define DEFAULT_GLB_PWROFF_TIMER_TIMEOUT ((u64)800 * 500) +#define DISABLE_GLB_PWROFF_TIMER (0) + +/* Index of the GPU_ACTIVE counter within the CSHW counter block */ +#define GPU_ACTIVE_CNT_IDX (4) /** * Maximum number of sessions that can be managed by the IPA Control component. @@ -863,14 +948,18 @@ struct kbase_ipa_control_prfcnt { /** * struct kbase_ipa_control_session - Session for an IPA Control client * - * @prfcnts: Sessions for individual performance counters. - * @num_prfcnts: Number of performance counters. - * @active: Status of the session. + * @prfcnts: Sessions for individual performance counters. + * @num_prfcnts: Number of performance counters. + * @active: Indicates whether this slot is in use or not + * @last_query_time: Time of last query, in ns + * @protm_time: Amount of time (in ns) that GPU has been in protected */ struct kbase_ipa_control_session { struct kbase_ipa_control_prfcnt prfcnts[KBASE_IPA_CONTROL_MAX_COUNTERS]; size_t num_prfcnts; bool active; + u64 last_query_time; + u64 protm_time; }; /** @@ -918,6 +1007,7 @@ struct kbase_ipa_control_prfcnt_block { * @cur_gpu_rate: Current GPU top-level operating frequency, in Hz. * @rtm_listener_data: Private data for allocating a GPU frequency change * listener. + * @protm_start: Time (in ns) at which the GPU entered protected mode */ struct kbase_ipa_control { struct kbase_ipa_control_prfcnt_block blocks[KBASE_IPA_CORE_TYPE_NUM]; @@ -927,6 +1017,21 @@ struct kbase_ipa_control { void *rtm_listener_data; size_t num_active_sessions; u32 cur_gpu_rate; + u64 protm_start; +}; + +/** + * struct kbase_csf_hwcnt - Object containing members for handling the dump of + * HW counters. + * + * @request_pending: Flag set when HWC requested and used for HWC sample + * done interrupt. + * @enable_pending: Flag set when HWC enable status change and used for + * enable done interrupt. + */ +struct kbase_csf_hwcnt { + bool request_pending; + bool enable_pending; }; /** @@ -1003,6 +1108,11 @@ struct kbase_ipa_control { * in GPU reset has completed. * @firmware_reload_needed: Flag for indicating that the firmware needs to be * reloaded as part of the GPU reset action. + * @firmware_hctl_core_pwr: Flag for indicating that the host diver is in + * charge of the shader core's power transitions, and + * the mcu_core_pwroff timeout feature is disabled + * (i.e. configured 0 in the register field). If + * false, the control is delegated to the MCU. * @firmware_reload_work: Work item for facilitating the procedural actions * on reloading the firmware. * @glb_init_request_pending: Flag to indicate that Global requests have been @@ -1011,12 +1121,26 @@ struct kbase_ipa_control { * @fw_error_work: Work item for handling the firmware internal error * fatal event. * @ipa_control: IPA Control component manager. + * @mcu_core_pwroff_dur_us: Sysfs attribute for the glb_pwroff timeout input + * in unit of micro-seconds. The firmware does not use + * it directly. + * @mcu_core_pwroff_dur_count: The counterpart of the glb_pwroff timeout input + * in interface required format, ready to be used + * directly in the firmware. + * @mcu_core_pwroff_reg_shadow: The actual value that has been programed into + * the glb_pwoff register. This is separated from + * the @p mcu_core_pwroff_dur_count as an update + * to the latter is asynchronous. * @gpu_idle_hysteresis_ms: Sysfs attribute for the idle hysteresis time * window in unit of ms. The firmware does not use it * directly. * @gpu_idle_dur_count: The counterpart of the hysteresis time window in * interface required format, ready to be used * directly in the firmware. + * @fw_timeout_ms: Timeout value (in milliseconds) used when waiting + * for any request sent to the firmware. + * @hwcnt: Contain members required for handling the dump of + * HW counters. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1043,12 +1167,18 @@ struct kbase_csf_device { bool firmware_inited; bool firmware_reloaded; bool firmware_reload_needed; + bool firmware_hctl_core_pwr; struct work_struct firmware_reload_work; bool glb_init_request_pending; struct work_struct fw_error_work; struct kbase_ipa_control ipa_control; + u32 mcu_core_pwroff_dur_us; + u32 mcu_core_pwroff_dur_count; + u32 mcu_core_pwroff_reg_shadow; u32 gpu_idle_hysteresis_ms; u32 gpu_idle_dur_count; + unsigned int fw_timeout_ms; + struct kbase_csf_hwcnt hwcnt; }; /** diff --git a/mali_kbase/csf/mali_kbase_csf_firmware.c b/mali_kbase/csf/mali_kbase_csf_firmware.c index e2067d2..22f162f 100644 --- a/mali_kbase/csf/mali_kbase_csf_firmware.c +++ b/mali_kbase/csf/mali_kbase_csf_firmware.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -33,6 +34,7 @@ #include "tl/mali_kbase_timeline_priv.h" #include "mali_kbase_csf_tl_reader.h" #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" +#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> #include <linux/list.h> #include <linux/slab.h> @@ -1232,22 +1234,43 @@ static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) int as; for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { + unsigned long flags; struct kbase_context *kctx; - struct kbase_fault fault = { - .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, - }; + struct kbase_fault fault; if (as == MCU_AS_NR) continue; - kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as); - if (!kctx) + /* Only handle the fault for an active address space. Lock is + * taken here to atomically get reference to context in an + * active address space and retain its refcount. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as); + + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); continue; + } + + fault = (struct kbase_fault) { + .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, + }; kbase_csf_ctx_handle_fault(kctx, &fault); kbase_ctx_sched_release_ctx_lock(kctx); } + /* Internal FW error could mean hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); + if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } @@ -1292,7 +1315,7 @@ static int wait_for_global_request(struct kbase_device *const kbdev, u32 const req_mask) { const long wait_timeout = - kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); + kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); long remaining; int err = 0; @@ -1335,23 +1358,25 @@ static void enable_endpoints_global( set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); } -static void enable_shader_poweroff_timer( - struct kbase_device *const kbdev, +static void enable_shader_poweroff_timer(struct kbase_device *const kbdev, const struct kbase_csf_global_iface *const global_iface) { - u32 pwroff_reg = 0; + u32 pwroff_reg; - pwroff_reg = GLB_PWROFF_TIMER_TIMEOUT_SET( - pwroff_reg, DEFAULT_GLB_PWROFF_TIMER_TIMEOUT / - GLB_PWROFF_TIMER_TIMEOUT_SCALE); - - pwroff_reg = GLB_PWROFF_TIMER_TIMER_SOURCE_SET( - pwroff_reg, GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); + if (kbdev->csf.firmware_hctl_core_pwr) + pwroff_reg = + GLB_PWROFF_TIMER_TIMER_SOURCE_SET(DISABLE_GLB_PWROFF_TIMER, + GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); + else + pwroff_reg = kbdev->csf.mcu_core_pwroff_dur_count; kbase_csf_firmware_global_input(global_iface, GLB_PWROFF_TIMER, pwroff_reg); set_global_request(global_iface, GLB_REQ_CFG_PWROFF_TIMER_MASK); + /* Save the programed reg value in its shadow field */ + kbdev->csf.mcu_core_pwroff_reg_shadow = pwroff_reg; + dev_dbg(kbdev->dev, "GLB_PWROFF_TIMER set to 0x%.8x\n", pwroff_reg); } @@ -1432,6 +1457,8 @@ static int global_init_on_boot(struct kbase_device *const kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = + kbase_pm_no_mcu_core_pwroff(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -1445,6 +1472,8 @@ void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; + kbdev->csf.firmware_hctl_core_pwr = + kbase_pm_no_mcu_core_pwroff(kbdev); global_init(kbdev, core_mask); } @@ -1459,24 +1488,29 @@ bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) return !kbdev->csf.glb_init_request_pending; } -void kbase_csf_firmware_update_core_mask(struct kbase_device *kbdev, - u64 new_core_mask) +void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, + bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask) { unsigned long flags; lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); - enable_endpoints_global(&kbdev->csf.global_iface, new_core_mask); + if (update_core_mask) + enable_endpoints_global(&kbdev->csf.global_iface, core_mask); + if (update_core_pwroff_timer) + enable_shader_poweroff_timer(kbdev, &kbdev->csf.global_iface); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); kbase_csf_scheduler_spin_unlock(kbdev, flags); } -bool kbase_csf_firmware_core_mask_updated(struct kbase_device *kbdev) +bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); - return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK); + return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK | + GLB_REQ_CFG_PWROFF_TIMER_MASK); } /** @@ -1608,6 +1642,67 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, return hysteresis_val; } +static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us) +{ +#define PWROFF_VAL_UNIT_SHIFT (10) + /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ + u64 freq = arch_timer_get_cntfrq(); + u64 dur_val = dur_us; + u32 cnt_val_u32, reg_val_u32; + bool src_system_timestamp = freq > 0; + + if (!src_system_timestamp) { + /* Get the cycle_counter source alternative */ + spin_lock(&kbdev->pm.clk_rtm.lock); + if (kbdev->pm.clk_rtm.clks[0]) + freq = kbdev->pm.clk_rtm.clks[0]->clock_val; + else + dev_warn(kbdev->dev, "No GPU clock, unexpected integration issue!"); + spin_unlock(&kbdev->pm.clk_rtm.lock); + + dev_info(kbdev->dev, "Can't get the timestamp frequency, " + "use cycle counter with MCU Core Poweroff timer!"); + } + + /* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */ + dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; + dur_val = div_u64(dur_val, 1000000); + + /* Interface limits the value field to S32_MAX */ + cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; + + reg_val_u32 = GLB_PWROFF_TIMER_TIMEOUT_SET(0, cnt_val_u32); + /* add the source flag */ + if (src_system_timestamp) + reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, + GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); + else + reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, + GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); + + return reg_val_u32; +} + +u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) +{ + return kbdev->csf.mcu_core_pwroff_dur_us; +} + +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur) +{ + unsigned long flags; + const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->csf.mcu_core_pwroff_dur_us = dur; + kbdev->csf.mcu_core_pwroff_dur_count = pwroff; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "MCU Core Poweroff input update: 0x%.8x", pwroff); + + return pwroff; +} + int kbase_csf_firmware_init(struct kbase_device *kbdev) { const struct firmware *firmware; @@ -1633,6 +1728,7 @@ int kbase_csf_firmware_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->csf.event_wait); kbdev->csf.interrupt_received = false; + kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS; INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); INIT_LIST_HEAD(&kbdev->csf.firmware_config); @@ -1648,6 +1744,10 @@ int kbase_csf_firmware_init(struct kbase_device *kbdev) kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev, FIRMWARE_IDLE_HYSTERESIS_TIME_MS); + kbdev->csf.mcu_core_pwroff_dur_us = DEFAULT_GLB_PWROFF_TIMEOUT_US; + kbdev->csf.mcu_core_pwroff_dur_count = + convert_dur_to_core_pwroff_count(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US); + ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); if (ret != 0) { dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n"); @@ -1790,21 +1890,32 @@ void kbase_csf_firmware_term(struct kbase_device *kbdev) cancel_work_sync(&kbdev->csf.fw_error_work); - while (kbase_reset_gpu_is_active(kbdev) && !ret) - ret = kbase_reset_gpu_wait(kbdev); + ret = kbase_reset_gpu_wait(kbdev); WARN(ret, "failed to wait for GPU reset"); - /* Make sure ongoing transitions have completed */ - kbase_pm_wait_for_desired_state(kbdev); - kbase_csf_firmware_cfg_term(kbdev); kbase_csf_timeout_term(kbdev); + kbase_csf_free_dummy_user_reg_page(kbdev); + + kbase_csf_scheduler_term(kbdev); + + kbase_csf_doorbell_mapping_term(kbdev); + + /* Explicitly trigger the disabling of MCU through the state machine and + * wait for its completion. It may not have been disabled yet due to the + * power policy. + */ + kbdev->pm.backend.mcu_desired = false; + kbase_pm_wait_for_desired_state(kbdev); + + free_global_iface(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->csf.firmware_inited = false; - if (kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) { + if (WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)) { kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; stop_csf_firmware(kbdev); } @@ -1812,18 +1923,7 @@ void kbase_csf_firmware_term(struct kbase_device *kbdev) unload_mmu_tables(kbdev); - kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); - - kbase_csf_scheduler_term(kbdev); - - kbase_csf_free_dummy_user_reg_page(kbdev); - - kbase_csf_doorbell_mapping_term(kbdev); - - free_global_iface(kbdev); - - /* Release the address space */ - kbdev->as_free |= MCU_AS_BITMASK; + kbase_csf_firmware_trace_buffers_term(kbdev); while (!list_empty(&kbdev->csf.firmware_interfaces)) { struct firmware_interface *interface; @@ -1859,18 +1959,21 @@ void kbase_csf_firmware_term(struct kbase_device *kbdev) kfree(metadata); } - kbase_csf_firmware_trace_buffers_term(kbdev); - #ifndef MALI_KBASE_BUILD mali_kutf_fw_utf_entry_cleanup(kbdev); #endif - mutex_destroy(&kbdev->csf.reg_lock); - /* This will also free up the region allocated for the shared interface * entry parsed from the firmware image. */ kbase_mcu_shared_interface_region_tracker_term(kbdev); + + mutex_destroy(&kbdev->csf.reg_lock); + + kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); + + /* Release the address space */ + kbdev->as_free |= MCU_AS_BITMASK; } void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev) @@ -1976,6 +2079,8 @@ void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->protected_mode = true; kbase_ipa_protection_mode_switch_event(kbdev); + kbase_ipa_control_protm_entered(kbdev); + kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } } @@ -2184,7 +2289,8 @@ page_list_alloc_error: kfree(phys); out: /* Zero-initialize the mapping to make sure that the termination - * function doesn't try to unmap or free random addresses. */ + * function doesn't try to unmap or free random addresses. + */ csf_mapping->phys = NULL; csf_mapping->cpu_addr = NULL; csf_mapping->va_reg = NULL; diff --git a/mali_kbase/csf/mali_kbase_csf_firmware.h b/mali_kbase/csf/mali_kbase_csf_firmware.h index 3a05062..baee9da 100644 --- a/mali_kbase/csf/mali_kbase_csf_firmware.h +++ b/mali_kbase/csf/mali_kbase_csf_firmware.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_FIRMWARE_H_ @@ -79,6 +98,9 @@ /* Maximum CSs per csg. */ #define MAX_SUPPORTED_STREAMS_PER_GROUP 32 +/* Waiting timeout for status change acknowledgment, in milliseconds */ +#define CSF_FIRMWARE_TIMEOUT_MS (800) /* Relaxed to 800ms from 100ms */ + struct kbase_device; @@ -518,26 +540,32 @@ void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev); /** - * kbase_csf_firmware_update_core_mask - Send the Global configuration request - * to update the mask of enabled shader cores. + * kbase_csf_firmware_update_core_attr - Send the Global configuration request + * to update the requested core attribute + * changes. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. - * @new_core_mask: New mask of the enabled cores. + * @update_core_pwroff_timer: If true, signal the firmware needs to update + * the MCU power-off timer value. + * @update_core_mask: If true, need to do the core_mask update with + * the supplied core_mask value. + * @core_mask: New core mask value if update_core_mask is true, + * otherwise unused. */ -void kbase_csf_firmware_update_core_mask(struct kbase_device *kbdev, - u64 new_core_mask); +void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, + bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask); /** - * kbase_csf_firmware_core_mask_updated - Check the Global configuration + * kbase_csf_firmware_core_attr_updated - Check the Global configuration * request has completed or not, that was sent to update - * to update the mask of enabled shader cores. + * the core attributes. * - * Return: true if the Global configuration request to update the mask of - * enabled shader cores has completed, otherwise false. + * Return: true if the Global configuration request to update the core + * attributes has completed, otherwise false. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. */ -bool kbase_csf_firmware_core_mask_updated(struct kbase_device *kbdev); +bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev); /** * Request the global control block of CSF interface capabilities @@ -713,4 +741,59 @@ u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev); */ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur); +/** + * kbase_csf_firmware_get_mcu_core_pwroff_time - Get the MCU core power-off + * time value + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + * + * Return: the internally recorded MCU core power-off (nominal) value. The unit + * of the value is in micro-seconds. + */ +u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_set_mcu_core_pwroff_time - Set the MCU core power-off + * time value + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + * @dur: The duration value (unit: micro-seconds) for configuring MCU + * core power-off timer, when the shader cores' power + * transitions are delegated to the MCU (normal operational + * mode) + * + * The supplied value will be recorded internally without any change. But the + * actual field value will be subject to core power-off timer source frequency + * scaling and maximum value limiting. The default source will be + * SYSTEM_TIMESTAMP counter. But in case the platform is not able to supply it, + * the GPU CYCLE_COUNTER source will be used as an alternative. Bit-31 on the + * returned value is the source configuration flag, and it is set to '1' + * when CYCLE_COUNTER alternative source is used. + * + * The configured MCU core power-off timer will only have effect when the host + * driver has delegated the shader cores' power management to MCU. + * + * Return: the actual internal core power-off timer value in register defined + * format. + */ +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur); + +/** + * kbase_csf_interface_version - Helper function to build the full firmware + * interface version in a format compatible with + * with GLB_VERSION register + * + * @major: major version of csf interface + * @minor: minor version of csf interface + * @patch: patch version of csf interface + * + * Return: firmware interface version + */ +static inline u32 kbase_csf_interface_version(u32 major, u32 minor, u32 patch) +{ + return ((major << GLB_VERSION_MAJOR_SHIFT) | + (minor << GLB_VERSION_MINOR_SHIFT) | + (patch << GLB_VERSION_PATCH_SHIFT)); +} + #endif diff --git a/mali_kbase/csf/mali_kbase_csf_firmware_cfg.c b/mali_kbase/csf/mali_kbase_csf_firmware_cfg.c index d282d5c..6a4bfda 100644 --- a/mali_kbase/csf/mali_kbase_csf_firmware_cfg.c +++ b/mali_kbase/csf/mali_kbase_csf_firmware_cfg.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/csf/mali_kbase_csf_firmware_cfg.h b/mali_kbase/csf/mali_kbase_csf_firmware_cfg.h index ab4b6eb..542e3b9 100644 --- a/mali_kbase/csf/mali_kbase_csf_firmware_cfg.h +++ b/mali_kbase/csf/mali_kbase_csf_firmware_cfg.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_FIRMWARE_CFG_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_firmware_no_mali.c b/mali_kbase/csf/mali_kbase_csf_firmware_no_mali.c index 1891778..d82139a 100644 --- a/mali_kbase/csf/mali_kbase_csf_firmware_no_mali.c +++ b/mali_kbase/csf/mali_kbase_csf_firmware_no_mali.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -434,22 +435,43 @@ static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) int as; for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { + unsigned long flags; struct kbase_context *kctx; - struct kbase_fault fault = { - .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, - }; + struct kbase_fault fault; if (as == MCU_AS_NR) continue; - kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as); - if (!kctx) + /* Only handle the fault for an active address space. Lock is + * taken here to atomically get reference to context in an + * active address space and retain its refcount. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as); + + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); continue; + } + + fault = (struct kbase_fault) { + .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, + }; kbase_csf_ctx_handle_fault(kctx, &fault); kbase_ctx_sched_release_ctx_lock(kctx); } + /* Internal FW error could mean hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); + if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } @@ -494,7 +516,7 @@ static int wait_for_global_request(struct kbase_device *const kbdev, u32 const req_mask) { const long wait_timeout = - kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); + kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); long remaining; int err = 0; @@ -537,21 +559,24 @@ static void enable_endpoints_global( set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); } -static void enable_shader_poweroff_timer( - const struct kbase_csf_global_iface *const global_iface) +static void enable_shader_poweroff_timer(struct kbase_device *const kbdev, + const struct kbase_csf_global_iface *const global_iface) { - u32 pwroff_reg = 0; - - pwroff_reg = GLB_PWROFF_TIMER_TIMEOUT_SET( - pwroff_reg, DEFAULT_GLB_PWROFF_TIMER_TIMEOUT / - GLB_PWROFF_TIMER_TIMEOUT_SCALE); + u32 pwroff_reg; - pwroff_reg = GLB_PWROFF_TIMER_TIMER_SOURCE_SET( - pwroff_reg, GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); + if (kbdev->csf.firmware_hctl_core_pwr) + pwroff_reg = + GLB_PWROFF_TIMER_TIMER_SOURCE_SET(DISABLE_GLB_PWROFF_TIMER, + GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); + else + pwroff_reg = kbdev->csf.mcu_core_pwroff_dur_count; kbase_csf_firmware_global_input(global_iface, GLB_PWROFF_TIMER, pwroff_reg); set_global_request(global_iface, GLB_REQ_CFG_PWROFF_TIMER_MASK); + + /* Save the programed reg value in its shadow field */ + kbdev->csf.mcu_core_pwroff_reg_shadow = pwroff_reg; } static void set_timeout_global( @@ -582,7 +607,7 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) /* Update shader core allocation enable mask */ enable_endpoints_global(global_iface, core_mask); - enable_shader_poweroff_timer(global_iface); + enable_shader_poweroff_timer(kbdev, global_iface); set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); @@ -611,6 +636,8 @@ static int global_init_on_boot(struct kbase_device *const kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = + kbase_pm_no_mcu_core_pwroff(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -624,6 +651,8 @@ void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; + kbdev->csf.firmware_hctl_core_pwr = + kbase_pm_no_mcu_core_pwroff(kbdev); global_init(kbdev, core_mask); } @@ -638,24 +667,29 @@ bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) return !kbdev->csf.glb_init_request_pending; } -void kbase_csf_firmware_update_core_mask(struct kbase_device *kbdev, - u64 new_core_mask) +void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, + bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask) { unsigned long flags; lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); - enable_endpoints_global(&kbdev->csf.global_iface, new_core_mask); + if (update_core_mask) + enable_endpoints_global(&kbdev->csf.global_iface, core_mask); + if (update_core_pwroff_timer) + enable_shader_poweroff_timer(kbdev, &kbdev->csf.global_iface); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); kbase_csf_scheduler_spin_unlock(kbdev, flags); } -bool kbase_csf_firmware_core_mask_updated(struct kbase_device *kbdev) +bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); - return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK); + return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK | + GLB_REQ_CFG_PWROFF_TIMER_MASK); } static void kbase_csf_firmware_reload_worker(struct work_struct *work) @@ -763,6 +797,67 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, return hysteresis_val; } +static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us) +{ +#define PWROFF_VAL_UNIT_SHIFT (10) + /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ + u64 freq = arch_timer_get_cntfrq(); + u64 dur_val = dur_us; + u32 cnt_val_u32, reg_val_u32; + bool src_system_timestamp = freq > 0; + + if (!src_system_timestamp) { + /* Get the cycle_counter source alternative */ + spin_lock(&kbdev->pm.clk_rtm.lock); + if (kbdev->pm.clk_rtm.clks[0]) + freq = kbdev->pm.clk_rtm.clks[0]->clock_val; + else + dev_warn(kbdev->dev, "No GPU clock, unexpected integration issue!"); + spin_unlock(&kbdev->pm.clk_rtm.lock); + + dev_info(kbdev->dev, "Can't get the timestamp frequency, " + "use cycle counter with MCU Core Poweroff timer!"); + } + + /* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */ + dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; + dur_val = div_u64(dur_val, 1000000); + + /* Interface limits the value field to S32_MAX */ + cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; + + reg_val_u32 = GLB_PWROFF_TIMER_TIMEOUT_SET(0, cnt_val_u32); + /* add the source flag */ + if (src_system_timestamp) + reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, + GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); + else + reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, + GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); + + return reg_val_u32; +} + +u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) +{ + return kbdev->csf.mcu_core_pwroff_dur_us; +} + +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur) +{ + unsigned long flags; + const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->csf.mcu_core_pwroff_dur_us = dur; + kbdev->csf.mcu_core_pwroff_dur_count = pwroff; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "MCU Core Poweroff input update: 0x%.8x", pwroff); + + return pwroff; +} + int kbase_csf_firmware_init(struct kbase_device *kbdev) { int ret; @@ -782,6 +877,7 @@ int kbase_csf_firmware_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->csf.event_wait); kbdev->csf.interrupt_received = false; + kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS; INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); INIT_LIST_HEAD(&kbdev->csf.firmware_config); @@ -1046,6 +1142,8 @@ static u32 copy_grp_and_stm( if (i < max_group_num) { group_data[i].features = iface->groups[i].features; group_data[i].stream_num = iface->groups[i].stream_num; + group_data[i].suspend_size = + iface->groups[i].suspend_size; } for (j = 0; j < iface->groups[i].stream_num; j++) { if (total_stream_num < max_total_stream_num) @@ -1197,7 +1295,8 @@ page_list_alloc_error: kfree(phys); out: /* Zero-initialize the mapping to make sure that the termination - * function doesn't try to unmap or free random addresses. */ + * function doesn't try to unmap or free random addresses. + */ csf_mapping->phys = NULL; csf_mapping->cpu_addr = NULL; csf_mapping->va_reg = NULL; diff --git a/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.c b/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.c index 087cc85..4dc1ac1 100644 --- a/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.c +++ b/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.h b/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.h index f71ea01..ecb0852 100644 --- a/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.h +++ b/mali_kbase/csf/mali_kbase_csf_heap_context_alloc.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #include <mali_kbase.h> diff --git a/mali_kbase/csf/mali_kbase_csf_ioctl.h b/mali_kbase/csf/mali_kbase_csf_ioctl.h index d7db345..0bbc96c 100644 --- a/mali_kbase/csf/mali_kbase_csf_ioctl.h +++ b/mali_kbase/csf/mali_kbase_csf_ioctl.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_IOCTL_H_ @@ -29,10 +48,16 @@ /* * 1.0: * - CSF IOCTL header separated from JM + * 1.1: + * - Add a new priority level BASE_QUEUE_GROUP_PRIORITY_REALTIME + * - Add ioctl 54: This controls the priority setting. + * 1.2: + * - Add new CSF GPU_FEATURES register into the property structure + * returned by KBASE_IOCTL_GET_GPUPROPS */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 0 +#define BASE_UK_VERSION_MINOR 2 /** * struct kbase_ioctl_version_check - Check version compatibility between @@ -46,7 +71,6 @@ struct kbase_ioctl_version_check { __u16 minor; }; - #define KBASE_IOCTL_VERSION_CHECK_RESERVED \ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) @@ -341,7 +365,6 @@ struct kbase_ioctl_cs_cpu_queue_info { #define KBASE_IOCTL_CS_CPU_QUEUE_DUMP \ _IOW(KBASE_IOCTL_TYPE, 53, struct kbase_ioctl_cs_cpu_queue_info) - /*************** * test ioctls * ***************/ diff --git a/mali_kbase/csf/mali_kbase_csf_kcpu.c b/mali_kbase/csf/mali_kbase_csf_kcpu.c index b9c2597..b073d05 100644 --- a/mali_kbase/csf/mali_kbase_csf_kcpu.c +++ b/mali_kbase/csf/mali_kbase_csf_kcpu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -576,8 +577,8 @@ static int kbase_csf_queue_group_suspend_prepare( u64 end_addr = addr + suspend_buf->size - 1; u64 last_page_addr = end_addr & PAGE_MASK; int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; - int pinned_pages; - int ret = 0; + int pinned_pages = 0, ret = 0; + struct kbase_va_region *reg; lockdep_assert_held(&kctx->csf.kcpu_queues.lock); @@ -604,17 +605,57 @@ static int kbase_csf_queue_group_suspend_prepare( goto out_clean_sus_buf; } - pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, - sus_buf->pages); - if (pinned_pages < 0) { - ret = pinned_pages; - goto out_clean_pages; - } - if (pinned_pages != nr_pages) { - ret = -EINVAL; - goto out_clean_pages; + /* Check if the page_addr is a valid GPU VA from SAME_VA zone, + * otherwise consider it is a CPU VA corresponding to the Host + * memory allocated by userspace. + */ + kbase_gpu_vm_lock(kctx); + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + page_addr); + + if (kbase_is_region_invalid_or_free(reg)) { + kbase_gpu_vm_unlock(kctx); + pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, + sus_buf->pages); + kbase_gpu_vm_lock(kctx); + + if (pinned_pages < 0) { + ret = pinned_pages; + goto out_clean_pages; + } + if (pinned_pages != nr_pages) { + ret = -EINVAL; + goto out_clean_pages; + } + } else { + struct tagged_addr *page_array; + u64 start, end, i; + + if (!(reg->flags & BASE_MEM_SAME_VA) || + reg->nr_pages < nr_pages || + kbase_reg_current_backed_size(reg) != + reg->nr_pages) { + ret = -EINVAL; + goto out_clean_pages; + } + + start = PFN_DOWN(page_addr) - reg->start_pfn; + end = start + nr_pages; + + if (end > reg->nr_pages) { + ret = -EINVAL; + goto out_clean_pages; + } + + sus_buf->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + page_array = kbase_get_cpu_phy_pages(reg); + page_array += start; + + for (i = 0; i < nr_pages; i++, page_array++) + sus_buf->pages[i] = as_page(*page_array); } + kbase_gpu_vm_unlock(kctx); current_command->type = BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND; current_command->info.suspend_buf_copy.sus_buf = sus_buf; current_command->info.suspend_buf_copy.group_handle = @@ -622,9 +663,11 @@ static int kbase_csf_queue_group_suspend_prepare( return ret; out_clean_pages: + kbase_gpu_vm_unlock(kctx); kfree(sus_buf->pages); out_clean_sus_buf: kfree(sus_buf); + return ret; } @@ -688,6 +731,8 @@ static int kbase_kcpu_cqs_wait_process(struct kbase_device *kbdev, KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( kbdev, queue); queue->command_started = true; + KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_START, + queue, cqs_wait->nr_objs, 0); } if (!evt) { @@ -699,10 +744,18 @@ static int kbase_kcpu_cqs_wait_process(struct kbase_device *kbdev, sig_set = evt[BASEP_EVENT_VAL_INDEX] > cqs_wait->objs[i].val; if (sig_set) { + bool error = false; + bitmap_set(cqs_wait->signaled, i, 1); if ((cqs_wait->inherit_err_flags & (1U << i)) && - evt[BASEP_EVENT_ERR_INDEX] > 0) + evt[BASEP_EVENT_ERR_INDEX] > 0) { queue->has_error = true; + error = true; + } + + KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_END, + queue, cqs_wait->objs[i].addr, + error); KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( kbdev, queue, @@ -813,6 +866,10 @@ static void kbase_kcpu_cqs_set_process(struct kbase_device *kbdev, /* Set to signaled */ evt[BASEP_EVENT_VAL_INDEX]++; kbase_phy_alloc_mapping_put(queue->kctx, mapping); + + KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_SET, + queue, cqs_set->objs[i].addr, + evt[BASEP_EVENT_ERR_INDEX]); } } @@ -872,6 +929,9 @@ static void kbase_csf_fence_wait_callback(struct dma_fence *fence, struct kbase_kcpu_command_queue *kcpu_queue = fence_info->kcpu_queue; struct kbase_context *const kctx = kcpu_queue->kctx; + KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END, kcpu_queue, + fence->context, fence->seqno); + /* Resume kcpu command queue processing. */ queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); } @@ -888,8 +948,13 @@ static void kbase_kcpu_fence_wait_cancel( return; if (kcpu_queue->fence_wait_processed) { - dma_fence_remove_callback(fence_info->fence, + bool removed = dma_fence_remove_callback(fence_info->fence, &fence_info->fence_cb); + + if (removed) + KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END, + kcpu_queue, fence_info->fence->context, + fence_info->fence->seqno); } /* Release the reference which is kept by the kcpu_queue */ @@ -933,6 +998,9 @@ static int kbase_kcpu_fence_wait_process( &fence_info->fence_cb, kbase_csf_fence_wait_callback); + KBASE_KTRACE_ADD_CSF_KCPU(kcpu_queue->kctx->kbdev, + FENCE_WAIT_START, kcpu_queue, + fence->context, fence->seqno); fence_status = cb_err; if (cb_err == 0) kcpu_queue->fence_wait_processed = true; @@ -1003,6 +1071,10 @@ static int kbase_kcpu_fence_signal_process( "fence_signal() failed with %d\n", ret); } + KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_SIGNAL, kcpu_queue, + fence_info->fence->context, + fence_info->fence->seqno); + dma_fence_put(fence_info->fence); fence_info->fence = NULL; @@ -1112,6 +1184,9 @@ static int delete_queue(struct kbase_context *kctx, u32 id) struct kbase_kcpu_command_queue *queue = kctx->csf.kcpu_queues.array[id]; + KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_DESTROY, + queue, queue->num_pending_cmds, queue->cqs_wait_count); + /* Drain the remaining work for this queue first and go past * all the waits. */ @@ -1394,13 +1469,15 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( kbdev, queue); break; - case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: + case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: { + struct kbase_suspend_copy_buffer *sus_buf = + cmd->info.suspend_buf_copy.sus_buf; + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( kbdev, queue); status = kbase_csf_queue_group_suspend_process( - queue->kctx, - cmd->info.suspend_buf_copy.sus_buf, + queue->kctx, sus_buf, cmd->info.suspend_buf_copy.group_handle); if (status) queue->has_error = true; @@ -1408,9 +1485,19 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( kbdev, queue, status); - kfree(cmd->info.suspend_buf_copy.sus_buf->pages); - kfree(cmd->info.suspend_buf_copy.sus_buf); + if (!sus_buf->cpu_alloc) { + int i; + + for (i = 0; i < sus_buf->nr_pages; i++) + put_page(sus_buf->pages[i]); + } else { + kbase_mem_phy_alloc_put(sus_buf->cpu_alloc); + } + + kfree(sus_buf->pages); + kfree(sus_buf); break; + } default: dev_warn(kbdev->dev, "Unrecognized command type\n"); @@ -1798,6 +1885,7 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, INIT_LIST_HEAD(&queue->jit_blocked); queue->has_error = false; INIT_WORK(&queue->work, kcpu_queue_process_worker); + queue->id = idx; newq->id = idx; @@ -1806,6 +1894,9 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, */ KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( kctx->kbdev, queue, kctx->id, queue->num_pending_cmds); + + KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_NEW, queue, + queue->fence_context, 0); out: mutex_unlock(&kctx->csf.kcpu_queues.lock); diff --git a/mali_kbase/csf/mali_kbase_csf_kcpu.h b/mali_kbase/csf/mali_kbase_csf_kcpu.h index 6c4c2d2..cf506ea 100644 --- a/mali_kbase/csf/mali_kbase_csf_kcpu.h +++ b/mali_kbase/csf/mali_kbase_csf_kcpu.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_KCPU_H_ @@ -145,12 +164,14 @@ struct kbase_kcpu_command_jit_free_info { * the user buffer. * @nr_pages: number of pages. * @offset: offset into the pages + * @cpu_alloc: Reference to physical pages of suspend buffer allocation. */ struct kbase_suspend_copy_buffer { size_t size; struct page **pages; int nr_pages; size_t offset; + struct kbase_mem_phy_alloc *cpu_alloc; }; /** @@ -201,6 +222,7 @@ struct kbase_kcpu_command { * commands enqueued into a kcpu command queue; * part of kernel API for processing workqueues * @start_offset: Index of the command to be executed next + * @id: KCPU command queue ID. * @num_pending_cmds: The number of commands enqueued but not yet * executed or pending * @cqs_wait_count: Tracks the number of CQS wait commands enqueued @@ -233,6 +255,7 @@ struct kbase_kcpu_command_queue { struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; struct work_struct work; u8 start_offset; + u8 id; u16 num_pending_cmds; u32 cqs_wait_count; u64 fence_context; diff --git a/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.c b/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.c index 55e3b64..004fc89 100644 --- a/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.c +++ b/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -196,4 +197,3 @@ void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) } #endif /* CONFIG_DEBUG_FS */ - diff --git a/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.h b/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.h index 359fe2c..11ee381 100644 --- a/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.h +++ b/mali_kbase/csf/mali_kbase_csf_kcpu_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_KCPU_DEBUGFS_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_protected_memory.c b/mali_kbase/csf/mali_kbase_csf_protected_memory.c index 987cbc2..0ace25a 100644 --- a/mali_kbase/csf/mali_kbase_csf_protected_memory.c +++ b/mali_kbase/csf/mali_kbase_csf_protected_memory.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/csf/mali_kbase_csf_protected_memory.h b/mali_kbase/csf/mali_kbase_csf_protected_memory.h index 2b45991..ba9411d 100644 --- a/mali_kbase/csf/mali_kbase_csf_protected_memory.h +++ b/mali_kbase/csf/mali_kbase_csf_protected_memory.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_PROTECTED_MEMORY_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_reset_gpu.c b/mali_kbase/csf/mali_kbase_csf_reset_gpu.c index 4a88ffc..8418320 100644 --- a/mali_kbase/csf/mali_kbase_csf_reset_gpu.c +++ b/mali_kbase/csf/mali_kbase_csf_reset_gpu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -29,6 +30,7 @@ #include <mali_kbase_regs_history_debugfs.h> #include <csf/mali_kbase_csf_trace_buffer.h> #include <csf/ipa_control/mali_kbase_csf_ipa_control.h> +#include <mali_kbase_reset_gpu.h> /* Waiting timeout for GPU reset to complete */ #define GPU_RESET_TIMEOUT_MS (5000) /* 5 seconds */ @@ -38,6 +40,199 @@ #define DUMP_HEX_CHARS_PER_LINE \ (DUMP_DWORDS_PER_LINE * DUMP_HEX_CHARS_PER_DWORD) +static inline bool +kbase_csf_reset_state_is_silent(enum kbase_csf_reset_gpu_state state) +{ + return (state == KBASE_CSF_RESET_GPU_COMMITTED_SILENT); +} + +static inline bool +kbase_csf_reset_state_is_committed(enum kbase_csf_reset_gpu_state state) +{ + return (state == KBASE_CSF_RESET_GPU_COMMITTED || + state == KBASE_CSF_RESET_GPU_COMMITTED_SILENT); +} + +static inline bool +kbase_csf_reset_state_is_active(enum kbase_csf_reset_gpu_state state) +{ + return (state == KBASE_CSF_RESET_GPU_HAPPENING); +} + +/** + * DOC: Mechanism for coherent access to the HW with respect to GPU reset + * + * Access to the HW from non-atomic context outside of the reset thread must + * use kbase_reset_gpu_prevent_and_wait() / kbase_reset_gpu_try_prevent(). + * + * This currently works by taking the &kbase_device's csf.reset.sem, for + * 'write' access by the GPU reset thread and 'read' access by every other + * thread. The use of this rw_semaphore means: + * + * - there will be mutual exclusion (and thus waiting) between the thread doing + * reset ('writer') and threads trying to access the GPU for 'normal' + * operations ('readers') + * + * - multiple threads may prevent reset from happening without serializing each + * other prematurely. Note that at present the wait for reset to finish has + * to be done higher up in the driver than actual GPU access, at a point + * where it won't cause lock ordering issues. At such a point, some paths may + * actually lead to no GPU access, but we would prefer to avoid serializing + * at that level + * + * - lockdep (if enabled in the kernel) will check such uses for deadlock + * + * If instead &kbase_device's csf.reset.wait &wait_queue_head_t were used on + * its own, we'd also need to add a &lockdep_map and appropriate lockdep calls + * to make use of lockdep checking in all places where the &wait_queue_head_t + * is waited upon or signaled. + * + * Indeed places where we wait on &kbase_device's csf.reset.wait (such as + * kbase_reset_gpu_wait()) are the only places where we need extra call(s) to + * lockdep, and they are made on the existing rw_semaphore. + * + * For non-atomic access, the &kbase_device's csf.reset.state member should be + * checked instead, such as by using kbase_reset_gpu_is_active(). + * + * Ideally the &rw_semaphore should be replaced in future with a single mutex + * that protects any access to the GPU, via reset or otherwise. + */ + +int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) +{ + down_read(&kbdev->csf.reset.sem); + + if (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_FAILED) { + up_read(&kbdev->csf.reset.sem); + return -ENOMEM; + } + + if (WARN_ON(kbase_reset_gpu_is_active(kbdev))) { + up_read(&kbdev->csf.reset.sem); + return -EFAULT; + } + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu_prevent_and_wait); + +int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) +{ + if (!down_read_trylock(&kbdev->csf.reset.sem)) + return -EAGAIN; + + if (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_FAILED) { + up_read(&kbdev->csf.reset.sem); + return -ENOMEM; + } + + if (WARN_ON(kbase_reset_gpu_is_active(kbdev))) { + up_read(&kbdev->csf.reset.sem); + return -EFAULT; + } + + return 0; +} + +void kbase_reset_gpu_allow(struct kbase_device *kbdev) +{ + up_read(&kbdev->csf.reset.sem); +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu_allow); + +void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) +{ +#if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE + lockdep_assert_held_read(&kbdev->csf.reset.sem); +#else + lockdep_assert_held(&kbdev->csf.reset.sem); +#endif + WARN_ON(kbase_reset_gpu_is_active(kbdev)); +} + +void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) +{ + if (atomic_read(&kbdev->csf.reset.state) == KBASE_CSF_RESET_GPU_FAILED) + return; + +#if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE + lockdep_assert_held_read(&kbdev->csf.reset.sem); +#else + lockdep_assert_held(&kbdev->csf.reset.sem); +#endif + WARN_ON(kbase_reset_gpu_is_active(kbdev)); +} + +/* Mark the reset as now happening, and synchronize with other threads that + * might be trying to access the GPU + */ +static void kbase_csf_reset_begin_hw_access_sync( + struct kbase_device *kbdev, + enum kbase_csf_reset_gpu_state initial_reset_state) +{ + unsigned long hwaccess_lock_flags; + unsigned long scheduler_spin_lock_flags; + + /* Note this is a WARN/atomic_set because it is a software issue for a + * race to be occurring here + */ + WARN_ON(!kbase_csf_reset_state_is_committed(initial_reset_state)); + + down_write(&kbdev->csf.reset.sem); + + /* Threads in atomic context accessing the HW will hold one of these + * locks, so synchronize with them too. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_lock_flags); + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_spin_lock_flags); + atomic_set(&kbdev->csf.reset.state, KBASE_RESET_GPU_HAPPENING); + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_spin_lock_flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_lock_flags); +} + +/* Mark the reset as finished and allow others threads to once more access the + * GPU + */ +static void kbase_csf_reset_end_hw_access(struct kbase_device *kbdev, + int err_during_reset, + bool firmware_inited) +{ + unsigned long hwaccess_lock_flags; + unsigned long scheduler_spin_lock_flags; + + WARN_ON(!kbase_csf_reset_state_is_active( + atomic_read(&kbdev->csf.reset.state))); + + /* Once again, we synchronize with atomic context threads accessing the + * HW, as otherwise any actions they defer could get lost + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_lock_flags); + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_spin_lock_flags); + + if (!err_during_reset) { + atomic_set(&kbdev->csf.reset.state, + KBASE_CSF_RESET_GPU_NOT_PENDING); + } else { + dev_err(kbdev->dev, "Reset failed to complete"); + atomic_set(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_FAILED); + } + + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_spin_lock_flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_lock_flags); + + /* Invoke the scheduling tick after formally finishing the reset, + * otherwise the tick might start too soon and notice that reset + * is still in progress. + */ + up_write(&kbdev->csf.reset.sem); + wake_up(&kbdev->csf.reset.wait); + + if (!err_during_reset && likely(firmware_inited)) + kbase_csf_scheduler_enable_tick_timer(kbdev); +} + static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) { kbase_io_history_dump(kbdev); @@ -115,26 +310,33 @@ static void kbase_csf_dump_firmware_trace_buffer(struct kbase_device *kbdev) } static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, - bool firmware_inited) + bool firmware_inited, bool silent) { unsigned long flags; - bool silent = false; int err; - if (atomic_read(&kbdev->csf.reset.state) == KBASE_CSF_RESET_GPU_SILENT) - silent = true; - WARN_ON(kbdev->irq_reset_flush); - - /* Reset the scheduler state before disabling the interrupts as suspend of active - * CSG slots would also be done as a part of reset. + /* The reset must now be happening otherwise other threads will not + * have been synchronized with to stop their access to the HW + */ +#if KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE + lockdep_assert_held_write(&kbdev->csf.reset.sem); +#elif KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE + lockdep_assert_held_exclusive(&kbdev->csf.reset.sem); +#else + lockdep_assert_held(&kbdev->csf.reset.sem); +#endif + WARN_ON(!kbase_reset_gpu_is_active(kbdev)); + + /* Reset the scheduler state before disabling the interrupts as suspend + * of active CSG slots would also be done as a part of reset. */ if (likely(firmware_inited)) kbase_csf_scheduler_reset(kbdev); cancel_work_sync(&kbdev->csf.firmware_reload_work); - /* Disable GPU hardware counters. - * This call will block until counters are disabled. + dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); + /* This call will block until counters are disabled. */ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); @@ -142,7 +344,8 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, spin_lock(&kbdev->mmu_mask_change); kbase_pm_reset_start_locked(kbdev); - /* We're about to flush out the IRQs and their bottom halves */ + dev_dbg(kbdev->dev, + "We're about to flush out the IRQs and their bottom halves\n"); kbdev->irq_reset_flush = true; /* Disable IRQ to avoid IRQ handlers to kick in after releasing the @@ -153,15 +356,16 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, spin_unlock(&kbdev->mmu_mask_change); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - /* Ensure that any IRQ handlers have finished - * Must be done without any locks IRQ handlers will take. + dev_dbg(kbdev->dev, "Ensure that any IRQ handlers have finished\n"); + /* Must be done without any locks IRQ handlers will take. */ kbase_synchronize_irqs(kbdev); - /* Flush out any in-flight work items */ + dev_dbg(kbdev->dev, "Flush out any in-flight work items\n"); kbase_flush_mmu_wqs(kbdev); - /* The flush has completed so reset the active indicator */ + dev_dbg(kbdev->dev, + "The flush has completed so reset the active indicator\n"); kbdev->irq_reset_flush = false; mutex_lock(&kbdev->pm.lock); @@ -179,17 +383,23 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, } spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->protected_mode = false; kbase_ipa_control_handle_gpu_reset_pre(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /* Tell hardware counters a reset is about to occur. + * If the backend is in an unrecoverable error state (e.g. due to + * firmware being unresponsive) this will transition the backend out of + * it, on the assumption a reset will fix whatever problem there was. + */ + kbase_hwcnt_backend_csf_on_before_reset(&kbdev->hwcnt_gpu_iface); + /* Reset the GPU */ err = kbase_pm_init_hw(kbdev, 0); mutex_unlock(&kbdev->pm.lock); if (WARN_ON(err)) - return err; + goto error; mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -207,7 +417,7 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, mutex_unlock(&kbdev->pm.lock); if (err) - return err; + goto error; /* Re-enable GPU hardware counters */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -218,6 +428,28 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, dev_err(kbdev->dev, "Reset complete"); return 0; +error: + WARN_ON(!err); + + /* If hardware init failed, we assume hardware counters will + * not work and put the backend into the unrecoverable error + * state. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); + + /* Re-enable counters to ensure matching enable/disable pair. + * This might reduce the hwcnt disable count to 0, and therefore + * trigger actual re-enabling of hwcnt. + * However, as the backend is now in the unrecoverable error state, + * re-enabling will immediately fail and put the context into the error + * state, preventing the hardware from being touched (which could have + * risked a hang). + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return err; } static void kbase_csf_reset_gpu_worker(struct work_struct *data) @@ -227,6 +459,13 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) bool firmware_inited; unsigned long flags; int err = 0; + const enum kbase_csf_reset_gpu_state initial_reset_state = + atomic_read(&kbdev->csf.reset.state); + + /* Ensure any threads (e.g. executing the CSF scheduler) have finished + * using the HW + */ + kbase_csf_reset_begin_hw_access_sync(kbdev, initial_reset_state); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); firmware_inited = kbdev->csf.firmware_inited; @@ -234,31 +473,24 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - err = kbase_csf_reset_gpu_now(kbdev, firmware_inited); + bool silent = + kbase_csf_reset_state_is_silent(initial_reset_state); + + err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); kbase_pm_context_idle(kbdev); } kbase_disjoint_state_down(kbdev); - if (!err) { - atomic_set(&kbdev->csf.reset.state, - KBASE_CSF_RESET_GPU_NOT_PENDING); - if (likely(firmware_inited)) - kbase_csf_scheduler_enable_tick_timer(kbdev); - } else { - dev_err(kbdev->dev, "Reset failed to complete"); - atomic_set(&kbdev->csf.reset.state, - KBASE_CSF_RESET_GPU_FAILED); - } - - wake_up(&kbdev->csf.reset.wait); + /* Allow other threads to once again use the GPU */ + kbase_csf_reset_end_hw_access(kbdev, err, firmware_inited); } bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) { if (atomic_cmpxchg(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_NOT_PENDING, - KBASE_CSF_RESET_GPU_HAPPENING) != + KBASE_CSF_RESET_GPU_PREPARED) != KBASE_CSF_RESET_GPU_NOT_PENDING) { /* Some other thread is already resetting the GPU */ return false; @@ -275,15 +507,21 @@ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) return kbase_prepare_to_reset_gpu(kbdev); } -int kbase_reset_gpu(struct kbase_device *kbdev) +void kbase_reset_gpu(struct kbase_device *kbdev) { + /* Note this is a WARN/atomic_set because it is a software issue for + * a race to be occurring here + */ + if (WARN_ON(atomic_read(&kbdev->csf.reset.state) != + KBASE_RESET_GPU_PREPARED)) + return; + + atomic_set(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_COMMITTED); dev_err(kbdev->dev, "Preparing to soft-reset GPU\n"); kbase_disjoint_state_up(kbdev); queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); - - return 0; } KBASE_EXPORT_TEST_API(kbase_reset_gpu); @@ -298,7 +536,7 @@ int kbase_reset_gpu_silent(struct kbase_device *kbdev) { if (atomic_cmpxchg(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_NOT_PENDING, - KBASE_CSF_RESET_GPU_SILENT) != + KBASE_CSF_RESET_GPU_COMMITTED_SILENT) != KBASE_CSF_RESET_GPU_NOT_PENDING) { /* Some other thread is already resetting the GPU */ return -EAGAIN; @@ -313,23 +551,42 @@ int kbase_reset_gpu_silent(struct kbase_device *kbdev) bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) { - if (atomic_read(&kbdev->csf.reset.state) == - KBASE_CSF_RESET_GPU_NOT_PENDING) - return false; + enum kbase_csf_reset_gpu_state reset_state = + atomic_read(&kbdev->csf.reset.state); - return true; + /* For CSF, the reset is considered active only when the reset worker + * is actually executing and other threads would have to wait for it to + * complete + */ + return kbase_csf_reset_state_is_active(reset_state); } int kbase_reset_gpu_wait(struct kbase_device *kbdev) { const long wait_timeout = kbase_csf_timeout_in_jiffies(GPU_RESET_TIMEOUT_MS); - long remaining = wait_event_timeout(kbdev->csf.reset.wait, - (atomic_read(&kbdev->csf.reset.state) == - KBASE_CSF_RESET_GPU_NOT_PENDING) || - (atomic_read(&kbdev->csf.reset.state) == - KBASE_CSF_RESET_GPU_FAILED), - wait_timeout); + long remaining; + + /* Inform lockdep we might be trying to wait on a reset (as + * would've been done with down_read() - which has no 'timeout' + * variant), then use wait_event_timeout() to implement the timed + * wait. + * + * in CONFIG_PROVE_LOCKING builds, this should catch potential 'time + * bound' deadlocks such as: + * - incorrect lock order with respect to others locks + * - current thread has prevented reset + * - current thread is executing the reset worker + */ + might_lock_read(&kbdev->csf.reset.sem); + + remaining = wait_event_timeout( + kbdev->csf.reset.wait, + (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_NOT_PENDING) || + (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_FAILED), + wait_timeout); if (!remaining) { dev_warn(kbdev->dev, "Timed out waiting for the GPU reset to complete"); @@ -352,6 +609,7 @@ int kbase_reset_gpu_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.reset.work, kbase_csf_reset_gpu_worker); init_waitqueue_head(&kbdev->csf.reset.wait); + init_rwsem(&kbdev->csf.reset.sem); return 0; } diff --git a/mali_kbase/csf/mali_kbase_csf_scheduler.c b/mali_kbase/csf/mali_kbase_csf_scheduler.c index a78088a..e78b912 100644 --- a/mali_kbase/csf/mali_kbase_csf_scheduler.c +++ b/mali_kbase/csf/mali_kbase_csf_scheduler.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -35,9 +36,6 @@ /* Value to indicate that a queue group is not groups_to_schedule list */ #define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) -/* Waiting timeout for status change acknowledgment, in milliseconds */ -#define CSF_STATE_WAIT_TIMEOUT_MS (800) /* Relaxed to 800ms from 100ms */ - /* Waiting timeout for scheduler state change for descheduling a CSG */ #define CSG_SCHED_STOP_TIMEOUT_MS (50) @@ -51,8 +49,6 @@ /* CSF scheduler time slice value */ #define CSF_SCHEDULER_TIME_TICK_MS (100) /* 100 milliseconds */ -#define CSF_SCHEDULER_TIME_TICK_JIFFIES \ - msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_MS) /* * CSF scheduler time threshold for converting "tock" requests into "tick" if @@ -81,19 +77,10 @@ /* CS suspended and is wait for a CQS condition */ #define CS_WAIT_SYNC_FLAG (1 << 1) -/* This is to avoid the immediate power down of GPU when then are no groups - * left for scheduling. GPUCORE-24250 would add the proper GPU idle detection - * logic. - */ -#define GPU_IDLE_POWEROFF_HYSTERESIS_DELAY msecs_to_jiffies((u32)10) - -/* This is a workaround before MIDHARC-3065, for avoiding some corner - * cases where a hang is possible for gpu-queues that happen to be on - * deferred commands when suspended. - * The MIDHARC-3065 is scheduled with header 10.x.7/11.x.4. +/* 2 GPU address space slots are reserved for MCU and privileged context for HW + * counter dumping. TODO remove the slot reserved for latter in GPUCORE-26293. */ - -#define USE_PRE_MIDHARC_3065_WORKAROUND (1) +#define NUM_RESERVED_AS_SLOTS (2) static int scheduler_group_schedule(struct kbase_queue_group *group); static void remove_group_from_idle_wait(struct kbase_queue_group *const group); @@ -112,6 +99,91 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, #define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) +/** + * tick_timer_callback() - Callback function for the scheduling tick hrtimer + * + * @timer: Pointer to the device + * + * This function will enqueue the scheduling tick work item for immediate + * execution, if it has not been queued already. + * + * Return: enum value to indicate that timer should not be restarted. + */ +static enum hrtimer_restart tick_timer_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev = container_of(timer, struct kbase_device, + csf.scheduler.tick_timer); + + kbase_csf_scheduler_advance_tick(kbdev); + return HRTIMER_NORESTART; +} + +/** + * start_tick_timer() - Start the scheduling tick hrtimer. + * + * @kbdev: Pointer to the device + * + * This function will start the scheduling tick hrtimer and is supposed to + * be called only from the tick work item function. The tick hrtimer should + * should not be active already. + */ +static void start_tick_timer(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + WARN_ON(scheduler->tick_timer_active); + if (likely(!work_pending(&scheduler->tick_work))) { + scheduler->tick_timer_active = true; + + hrtimer_start(&scheduler->tick_timer, + HR_TIMER_DELAY_MSEC(scheduler->csg_scheduling_period_ms), + HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); +} + +/** + * cancel_tick_timer() - Cancel the scheduling tick hrtimer + * + * @kbdev: Pointer to the device + */ +static void cancel_tick_timer(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + scheduler->tick_timer_active = false; + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + hrtimer_cancel(&scheduler->tick_timer); +} + +/** + * enqueue_tick_work() - Enqueue the scheduling tick work item + * + * @kbdev: Pointer to the device + * + * This function will queue the scheduling tick work item for immediate + * execution. This shall only be called when both the tick hrtimer and tick + * work item are not active/pending. + */ +static void enqueue_tick_work(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + WARN_ON(scheduler->tick_timer_active); + queue_work(scheduler->wq, &scheduler->tick_work); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); +} + static void release_doorbell(struct kbase_device *kbdev, int doorbell_nr) { WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); @@ -333,15 +405,7 @@ static void enable_gpu_idle_fw_timer(struct kbase_device *kbdev) /* Update the timer_enabled flag requires holding interrupt_lock */ scheduler->gpu_idle_fw_timer_enabled = true; - - /* Only send the enable when walkaround is not used, before - * MIDHARC-3065 is supported. This is for avoiding some - * corner cases where a hang is possible for gpu-queues that - * happen to be on some deferred commands when suspended. - */ -#if (!USE_PRE_MIDHARC_3065_WORKAROUND) kbase_csf_firmware_enable_gpu_idle_timer(kbdev); -#endif spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); } @@ -356,13 +420,6 @@ static void disable_gpu_idle_fw_timer_locked(struct kbase_device *kbdev) /* Update of the timer_enabled flag requires holding interrupt_lock */ if (scheduler->gpu_idle_fw_timer_enabled) { scheduler->gpu_idle_fw_timer_enabled = false; - - /* Disable can always be sent, even with build configured - * to use USE_PRE_MIDHARC_3065_WORKAROUND. There is no adverse - * side effect for disabling a not yet enabled item with the - * firmware interface call - it just has no effect (i.e. NOP). - */ - kbase_csf_firmware_disable_gpu_idle_timer(kbdev); } } @@ -525,8 +582,7 @@ static int halt_stream_sync(struct kbase_queue *queue) struct kbase_csf_cmd_stream_group_info *ginfo; struct kbase_csf_cmd_stream_info *stream; int csi_index = queue->csi_index; - long remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); if (WARN_ON(!group) || WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) @@ -553,7 +609,7 @@ static int halt_stream_sync(struct kbase_queue *queue) } remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); } /* Set state to STOP */ @@ -561,16 +617,32 @@ static int halt_stream_sync(struct kbase_queue *queue) CS_REQ_STATE_MASK); KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP_REQUESTED, group, queue, 0u); - kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr); + kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr, true); /* Timed wait */ remaining = wait_event_timeout(kbdev->csf.event_wait, (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) == CS_ACK_STATE_STOP), remaining); + /* Queues that have failed to stop in time shall raise a fatal error + * as their group would fail to suspend which could no longer be safely + * resumed. + */ if (!remaining) { + unsigned long flags; + dev_warn(kbdev->dev, "Timed out waiting for queue to stop on csi %d bound to group %d on slot %d", queue->csi_index, group->handle, group->csg_nr); + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + kbase_csf_add_queue_fatal_error( + queue, GPU_EXCEPTION_TYPE_SW_FAULT_2, 0); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, + flags); + + /* TODO GPUCORE-25328: The CSG can't be terminated, the GPU + * will be reset as a work-around. + */ if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } @@ -654,8 +726,9 @@ retry: while (group->run_state == KBASE_CSF_GROUP_IDLE) { mutex_unlock(&scheduler->lock); remaining = wait_event_timeout(kbdev->csf.event_wait, - group->run_state != KBASE_CSF_GROUP_IDLE, - CSF_STATE_WAIT_TIMEOUT_MS); + group->run_state != + KBASE_CSF_GROUP_IDLE, + kbdev->csf.fw_timeout_ms); mutex_lock(&scheduler->lock); if (!remaining) { dev_warn(kbdev->dev, @@ -687,9 +760,10 @@ retry: * CSF context is locked. Therefore, the scheduler would be * the only one to update the run_state of the group. */ - remaining = wait_event_timeout(kbdev->csf.event_wait, - can_halt_stream(kbdev, group), - kbase_csf_timeout_in_jiffies(20 * CSF_SCHEDULER_TIME_TICK_MS)); + remaining = wait_event_timeout( + kbdev->csf.event_wait, can_halt_stream(kbdev, group), + kbase_csf_timeout_in_jiffies( + 20 * kbdev->csf.scheduler.csg_scheduling_period_ms)); mutex_lock(&scheduler->lock); @@ -747,9 +821,11 @@ retry: /* Timed wait */ remaining = wait_event_timeout( kbdev->csf.event_wait, - (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) - == CS_ACK_STATE_STOP), - CSF_STATE_WAIT_TIMEOUT_MS); + (CS_ACK_STATE_GET( + kbase_csf_firmware_cs_output( + stream, CS_ACK)) == + CS_ACK_STATE_STOP), + kbdev->csf.fw_timeout_ms); if (!remaining) { dev_warn(kbdev->dev, @@ -769,21 +845,6 @@ retry: return err; } -static int wait_gpu_reset(struct kbase_device *kbdev) -{ - int ret = 0; - - lockdep_assert_held(&kbdev->csf.scheduler.lock); - - while (kbase_reset_gpu_is_active(kbdev) && !ret) { - mutex_unlock(&kbdev->csf.scheduler.lock); - ret = kbase_reset_gpu_wait(kbdev); - mutex_lock(&kbdev->csf.scheduler.lock); - } - - return ret; -} - int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue) { struct kbase_device *kbdev = queue->kctx->kbdev; @@ -794,14 +855,13 @@ int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue) if (WARN_ON(!group)) return -EINVAL; + kbase_reset_gpu_assert_failed_or_prevented(kbdev); lockdep_assert_held(&queue->kctx->csf.lock); mutex_lock(&kbdev->csf.scheduler.lock); queue->enabled = false; KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP, group, queue, cs_enabled); - wait_gpu_reset(kbdev); - if (cs_enabled && queue_group_scheduled_locked(group)) { struct kbase_csf_csg_slot *const csg_slot = kbdev->csf.scheduler.csg_slots; @@ -849,7 +909,7 @@ static void program_cs_extract_init(struct kbase_queue *queue) } static void program_cs(struct kbase_device *kbdev, - struct kbase_queue *queue) + struct kbase_queue *queue, bool ring_csg_doorbell) { struct kbase_queue_group *group = queue->group; struct kbase_csf_cmd_stream_group_info *ginfo; @@ -924,7 +984,8 @@ static void program_cs(struct kbase_device *kbdev, KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_START, group, queue, queue->enabled); - kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr); + kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr, + ring_csg_doorbell); update_hw_active(queue, true); } @@ -936,6 +997,7 @@ int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) int err = 0; bool evicted = false; + kbase_reset_gpu_assert_prevented(kbdev); lockdep_assert_held(&queue->kctx->csf.lock); if (WARN_ON(!group || queue->bind_state != KBASE_CSF_QUEUE_BOUND)) @@ -944,12 +1006,8 @@ int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) mutex_lock(&kbdev->csf.scheduler.lock); KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_START, group, queue, group->run_state); - err = wait_gpu_reset(kbdev); - if (err) { - dev_warn(kbdev->dev, "Unsuccessful GPU reset detected when kicking queue (csi_index=%d) of group %d", - queue->csi_index, group->handle); - } else if (group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) { + if (group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) { err = -EIO; evicted = true; } else if ((group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) @@ -977,7 +1035,7 @@ int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) */ kbase_csf_ring_cs_user_doorbell(kbdev, queue); } else - program_cs(kbdev, queue); + program_cs(kbdev, queue, true); } queue_delayed_work(system_long_wq, &kbdev->csf.scheduler.ping_work, @@ -1101,7 +1159,7 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) /* When in transition, wait for it to complete */ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { long remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); dev_dbg(kbdev->dev, "slot %d wait for up-running\n", slot); remaining = wait_event_timeout(kbdev->csf.event_wait, @@ -1230,6 +1288,13 @@ bool save_slot_cs(struct kbase_csf_cmd_stream_group_info const *const ginfo, queue->sync_value = kbase_csf_firmware_cs_output(stream, CS_STATUS_WAIT_SYNC_VALUE); + queue->sb_status = CS_STATUS_SCOREBOARDS_NONZERO_GET( + kbase_csf_firmware_cs_output(stream, + CS_STATUS_SCOREBOARDS)); + queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_GET( + kbase_csf_firmware_cs_output(stream, + CS_STATUS_BLOCKED_REASON)); + if (!evaluate_sync_update(queue)) { is_waiting = true; } else { @@ -1321,7 +1386,7 @@ void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); - if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + if (WARN_ON(group->priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) return; group->run_state = run_state; @@ -1346,8 +1411,7 @@ void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, scheduler->state == SCHED_SUSPENDED)) { dev_dbg(kbdev->dev, "Kicking scheduler on first runnable group\n"); /* Fire a scheduling to start the time-slice */ - mod_delayed_work(kbdev->csf.scheduler.wq, - &kbdev->csf.scheduler.tick_work, 0); + enqueue_tick_work(kbdev); } else schedule_in_cycle(group, false); @@ -1404,11 +1468,12 @@ void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, WARN_ON(scheduler->total_runnable_grps == 0); scheduler->total_runnable_grps--; - if (!scheduler->total_runnable_grps && - scheduler->state != SCHED_SUSPENDED) { + if (!scheduler->total_runnable_grps) { dev_dbg(kctx->kbdev->dev, "Scheduler idle has no runnable groups"); - mod_delayed_work(system_wq, &scheduler->gpu_idle_work, - GPU_IDLE_POWEROFF_HYSTERESIS_DELAY); + cancel_tick_timer(kctx->kbdev); + WARN_ON(atomic_read(&scheduler->non_idle_offslot_grps)); + if (scheduler->state != SCHED_SUSPENDED) + queue_work(system_wq, &scheduler->gpu_idle_work); } KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, scheduler->num_active_address_spaces | @@ -1496,13 +1561,38 @@ static void update_offslot_non_idle_cnt_on_grp_suspend( } } -static bool confirm_cs_idle(struct kbase_queue *queue) +static bool confirm_cmd_buf_empty(struct kbase_queue *queue) { + bool cs_empty; + bool cs_idle; + u32 sb_status = 0; + + struct kbase_device const *const kbdev = queue->group->kctx->kbdev; + struct kbase_csf_global_iface const *const iface = + &kbdev->csf.global_iface; + + u32 glb_version = iface->version; + u64 *input_addr = (u64 *)queue->user_io_addr; u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); - return (input_addr[CS_INSERT_LO / sizeof(u64)] == - output_addr[CS_EXTRACT_LO / sizeof(u64)]); + if (glb_version >= kbase_csf_interface_version(1, 0, 0)) { + /* CS_STATUS_SCOREBOARD supported from CSF 1.0 */ + struct kbase_csf_cmd_stream_group_info const *const ginfo = + &kbdev->csf.global_iface.groups[queue->group->csg_nr]; + struct kbase_csf_cmd_stream_info const *const stream = + &ginfo->streams[queue->csi_index]; + + sb_status = CS_STATUS_SCOREBOARDS_NONZERO_GET( + kbase_csf_firmware_cs_output(stream, + CS_STATUS_SCOREBOARDS)); + } + + cs_empty = (input_addr[CS_INSERT_LO / sizeof(u64)] == + output_addr[CS_EXTRACT_LO / sizeof(u64)]); + cs_idle = cs_empty && (!sb_status); + + return cs_idle; } static void save_csg_slot(struct kbase_queue_group *group) @@ -1524,51 +1614,53 @@ static void save_csg_slot(struct kbase_queue_group *group) if (!WARN_ON((state != CSG_ACK_STATE_SUSPEND) && (state != CSG_ACK_STATE_TERMINATE))) { - int i; + u32 max_streams = ginfo->stream_num; + u32 i; bool sync_wait = false; - bool idle = true; + bool idle = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_STATE) & + CSG_STATUS_STATE_IDLE_MASK; #ifdef CONFIG_MALI_NO_MALI - for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) + for (i = 0; i < max_streams; i++) update_hw_active(group->bound_queues[i], false); #endif - /* The CSG idle status on suspension will be simplified when - * MIDHARC_3065 is applied. Prior to it, we will have to - * evaluate per-CS at the best effort (i.e. in case the - * CS is executing some deffered instructions, the host - * has no way to be sure a CS is actually idle). However - * it is considered that the best effort approach is able to - * handle the tests from base, where the CS commands are - * generally simple ones. - * With the inclusion of support for MIDHARC_3064, the CSG - * run-state assertion previously in this function and its - * utility helpers need to be dropped. This is because the - * suspension actions can be triggered with the idle_worker - * thread asynchronously to the scheduler ticks. - */ - for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + for (i = 0; idle && i < max_streams; i++) { struct kbase_queue *const queue = - group->bound_queues[i]; + group->bound_queues[i]; - if (queue && queue->enabled) { - if (save_slot_cs(ginfo, queue)) - sync_wait = true; - else if (idle) - idle = confirm_cs_idle(queue); + if (!queue || !queue->enabled) + continue; + + if (save_slot_cs(ginfo, queue)) + sync_wait = true; + else { + /* Need to confirm if ringbuffer of the GPU + * queue is empty or not. A race can arise + * between the flush of GPU queue and suspend + * of CSG. If a queue is flushed after FW has + * set the IDLE bit in CSG_STATUS_STATE, then + * Scheduler will incorrectly consider CSG + * as idle. And there may not be any further + * flush call for the GPU queue, which would + * have de-idled the CSG. + */ + idle = confirm_cmd_buf_empty(queue); } } - /* Take the suspended group out of the runnable_groups - * list of the context and move it to the - * idle_wait_groups list. - */ - if (sync_wait && idle) - deschedule_idle_wait_group(scheduler, group); - else if (idle) { - group->run_state = - KBASE_CSF_GROUP_SUSPENDED_ON_IDLE; - dev_dbg(kbdev->dev, "Group-%d suspended: idle\n", - group->handle); + if (idle) { + /* Take the suspended group out of the runnable_groups + * list of the context and move it to the + * idle_wait_groups list. + */ + if (sync_wait) + deschedule_idle_wait_group(scheduler, group); + else { + group->run_state = + KBASE_CSF_GROUP_SUSPENDED_ON_IDLE; + dev_dbg(kbdev->dev, "Group-%d suspended: idle", + group->handle); + } } else { group->run_state = KBASE_CSF_GROUP_SUSPENDED; } @@ -1623,6 +1715,7 @@ static bool cleanup_csg_slot(struct kbase_queue_group *group) /* The csg does not need cleanup other than drop its AS */ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); as_fault = kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT); + WARN_ON(kctx->mmu_flush_pend_state != KCTX_MMU_FLUSH_NOT_PEND); kbase_ctx_sched_release_ctx(kctx); if (unlikely(group->faulted)) as_fault = true; @@ -1749,6 +1842,7 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_ctx_sched_retain_ctx(kctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_mmu_deferred_flush_invalidate(kctx); mutex_unlock(&kbdev->mmu_hw_mutex); if (kctx->as_nr == KBASEP_AS_NR_INVALID) { @@ -1770,7 +1864,7 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, struct kbase_queue *queue = group->bound_queues[i]; if (queue) - program_cs(kbdev, queue); + program_cs(kbdev, queue, false); } @@ -1922,8 +2016,7 @@ static void sched_evict_group(struct kbase_queue_group *group, bool fault, static int term_group_sync(struct kbase_queue_group *group) { struct kbase_device *kbdev = group->kctx->kbdev; - long remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); int err = 0; term_csg_slot(group); @@ -1950,21 +2043,20 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) kbase_csf_timeout_in_jiffies(CSG_SCHED_STOP_TIMEOUT_MS); bool force = false; + kbase_reset_gpu_assert_failed_or_prevented(kbdev); lockdep_assert_held(&group->kctx->csf.lock); mutex_lock(&scheduler->lock); KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_DESCHEDULE, group, group->run_state); while (queue_group_scheduled_locked(group)) { u32 saved_state = scheduler->state; - bool reset = kbase_reset_gpu_is_active(kbdev); if (!kbasep_csf_scheduler_group_is_on_slot_locked(group)) { sched_evict_group(group, false, true); - } else if (reset || saved_state == SCHED_INACTIVE || force) { + } else if (saved_state == SCHED_INACTIVE || force) { bool as_faulty; - if (!reset) - term_group_sync(group); + term_group_sync(group); /* Treat the csg been terminated */ as_faulty = cleanup_csg_slot(group); /* remove from the scheduler list */ @@ -2049,7 +2141,8 @@ static inline void set_max_csg_slots(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; - unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; + unsigned int max_address_space_slots = + kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; WARN_ON(scheduler->num_active_address_spaces > total_csg_slots); @@ -2073,7 +2166,8 @@ static inline void count_active_address_space(struct kbase_device *kbdev, { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; - unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; + unsigned int max_address_space_slots = + kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; if (scheduler->ngrp_to_schedule <= total_csg_slots) { if (kctx->csf.sched.ngrp_to_schedule == 1) { @@ -2255,11 +2349,13 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS); DECLARE_BITMAP(evicted_mask, MAX_SUPPORTED_CSGS) = {0}; bool suspend_wait_failed = false; - long remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); lockdep_assert_held(&kbdev->csf.scheduler.lock); + /* In the current implementation, csgs_events_enable_mask would be used + * only to indicate suspending CSGs. + */ bitmap_complement(slot_mask, scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); @@ -2304,13 +2400,49 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) program_vacant_csg_slot(kbdev, (s8)i); } } else { - dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", - num_groups, slot_mask); + u32 i; + + /* Groups that have failed to suspend in time shall + * raise a fatal error as they could no longer be + * safely resumed. + */ + for_each_set_bit(i, slot_mask, num_groups) { + struct kbase_queue_group *const group = + scheduler->csg_slots[i].resident_group; + + struct base_gpu_queue_group_error const + err_payload = { .error_type = + BASE_GPU_QUEUE_GROUP_ERROR_FATAL, + .payload = { + .fatal_group = { + .status = + GPU_EXCEPTION_TYPE_SW_FAULT_2, + } } }; + + if (unlikely(group == NULL)) + continue; + + kbase_csf_add_group_fatal_error(group, + &err_payload); + kbase_event_wakeup(group->kctx); + + /* TODO GPUCORE-25328: The CSG can't be + * terminated, the GPU will be reset as a + * work-around. + */ + dev_warn( + kbdev->dev, + "Group %p on slot %u failed to suspend\n", + (void *)group, i); + + /* The group has failed suspension, stop + * further examination. + */ + clear_bit(i, slot_mask); + set_bit(i, scheduler->csgs_events_enable_mask); + } - if (kbase_prepare_to_reset_gpu(kbdev)) - kbase_reset_gpu(kbdev); suspend_wait_failed = true; - break; } } @@ -2318,7 +2450,7 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) dev_info(kbdev->dev, "Scheduler evicting slots: 0x%*pb\n", num_groups, evicted_mask); - if (unlikely(!suspend_wait_failed)) { + if (likely(!suspend_wait_failed)) { u32 i; while (scheduler->ngrp_to_schedule && @@ -2332,6 +2464,9 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) if (WARN_ON(!csg_slot_in_use(kbdev, (int)i))) break; } + } else { + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); } } @@ -2342,6 +2477,11 @@ static void suspend_queue_group(struct kbase_queue_group *group) &group->kctx->kbdev->csf.scheduler; spin_lock_irqsave(&scheduler->interrupt_lock, flags); + /* This shall be used in program_suspending_csg_slots() where we + * assume that whilst CSGs are being suspended, this bitmask is not + * used by anything else i.e., it indicates only the CSGs going + * through suspension. + */ clear_bit(group->csg_nr, scheduler->csgs_events_enable_mask); spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); @@ -2356,8 +2496,7 @@ static void wait_csg_slots_start(struct kbase_device *kbdev) { u32 num_groups = kbdev->csf.global_iface.group_num; struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - long remaining = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; u32 i; @@ -2400,48 +2539,32 @@ static void wait_csg_slots_start(struct kbase_device *kbdev) } /** - * group_on_slot_is_idle() - Check if the queue group resident on a CSG slot - * is idle. + * group_on_slot_is_idle() - Check if the given slot has a CSG-idle state + * flagged after the completion of a CSG status + * update command * * This function is called at the start of scheduling tick to check the * idle status of a queue group resident on a CSG slot. - * The group's idleness is determined by looping over all the bound command - * queues and checking their respective CS_STATUS_WAIT register as well as - * the insert and extract offsets. - * - * This function would be simplified in future after the changes under - * consideration with MIDHARC-3065 are introduced. + * The caller must make sure the corresponding status update command has + * been called and completed before checking this status. * * @kbdev: Pointer to the GPU device. - * @group: Pointer to the resident group on the given slot. - * @slot: The slot that the given group is resident on. + * @slot: The given slot for checking an occupying resident group's idle + * state. * * Return: true if the group resident on slot is idle, otherwise false. */ static bool group_on_slot_is_idle(struct kbase_device *kbdev, - struct kbase_queue_group *group, unsigned long slot) + unsigned long slot) { struct kbase_csf_cmd_stream_group_info *ginfo = &kbdev->csf.global_iface.groups[slot]; - u32 i; + bool idle = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_STATE) & + CSG_STATUS_STATE_IDLE_MASK; lockdep_assert_held(&kbdev->csf.scheduler.lock); - for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { - struct kbase_queue *queue = group->bound_queues[i]; - - if (queue && queue->enabled) { - struct kbase_csf_cmd_stream_info *stream = - &ginfo->streams[queue->csi_index]; - u32 status = kbase_csf_firmware_cs_output(stream, - CS_STATUS_WAIT); - if (!CS_STATUS_WAIT_SYNC_WAIT_GET(status) && - !confirm_cs_idle(group->bound_queues[i])) - return false; - } - } - - return true; + return idle; } /** @@ -2537,8 +2660,7 @@ static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) { unsigned long *slot_mask = kbdev->csf.scheduler.csg_slots_prio_update; - long wait_time = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long wait_time = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); int ret = wait_csg_slots_handshake_ack(kbdev, CSG_REQ_EP_CFG_MASK, slot_mask, wait_time); @@ -2549,8 +2671,10 @@ static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) * issue, no major consequences are expected as a * result, so just warn the case. */ - dev_warn(kbdev->dev, "Timeout, skipping the update wait: slot mask=0x%lx", - slot_mask[0]); + dev_warn( + kbdev->dev, + "Timeout on CSG_REQ:EP_CFG, skipping the update wait: slot mask=0x%lx", + slot_mask[0]); } } @@ -2562,18 +2686,27 @@ void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, u32 num_groups = kbdev->csf.global_iface.group_num; u32 slot; DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; - DECLARE_BITMAP(terminated_slot_mask, MAX_SUPPORTED_CSGS); - long remaining = - kbase_csf_timeout_in_jiffies(DEFAULT_RESET_TIMEOUT_MS); lockdep_assert_held(&kctx->csf.lock); mutex_lock(&scheduler->lock); + /* This code is only called during reset, so we don't wait for the CSG + * slots to be stopped + */ + WARN_ON(!kbase_reset_gpu_is_active(kbdev)); + KBASE_KTRACE_ADD(kbdev, EVICT_CTX_SLOTS, kctx, 0u); for (slot = 0; slot < num_groups; slot++) { group = kbdev->csf.scheduler.csg_slots[slot].resident_group; if (group && group->kctx == kctx) { + bool as_fault; + term_csg_slot(group); + as_fault = cleanup_csg_slot(group); + /* remove the group from the scheduler list */ + sched_evict_group(group, as_fault, false); + /* return the evicted group to the caller */ + list_add_tail(&group->link, evicted_groups); set_bit(slot, slot_mask); } } @@ -2581,48 +2714,6 @@ void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, dev_info(kbdev->dev, "Evicting context %d_%d slots: 0x%*pb\n", kctx->tgid, kctx->id, num_groups, slot_mask); - bitmap_copy(terminated_slot_mask, slot_mask, MAX_SUPPORTED_CSGS); - /* Only check for GPU reset once - this thread has the scheduler lock, - * so even if the return value of kbase_reset_gpu_is_active changes, - * no reset work would be done anyway until the scheduler lock was - * released. - */ - if (!kbase_reset_gpu_is_active(kbdev)) { - while (remaining - && !bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { - DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); - - bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); - - remaining = wait_event_timeout(kbdev->csf.event_wait, - slots_state_changed(kbdev, changed, - csg_slot_stopped_raw), - remaining); - - if (remaining) - bitmap_andnot(slot_mask, slot_mask, changed, - MAX_SUPPORTED_CSGS); - } - } - - for_each_set_bit(slot, terminated_slot_mask, num_groups) { - bool as_fault; - - group = scheduler->csg_slots[slot].resident_group; - as_fault = cleanup_csg_slot(group); - /* remove the group from the scheduler list */ - sched_evict_group(group, as_fault, false); - /* return the evicted group to the caller */ - list_add_tail(&group->link, evicted_groups); - } - - if (!remaining) { - dev_warn(kbdev->dev, "Timeout on evicting ctx slots: 0x%*pb\n", - num_groups, slot_mask); - if (kbase_prepare_to_reset_gpu(kbdev)) - kbase_reset_gpu(kbdev); - } - mutex_unlock(&scheduler->lock); } @@ -2861,7 +2952,7 @@ static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, lockdep_assert_held(&scheduler->lock); if (WARN_ON(priority < 0) || - WARN_ON(priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + WARN_ON(priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) return; if (!kctx_as_enabled(kctx)) @@ -3044,15 +3135,17 @@ static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, /* The groups are aggregated into a single kernel doorbell request */ if (!bitmap_empty(csg_bitmap, num_groups)) { long wt = - kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); u32 db_slots = (u32)csg_bitmap[0]; kbase_csf_ring_csg_slots_doorbell(kbdev, db_slots); if (wait_csg_slots_handshake_ack(kbdev, CSG_REQ_STATUS_UPDATE_MASK, csg_bitmap, wt)) { - dev_warn(kbdev->dev, "Timeout, treat groups as not idle: slot mask=0x%lx", - csg_bitmap[0]); + dev_warn( + kbdev->dev, + "Timeout on CSG_REQ:STATUS_UPDATE, treat groups as not idle: slot mask=0x%lx", + csg_bitmap[0]); /* Store the bitmap of timed out slots */ bitmap_copy(failed_csg_bitmap, csg_bitmap, num_groups); @@ -3073,9 +3166,9 @@ static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, * which idle notification was received previously, i.e. all the CSG slots * present in the bitmap scheduler->csg_slots_idle_mask. * The confirmation is done by sending the CSG status update request to the - * firmware. The idleness of a CSG is determined by looping over all the - * bound CSs and checking their respective CS_STATUS_WAIT register - * as well as the insert and extract offset. + * firmware. On completion, the firmware will mark the idleness at the + * slot's interface CSG_STATUS_STATE register accordingly. + * * The run state of the groups resident on still idle CSG slots is changed to * KBASE_CSF_GROUP_IDLE and the bitmap scheduler->csg_slots_idle_mask is * updated accordingly. @@ -3108,10 +3201,10 @@ static void scheduler_handle_idle_slots(struct kbase_device *kbdev) continue; if (WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE)) continue; - if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + if (WARN_ON(group->priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) continue; - if (group_on_slot_is_idle(kbdev, group, i)) { + if (group_on_slot_is_idle(kbdev, i)) { group->run_state = KBASE_CSF_GROUP_IDLE; set_bit(i, scheduler->csg_slots_idle_mask); } @@ -3166,7 +3259,7 @@ static struct kbase_queue_group *get_tock_top_group( int i; lockdep_assert_held(&scheduler->lock); - for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { + for (i = 0; i < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) { struct kbase_queue_group *group; @@ -3222,7 +3315,8 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, static bool scheduler_idle_suspendable(struct kbase_device *kbdev) { - bool suspendable; + bool suspend; + unsigned long flags; struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; lockdep_assert_held(&scheduler->lock); @@ -3230,32 +3324,39 @@ static bool scheduler_idle_suspendable(struct kbase_device *kbdev) if (scheduler->state == SCHED_SUSPENDED) return false; - if (scheduler->total_runnable_grps) { -#if USE_PRE_MIDHARC_3065_WORKAROUND - suspendable = false; -#else - unsigned long flags; + /* Work around for TTUX, skip scheduler suspend on idle-groups. + * ToDo: GPUCORE-26768, or its descendants after root cause + * investigation, needs to revert this WA. + */ + if (scheduler->total_runnable_grps && + (kbdev->gpu_props.props.raw_props.gpu_id & + GPU_ID2_PRODUCT_MODEL) == GPU_ID2_PRODUCT_TTUX) { + dev_dbg(kbdev->dev, "GPU: TTUX, skipping idle suspend"); + return false; + } - spin_lock_irqsave(&scheduler->interrupt_lock, flags); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (scheduler->total_runnable_grps) { + spin_lock(&scheduler->interrupt_lock); /* Check both on-slots and off-slots groups idle status */ - suspendable = kbase_csf_scheduler_all_csgs_idle(kbdev) && - !atomic_read(&scheduler->non_idle_offslot_grps); + suspend = kbase_csf_scheduler_all_csgs_idle(kbdev) && + !atomic_read(&scheduler->non_idle_offslot_grps) && + kbase_pm_idle_groups_sched_suspendable(kbdev); - spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -#endif - } else { - /* No runnables inside the scheduler, can suspend */ - suspendable = true; - } + spin_unlock(&scheduler->interrupt_lock); + } else + suspend = kbase_pm_no_runnables_sched_suspendable(kbdev); - return suspendable; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return suspend; } static void gpu_idle_worker(struct work_struct *work) { struct kbase_device *kbdev = container_of( - work, struct kbase_device, csf.scheduler.gpu_idle_work.work); + work, struct kbase_device, csf.scheduler.gpu_idle_work); struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; mutex_lock(&scheduler->lock); @@ -3269,12 +3370,12 @@ static void gpu_idle_worker(struct work_struct *work) if (!ret) { dev_dbg(kbdev->dev, "Scheduler becomes idle suspended now"); scheduler_suspend(kbdev); + cancel_tick_timer(kbdev); } else { dev_dbg(kbdev->dev, "Aborting suspend scheduler (grps: %d)", atomic_read(&scheduler->non_idle_offslot_grps)); /* Bring forward the next tick */ - mod_delayed_work(scheduler->wq, - &scheduler->tick_work, 0); + kbase_csf_scheduler_advance_tick(kbdev); } } @@ -3310,7 +3411,7 @@ static int scheduler_prepare(struct kbase_device *kbdev) bitmap_zero(scheduler->csg_slots_prio_update, MAX_SUPPORTED_CSGS); /* Scan out to run groups */ - for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { + for (i = 0; i < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { struct kbase_context *kctx; list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) @@ -3343,7 +3444,7 @@ static int scheduler_prepare(struct kbase_device *kbdev) static void scheduler_wait_protm_quit(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - long wt = kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long wt = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); long remaining; lockdep_assert_held(&scheduler->lock); @@ -3387,6 +3488,7 @@ static void schedule_actions(struct kbase_device *kbdev) struct kbase_queue_group *protm_grp; int ret; + kbase_reset_gpu_assert_prevented(kbdev); lockdep_assert_held(&scheduler->lock); ret = kbase_pm_wait_for_desired_state(kbdev); @@ -3413,6 +3515,7 @@ static void schedule_actions(struct kbase_device *kbdev) if (protm_grp && scheduler->top_grp == protm_grp) { dev_dbg(kbdev->dev, "Scheduler keep protm exec: group-%d", protm_grp->handle); + atomic_dec(&scheduler->non_idle_offslot_grps); } else if (scheduler->top_grp) { if (protm_grp) dev_dbg(kbdev->dev, "Scheduler drop protm exec: group-%d", @@ -3470,13 +3573,16 @@ static void schedule_on_tock(struct work_struct *work) csf.scheduler.tock_work.work); struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - mutex_lock(&scheduler->lock); - - if (kbase_reset_gpu_is_active(kbdev) || - (scheduler->state == SCHED_SUSPENDED)) { - mutex_unlock(&scheduler->lock); + int err = kbase_reset_gpu_try_prevent(kbdev); + /* Regardless of whether reset failed or is currently happening, exit + * early + */ + if (err) return; - } + + mutex_lock(&scheduler->lock); + if (scheduler->state == SCHED_SUSPENDED) + goto exit_no_schedule_unlock; WARN_ON(!(scheduler->state == SCHED_INACTIVE)); scheduler->state = SCHED_BUSY; @@ -3493,25 +3599,36 @@ static void schedule_on_tock(struct work_struct *work) scheduler->state = SCHED_INACTIVE; mutex_unlock(&scheduler->lock); + kbase_reset_gpu_allow(kbdev); dev_dbg(kbdev->dev, "Waking up for event after schedule-on-tock completes."); wake_up_all(&kbdev->csf.event_wait); + return; + +exit_no_schedule_unlock: + mutex_unlock(&scheduler->lock); + kbase_reset_gpu_allow(kbdev); } static void schedule_on_tick(struct work_struct *work) { struct kbase_device *kbdev = container_of(work, struct kbase_device, - csf.scheduler.tick_work.work); + csf.scheduler.tick_work); struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + int err = kbase_reset_gpu_try_prevent(kbdev); + /* Regardless of whether reset failed or is currently happening, exit + * early + */ + if (err) + return; + mutex_lock(&scheduler->lock); - if (kbase_reset_gpu_is_active(kbdev) || - (scheduler->state == SCHED_SUSPENDED)) { - mutex_unlock(&scheduler->lock); - return; - } + WARN_ON(scheduler->tick_timer_active); + if (scheduler->state == SCHED_SUSPENDED) + goto exit_no_schedule_unlock; scheduler->state = SCHED_BUSY; /* Do scheduling stuff */ @@ -3527,17 +3644,23 @@ static void schedule_on_tick(struct work_struct *work) /* Kicking next scheduling if needed */ if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && (scheduler->total_runnable_grps > 0)) { - mod_delayed_work(scheduler->wq, &scheduler->tick_work, - CSF_SCHEDULER_TIME_TICK_JIFFIES); - dev_dbg(kbdev->dev, "scheduling for next tick, num_runnable_groups:%u\n", + start_tick_timer(kbdev); + dev_dbg(kbdev->dev, + "scheduling for next tick, num_runnable_groups:%u\n", scheduler->total_runnable_grps); } scheduler->state = SCHED_INACTIVE; mutex_unlock(&scheduler->lock); + kbase_reset_gpu_allow(kbdev); dev_dbg(kbdev->dev, "Waking up for event after schedule-on-tick completes."); wake_up_all(&kbdev->csf.event_wait); + return; + +exit_no_schedule_unlock: + mutex_unlock(&scheduler->lock); + kbase_reset_gpu_allow(kbdev); } int wait_csg_slots_suspend(struct kbase_device *kbdev, @@ -3618,8 +3741,7 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, } } - ret = wait_csg_slots_suspend(kbdev, slot_mask, - CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS); + ret = wait_csg_slots_suspend(kbdev, slot_mask, kbdev->reset_timeout_ms); return ret; } @@ -3652,7 +3774,7 @@ static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) */ kbase_gpu_start_cache_clean(kbdev); ret2 = kbase_gpu_wait_cache_clean_timeout(kbdev, - DEFAULT_RESET_TIMEOUT_MS); + kbdev->reset_timeout_ms); if (ret2) { dev_warn(kbdev->dev, "Timed out waiting for cache clean to complete before reset"); if (!ret) @@ -3673,8 +3795,9 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) WARN_ON(csgs_active(kbdev)); /* Cancel any potential queued delayed work(s) */ - cancel_delayed_work_sync(&kbdev->csf.scheduler.gpu_idle_work); - cancel_delayed_work_sync(&scheduler->tick_work); + cancel_work_sync(&kbdev->csf.scheduler.gpu_idle_work); + cancel_tick_timer(kbdev); + cancel_work_sync(&scheduler->tick_work); cancel_delayed_work_sync(&scheduler->tock_work); cancel_delayed_work_sync(&scheduler->ping_work); @@ -3751,10 +3874,23 @@ static void firmware_aliveness_monitor(struct work_struct *work) csf.scheduler.ping_work.work); int err; - /* Get the scheduler mutex to ensure that reset will not change while - * this function is being executed as otherwise calling kbase_reset_gpu - * when reset is already occurring is a programming error. + /* Ensure that reset will not be occurring while this function is being + * executed as otherwise calling kbase_reset_gpu when reset is already + * occurring is a programming error. + * + * We must use the 'try' variant as the Reset worker can try to flush + * this workqueue, which would otherwise deadlock here if we tried to + * wait for the reset (and thus ourselves) to complete. */ + err = kbase_reset_gpu_try_prevent(kbdev); + if (err) { + /* It doesn't matter whether the value was -EAGAIN or a fatal + * error, just stop processing. In case of -EAGAIN, the Reset + * worker will restart the scheduler later to resume ping + */ + return; + } + mutex_lock(&kbdev->csf.scheduler.lock); #ifdef CONFIG_MALI_DEBUG @@ -3767,9 +3903,6 @@ static void firmware_aliveness_monitor(struct work_struct *work) if (kbdev->csf.scheduler.state == SCHED_SUSPENDED) goto exit; - if (kbase_reset_gpu_is_active(kbdev)) - goto exit; - if (get_nr_active_csgs(kbdev) != 1) goto exit; @@ -3787,6 +3920,17 @@ static void firmware_aliveness_monitor(struct work_struct *work) err = kbase_csf_firmware_ping(kbdev); if (err) { + /* FW not responding means hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error( + &kbdev->hwcnt_gpu_iface); + + /* It is acceptable to enqueue a reset whilst we've prevented + * them, it will happen after we've allowed them again + */ if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } else if (get_nr_active_csgs(kbdev) == 1) { @@ -3798,6 +3942,8 @@ static void firmware_aliveness_monitor(struct work_struct *work) kbase_pm_context_idle(kbdev); exit: mutex_unlock(&kbdev->csf.scheduler.lock); + kbase_reset_gpu_allow(kbdev); + return; } int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, @@ -3806,18 +3952,12 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, struct kbase_context *const kctx = group->kctx; struct kbase_device *const kbdev = kctx->kbdev; struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - int err; + int err = 0; + kbase_reset_gpu_assert_prevented(kbdev); lockdep_assert_held(&kctx->csf.lock); mutex_lock(&scheduler->lock); - err = wait_gpu_reset(kbdev); - if (err) { - dev_warn(kbdev->dev, "Error while waiting for the GPU reset to complete when suspending group %d on slot %d", - group->handle, group->csg_nr); - goto exit; - } - if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; @@ -3826,7 +3966,7 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) suspend_queue_group(group); err = wait_csg_slots_suspend(kbdev, slot_mask, - CSF_STATE_WAIT_TIMEOUT_MS); + kbdev->csf.fw_timeout_ms); if (err) { dev_warn(kbdev->dev, "Timed out waiting for the group %d to suspend on slot %d", group->handle, group->csg_nr); @@ -3999,6 +4139,13 @@ void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group) struct kbase_device *const kbdev = group->kctx->kbdev; struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + int err = kbase_reset_gpu_try_prevent(kbdev); + /* Regardless of whether reset failed or is currently happening, exit + * early + */ + if (err) + return; + mutex_lock(&scheduler->lock); /* Check if the group is now eligible for execution in protected mode @@ -4013,6 +4160,7 @@ void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group) } mutex_unlock(&scheduler->lock); + kbase_reset_gpu_allow(kbdev); } /** @@ -4074,7 +4222,7 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) int priority; int err; - for (priority = 0; priority < BASE_QUEUE_GROUP_PRIORITY_COUNT; + for (priority = 0; priority < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++priority) { INIT_LIST_HEAD(&kctx->csf.sched.runnable_groups[priority]); } @@ -4142,11 +4290,11 @@ int kbase_csf_scheduler_init(struct kbase_device *kbdev) return -ENOMEM; } - INIT_DEFERRABLE_WORK(&scheduler->tick_work, schedule_on_tick); + INIT_WORK(&scheduler->tick_work, schedule_on_tick); INIT_DEFERRABLE_WORK(&scheduler->tock_work, schedule_on_tock); INIT_DEFERRABLE_WORK(&scheduler->ping_work, firmware_aliveness_monitor); - BUILD_BUG_ON(GLB_REQ_WAIT_TIMEOUT_MS >= FIRMWARE_PING_INTERVAL_MS); + BUILD_BUG_ON(CSF_FIRMWARE_TIMEOUT_MS >= FIRMWARE_PING_INTERVAL_MS); mutex_init(&scheduler->lock); spin_lock_init(&scheduler->interrupt_lock); @@ -4170,11 +4318,16 @@ int kbase_csf_scheduler_init(struct kbase_device *kbdev) scheduler->active_protm_grp = NULL; scheduler->gpu_idle_fw_timer_enabled = false; scheduler->apply_async_protm = false; + scheduler->csg_scheduling_period_ms = CSF_SCHEDULER_TIME_TICK_MS; scheduler_doorbell_init(kbdev); - INIT_DEFERRABLE_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); + INIT_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); atomic_set(&scheduler->non_idle_offslot_grps, 0); + hrtimer_init(&scheduler->tick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + scheduler->tick_timer.function = tick_timer_callback; + scheduler->tick_timer_active = false; + return 0; } @@ -4183,9 +4336,14 @@ void kbase_csf_scheduler_term(struct kbase_device *kbdev) if (kbdev->csf.scheduler.csg_slots) { WARN_ON(atomic_read(&kbdev->csf.scheduler.non_idle_offslot_grps)); WARN_ON(csgs_active(kbdev)); - cancel_delayed_work_sync(&kbdev->csf.scheduler.gpu_idle_work); + flush_work(&kbdev->csf.scheduler.gpu_idle_work); + mutex_lock(&kbdev->csf.scheduler.lock); + if (WARN_ON(kbdev->csf.scheduler.state != SCHED_SUSPENDED)) + scheduler_suspend(kbdev); + mutex_unlock(&kbdev->csf.scheduler.lock); cancel_delayed_work_sync(&kbdev->csf.scheduler.ping_work); - cancel_delayed_work_sync(&kbdev->csf.scheduler.tick_work); + cancel_tick_timer(kbdev); + cancel_work_sync(&kbdev->csf.scheduler.tick_work); cancel_delayed_work_sync(&kbdev->csf.scheduler.tock_work); destroy_workqueue(kbdev->csf.scheduler.wq); mutex_destroy(&kbdev->csf.scheduler.lock); @@ -4217,8 +4375,10 @@ static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev) (scheduler->state != SCHED_SUSPENDED)); if (scheduler->total_runnable_grps > 0) { - mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); + enqueue_tick_work(kbdev); dev_dbg(kbdev->dev, "Re-enabling the scheduler timer\n"); + } else if (scheduler->state != SCHED_SUSPENDED) { + queue_work(system_wq, &scheduler->gpu_idle_work); } } @@ -4252,16 +4412,19 @@ void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, currently_enabled = scheduler_timer_is_enabled_nolock(kbdev); if (currently_enabled && !enable) { scheduler->timer_enabled = false; - - cancel_delayed_work(&scheduler->tick_work); + cancel_tick_timer(kbdev); cancel_delayed_work(&scheduler->tock_work); + mutex_unlock(&scheduler->lock); + /* The non-sync version to cancel the normal work item is not + * available, so need to drop the lock before cancellation. + */ + cancel_work_sync(&scheduler->tick_work); } else if (!currently_enabled && enable) { scheduler->timer_enabled = true; scheduler_enable_tick_timer_nolock(kbdev); + mutex_unlock(&scheduler->lock); } - - mutex_unlock(&scheduler->lock); } void kbase_csf_scheduler_kick(struct kbase_device *kbdev) @@ -4274,7 +4437,7 @@ void kbase_csf_scheduler_kick(struct kbase_device *kbdev) goto out; if (scheduler->total_runnable_grps > 0) { - mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); + enqueue_tick_work(kbdev); dev_dbg(kbdev->dev, "Kicking the scheduler manually\n"); } @@ -4287,7 +4450,7 @@ void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev) struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; /* Cancel any potential queued delayed work(s) */ - cancel_delayed_work_sync(&scheduler->tick_work); + cancel_work_sync(&scheduler->tick_work); cancel_delayed_work_sync(&scheduler->tock_work); mutex_lock(&scheduler->lock); @@ -4298,6 +4461,7 @@ void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev) suspend_active_groups_on_powerdown(kbdev, true); dev_info(kbdev->dev, "Scheduler PM suspend"); scheduler_suspend(kbdev); + cancel_tick_timer(kbdev); } mutex_unlock(&scheduler->lock); } diff --git a/mali_kbase/csf/mali_kbase_csf_scheduler.h b/mali_kbase/csf/mali_kbase_csf_scheduler.h index 24103b7..6e42c6e 100644 --- a/mali_kbase/csf/mali_kbase_csf_scheduler.h +++ b/mali_kbase/csf/mali_kbase_csf_scheduler.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_SCHEDULER_H_ @@ -405,4 +424,46 @@ static inline bool kbase_csf_scheduler_all_csgs_idle(struct kbase_device *kbdev) kbdev->csf.global_iface.group_num); } +/** + * kbase_csf_scheduler_advance_tick_nolock() - Advance the scheduling tick + * + * @kbdev: Pointer to the device + * + * This function advances the scheduling tick by enqueing the tick work item for + * immediate execution, but only if the tick hrtimer is active. If the timer + * is inactive then the tick work item is already in flight. + * The caller must hold the interrupt lock. + */ +static inline void +kbase_csf_scheduler_advance_tick_nolock(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->interrupt_lock); + + if (scheduler->tick_timer_active) { + scheduler->tick_timer_active = false; + queue_work(scheduler->wq, &scheduler->tick_work); + } +} + +/** + * kbase_csf_scheduler_advance_tick() - Advance the scheduling tick + * + * @kbdev: Pointer to the device + * + * This function advances the scheduling tick by enqueing the tick work item for + * immediate execution, but only if the tick hrtimer is active. If the timer + * is inactive then the tick work item is already in flight. + */ +static inline void kbase_csf_scheduler_advance_tick(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + kbase_csf_scheduler_advance_tick_nolock(kbdev); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); +} + #endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/mali_kbase/csf/mali_kbase_csf_tiler_heap.c b/mali_kbase/csf/mali_kbase_csf_tiler_heap.c index a3c9826..2da7a7d 100644 --- a/mali_kbase/csf/mali_kbase_csf_tiler_heap.c +++ b/mali_kbase/csf/mali_kbase_csf_tiler_heap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -20,6 +21,8 @@ * */ +#include <tl/mali_kbase_tracepoints.h> + #include "mali_kbase_csf_tiler_heap.h" #include "mali_kbase_csf_tiler_heap_def.h" #include "mali_kbase_csf_heap_context_alloc.h" @@ -337,6 +340,12 @@ static void delete_heap(struct kbase_csf_tiler_heap *heap) heap->gpu_va); list_del(&heap->link); + + WARN_ON(heap->chunk_count); + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(kctx->kbdev, kctx->id, + heap->heap_id, 0, 0, heap->max_chunks, heap->chunk_size, 0, + heap->target_in_flight, 0); + kfree(heap); } @@ -473,11 +482,20 @@ int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, list_first_entry(&heap->chunks_list, struct kbase_csf_tiler_heap_chunk, link); + kctx->csf.tiler_heaps.nr_of_heaps++; + heap->heap_id = kctx->csf.tiler_heaps.nr_of_heaps; list_add(&heap->link, &kctx->csf.tiler_heaps.list); *heap_gpu_va = heap->gpu_va; *first_chunk_va = first_chunk->gpu_va; + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( + kctx->kbdev, kctx->id, heap->heap_id, + PFN_UP(heap->chunk_size * heap->max_chunks), + PFN_UP(heap->chunk_size * heap->chunk_count), + heap->max_chunks, heap->chunk_size, heap->chunk_count, + heap->target_in_flight, 0); + dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", heap->gpu_va); } @@ -582,6 +600,13 @@ int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, new_chunk_ptr); } + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( + kctx->kbdev, kctx->id, heap->heap_id, + PFN_UP(heap->chunk_size * heap->max_chunks), + PFN_UP(heap->chunk_size * heap->chunk_count), heap->max_chunks, + heap->chunk_size, heap->chunk_count, heap->target_in_flight, + nr_in_flight); + mutex_unlock(&kctx->csf.tiler_heaps.lock); return err; diff --git a/mali_kbase/csf/mali_kbase_csf_tiler_heap.h b/mali_kbase/csf/mali_kbase_csf_tiler_heap.h index d85ac11..13d0564 100644 --- a/mali_kbase/csf/mali_kbase_csf_tiler_heap.h +++ b/mali_kbase/csf/mali_kbase_csf_tiler_heap.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_TILER_HEAP_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.c b/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.c index 5d744b8..e38863c 100644 --- a/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.c +++ b/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.h b/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.h index 44c580d..c8f37fd 100644 --- a/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.h +++ b/mali_kbase/csf/mali_kbase_csf_tiler_heap_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_tiler_heap_def.h b/mali_kbase/csf/mali_kbase_csf_tiler_heap_def.h index 1f9e208..fa94dcb 100644 --- a/mali_kbase/csf/mali_kbase_csf_tiler_heap_def.h +++ b/mali_kbase/csf/mali_kbase_csf_tiler_heap_def.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_TILER_HEAP_DEF_H_ @@ -97,6 +116,8 @@ struct kbase_csf_tiler_heap_chunk { * @gpu_va: The GPU virtual address of the heap context structure that * was allocated for the firmware. This is also used to * uniquely identify the heap. + * @heap_id: Unique id representing the heap, assigned during heap + * initialization. * @chunks_list: Linked list of allocated chunks. */ struct kbase_csf_tiler_heap { @@ -107,6 +128,7 @@ struct kbase_csf_tiler_heap { u32 max_chunks; u16 target_in_flight; u64 gpu_va; + u64 heap_id; struct list_head chunks_list; }; #endif /* !_KBASE_CSF_TILER_HEAP_DEF_H_ */ diff --git a/mali_kbase/csf/mali_kbase_csf_timeout.c b/mali_kbase/csf/mali_kbase_csf_timeout.c index 43b63bd..1396899 100644 --- a/mali_kbase/csf/mali_kbase_csf_timeout.c +++ b/mali_kbase/csf/mali_kbase_csf_timeout.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/csf/mali_kbase_csf_timeout.h b/mali_kbase/csf/mali_kbase_csf_timeout.h index dc6f2f2..4ec113d 100644 --- a/mali_kbase/csf/mali_kbase_csf_timeout.h +++ b/mali_kbase/csf/mali_kbase_csf_timeout.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_TIMEOUT_H_ diff --git a/mali_kbase/csf/mali_kbase_csf_tl_reader.c b/mali_kbase/csf/mali_kbase_csf_tl_reader.c index d5b1649..d7006e6 100644 --- a/mali_kbase/csf/mali_kbase_csf_tl_reader.c +++ b/mali_kbase/csf/mali_kbase_csf_tl_reader.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -40,7 +41,7 @@ #include "tl/mali_kbase_timeline_priv.h" #include <linux/debugfs.h> -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) #define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE #endif #endif diff --git a/mali_kbase/csf/mali_kbase_csf_tl_reader.h b/mali_kbase/csf/mali_kbase_csf_tl_reader.h index f5ce9d6..14175b3 100644 --- a/mali_kbase/csf/mali_kbase_csf_tl_reader.h +++ b/mali_kbase/csf/mali_kbase_csf_tl_reader.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSFFW_TL_READER_H_ @@ -28,7 +47,7 @@ #include <asm/page.h> /** The number of pages used for CSFFW trace buffer. Can be tweaked. */ -#define KBASE_CSF_TL_BUFFER_NR_PAGES 4 +#define KBASE_CSF_TL_BUFFER_NR_PAGES 128 /** CSFFW Timeline read polling minimum period in milliseconds. */ #define KBASE_CSF_TL_READ_INTERVAL_MIN 20 /** CSFFW Timeline read polling default period in milliseconds. */ diff --git a/mali_kbase/csf/mali_kbase_csf_trace_buffer.c b/mali_kbase/csf/mali_kbase_csf_trace_buffer.c index c39c789..8eb0408 100644 --- a/mali_kbase/csf/mali_kbase_csf_trace_buffer.c +++ b/mali_kbase/csf/mali_kbase_csf_trace_buffer.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -31,7 +32,7 @@ #include <linux/mman.h> #ifdef CONFIG_DEBUG_FS -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) #define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE #endif #endif diff --git a/mali_kbase/csf/mali_kbase_csf_trace_buffer.h b/mali_kbase/csf/mali_kbase_csf_trace_buffer.h index 2cac55e..887a5e7 100644 --- a/mali_kbase/csf/mali_kbase_csf_trace_buffer.h +++ b/mali_kbase/csf/mali_kbase_csf_trace_buffer.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CSF_TRACE_BUFFER_H_ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_csf.h index 32181d7..51ab2f5 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_csf.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -109,6 +128,46 @@ int dummy_array[] = { KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_START), KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_STOP), + /* + * KCPU queue events + */ + /* KTrace info_val == KCPU queue fence context + * KCPU extra_info_val == N/A. + */ + KBASE_KTRACE_CODE_MAKE_CODE(KCPU_QUEUE_NEW), + /* KTrace info_val == Number of pending commands in KCPU queue when + * it is destroyed. + * KCPU extra_info_val == Number of CQS wait operations present in + * the KCPU queue when it is destroyed. + */ + KBASE_KTRACE_CODE_MAKE_CODE(KCPU_QUEUE_DESTROY), + /* KTrace info_val == CQS event memory address + * KCPU extra_info_val == Upper 32 bits of event memory, i.e. contents + * of error field. + */ + KBASE_KTRACE_CODE_MAKE_CODE(CQS_SET), + /* KTrace info_val == Number of CQS objects to be waited upon + * KCPU extra_info_val == N/A. + */ + KBASE_KTRACE_CODE_MAKE_CODE(CQS_WAIT_START), + /* KTrace info_val == CQS event memory address + * KCPU extra_info_val == 1 if CQS was signaled with an error and queue + * inherited the error, otherwise 0. + */ + KBASE_KTRACE_CODE_MAKE_CODE(CQS_WAIT_END), + /* KTrace info_val == Fence context + * KCPU extra_info_val == Fence seqno. + */ + KBASE_KTRACE_CODE_MAKE_CODE(FENCE_SIGNAL), + /* KTrace info_val == Fence context + * KCPU extra_info_val == Fence seqno. + */ + KBASE_KTRACE_CODE_MAKE_CODE(FENCE_WAIT_START), + /* KTrace info_val == Fence context + * KCPU extra_info_val == Fence seqno. + */ + KBASE_KTRACE_CODE_MAKE_CODE(FENCE_WAIT_END), + #if 0 /* Dummy section to avoid breaking formatting */ }; #endif diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h index b201e49..8e846ad 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.c b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.c index 2ea901b..93a082a 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.c +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <mali_kbase.h> #include "debug/mali_kbase_debug_ktrace_internal.h" #include "debug/backend/mali_kbase_debug_ktrace_csf.h" @@ -28,30 +30,30 @@ void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) { *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "group,slot,prio,csi"), 0); + "group,slot,prio,csi,kcpu"), 0); } void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *buffer, int sz, s32 *written) { - const struct kbase_ktrace_backend * const be_msg = &trace_msg->backend; + const union kbase_ktrace_backend * const be_msg = &trace_msg->backend; /* At present, no need to check for KBASE_KTRACE_FLAG_BACKEND, as the * other backend-specific flags currently imply this anyway */ /* group parts */ - if (be_msg->flags & KBASE_KTRACE_FLAG_CSF_GROUP) { - const s8 slot = be_msg->csg_nr; + if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_GROUP) { + const s8 slot = be_msg->gpu.csg_nr; /* group,slot, */ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "%u,%d,", be_msg->group_handle, slot), 0); + "%u,%d,", be_msg->gpu.group_handle, slot), 0); /* prio */ if (slot >= 0) *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "%u", be_msg->slot_prio), 0); + "%u", be_msg->gpu.slot_prio), 0); /* , */ *written += MAX(snprintf(buffer + *written, @@ -65,10 +67,24 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, } /* queue parts: csi */ - if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) + if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", be_msg->gpu.csi_index), 0); + + /* , */ + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ","), 0); + + if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_KCPU) { + /* kcpu data */ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "%d", be_msg->csi_index), 0); + "kcpu %d (0x%llx)", + be_msg->kcpu.id, + be_msg->kcpu.extra_info_val), 0); + } /* Don't end with a trailing "," - this is a 'standalone' formatted * msg, caller will handle the delimiters @@ -95,14 +111,14 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, else if (queue) kctx = queue->kctx; - /* Fill the common part of the message (including backend.flags) */ + /* Fill the common part of the message (including backend.gpu.flags) */ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, info_val); /* Indicate to the common code that backend-specific parts will be * valid */ - trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; /* Fill the CSF-specific parts of the message * @@ -111,30 +127,66 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, */ if (queue) { - trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_QUEUE; - trace_msg->backend.csi_index = queue->csi_index; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_QUEUE; + trace_msg->backend.gpu.csi_index = queue->csi_index; } if (group) { const s8 slot = group->csg_nr; - trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_GROUP; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_GROUP; - trace_msg->backend.csg_nr = slot; + trace_msg->backend.gpu.csg_nr = slot; if (slot >= 0) { - struct kbase_csf_csg_slot *csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; + struct kbase_csf_csg_slot *csg_slot = + &kbdev->csf.scheduler.csg_slots[slot]; - trace_msg->backend.slot_prio = csg_slot->priority; + trace_msg->backend.gpu.slot_prio = + csg_slot->priority; } /* slot >=0 indicates whether slot_prio valid, so no need to * initialize in the case where it's invalid */ - trace_msg->backend.group_handle = group->handle; + trace_msg->backend.gpu.group_handle = group->handle; } - WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); + WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +void kbasep_ktrace_add_csf_kcpu(struct kbase_device *kbdev, + enum kbase_ktrace_code code, + struct kbase_kcpu_command_queue *queue, + u64 info_val1, u64 info_val2) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + struct kbase_context *kctx = queue->kctx; + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Fill the common part of the message */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, 0, + info_val1); + + /* Indicate to the common code that backend-specific parts will be + * valid + */ + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; + + /* Fill the KCPU-specific parts of the message */ + trace_msg->backend.kcpu.id = queue->id; + trace_msg->backend.kcpu.extra_info_val = info_val2; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_KCPU; + + WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); /* Done */ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.h index 6c539b7..a229333 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_CSF_H_ @@ -44,9 +63,33 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, struct kbase_queue *queue, kbase_ktrace_flag_t flags, u64 info_val); +/** + * kbasep_ktrace_add_csf_kcpu - internal function to add trace about the CSF + * KCPU queues. + * @kbdev: kbase device + * @code: trace code + * @queue: queue, or NULL if no queue + * @info_val1: Main infoval variable with information based on the KCPU + * ktrace call. Refer to mali_kbase_debug_ktrace_codes_csf.h + * for information on the infoval values. + * @info_val2: Extra infoval variable with information based on the KCPU + * ktrace call. Refer to mali_kbase_debug_ktrace_codes_csf.h + * for information on the infoval values. + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_CSF_KCPU() instead. + */ +void kbasep_ktrace_add_csf_kcpu(struct kbase_device *kbdev, + enum kbase_ktrace_code code, + struct kbase_kcpu_command_queue *queue, + u64 info_val1, u64 info_val2); + #define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ kbasep_ktrace_add_csf(kbdev, KBASE_KTRACE_CODE(code), group, queue, \ - flags, info_val) + flags, info_val) + +#define KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, queue, info_val1, \ + info_val2) kbasep_ktrace_add_csf_kcpu(kbdev, KBASE_KTRACE_CODE(code), \ + queue, info_val1, info_val2) #else /* KBASE_KTRACE_TARGET_RBUF */ @@ -55,12 +98,21 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, CSTD_UNUSED(kbdev);\ CSTD_NOP(code);\ CSTD_UNUSED(group);\ - CSTD_UNUSED(queue); \ + CSTD_UNUSED(queue);\ CSTD_UNUSED(flags);\ CSTD_UNUSED(info_val);\ CSTD_NOP(0);\ } while (0) +#define KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, queue, info_val1, info_val2) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(queue);\ + CSTD_UNUSED(info_val1);\ + CSTD_UNUSED(info_val2);\ + } while (0) + #endif /* KBASE_KTRACE_TARGET_RBUF */ /* @@ -74,6 +126,9 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, #define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ trace_mali_##code(kbdev, group, queue, info_val) +#define KBASE_KTRACE_FTRACE_ADD_KCPU(code, queue, info_val1, info_val2) \ + trace_mali_##code(queue, info_val1, info_val2) + #else /* KBASE_KTRACE_TARGET_FTRACE */ #define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ @@ -86,6 +141,14 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, CSTD_NOP(0);\ } while (0) +#define KBASE_KTRACE_FTRACE_ADD_KCPU(code, queue, info_val1, info_val2) \ + do {\ + CSTD_NOP(code);\ + CSTD_UNUSED(queue);\ + CSTD_UNUSED(info_val1);\ + CSTD_UNUSED(info_val2);\ + } while (0) + #endif /* KBASE_KTRACE_TARGET_FTRACE */ /* @@ -108,7 +171,7 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, */ #define KBASE_KTRACE_ADD_CSF_GRP(kbdev, code, group, info_val) \ do { \ - /* capture values that could come from non-pure function calls */ \ + /* capture values that could come from non-pure fn calls */ \ struct kbase_queue_group *__group = group; \ u64 __info_val = info_val; \ KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, NULL, 0u, \ @@ -134,7 +197,7 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, */ #define KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, code, group, queue, info_val) \ do { \ - /* capture values that could come from non-pure function calls */ \ + /* capture values that could come from non-pure fn calls */ \ struct kbase_queue_group *__group = group; \ struct kbase_queue *__queue = queue; \ u64 __info_val = info_val; \ @@ -144,4 +207,17 @@ void kbasep_ktrace_add_csf(struct kbase_device *kbdev, __queue, __info_val); \ } while (0) + +#define KBASE_KTRACE_ADD_CSF_KCPU(kbdev, code, queue, info_val1, info_val2) \ + do { \ + /* capture values that could come from non-pure fn calls */ \ + struct kbase_kcpu_command_queue *__queue = queue; \ + u64 __info_val1 = info_val1; \ + u64 __info_val2 = info_val2; \ + KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, __queue, \ + __info_val1, __info_val2); \ + KBASE_KTRACE_FTRACE_ADD_KCPU(code, __queue, \ + __info_val1, __info_val2); \ + } while (0) + #endif /* _KBASE_DEBUG_KTRACE_CSF_H_ */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_csf.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_csf.h index 722b410..426218b 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_csf.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ @@ -38,9 +57,15 @@ * ftrace backend now outputs kctx field (as %d_%u format). * * Add fields group, slot, prio, csi into backend-specific part. + * + * 1.2: + * There is a new class of KCPU traces; with this, a new KCPU column in the + * ringbuffer RBUF (mali_trace) between csi and info_val, which is empty + * for non-kcpu related traces, and usually displays the KCPU Queue ID and + * an extra information value. ftrace also displays these KCPU traces. */ #define KBASE_KTRACE_VERSION_MAJOR 1 -#define KBASE_KTRACE_VERSION_MINOR 1 +#define KBASE_KTRACE_VERSION_MINOR 2 /* indicates if the trace message has valid queue-group related info. */ #define KBASE_KTRACE_FLAG_CSF_GROUP (((kbase_ktrace_flag_t)1) << 0) @@ -48,37 +73,55 @@ /* indicates if the trace message has valid queue related info. */ #define KBASE_KTRACE_FLAG_CSF_QUEUE (((kbase_ktrace_flag_t)1) << 1) +/* indicates if the trace message has valid KCPU-queue related info. */ +#define KBASE_KTRACE_FLAG_CSF_KCPU (((kbase_ktrace_flag_t)1) << 2) + /* Collect all the flags together for debug checking */ #define KBASE_KTRACE_FLAG_BACKEND_ALL \ - (KBASE_KTRACE_FLAG_CSF_GROUP | KBASE_KTRACE_FLAG_CSF_QUEUE) + (KBASE_KTRACE_FLAG_CSF_GROUP | KBASE_KTRACE_FLAG_CSF_QUEUE | \ + KBASE_KTRACE_FLAG_CSF_KCPU) /** - * struct kbase_ktrace_backend - backend specific part of a trace message - * - * @code: Identifies the event, refer to enum kbase_ktrace_code. - * @flags: indicates information about the trace message itself. Used - * during dumping of the message. - * @group_handle: Handle identifying the associated queue group. Only valid - * when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. - * @csg_nr: Number/index of the associated queue group's CS - * group to which it is mapped, or negative if none associated. - * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. - * @slot_prio: The priority of the slot for the associated group, if it was - * scheduled. Hence, only valid when @csg_nr >=0 and @flags - * contains KBASE_KTRACE_FLAG_CSF_GROUP. - * @csi_index: ID of the associated queue's CS HW interface. - * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_QUEUE. + * union kbase_ktrace_backend - backend specific part of a trace message + * + * @code: Identifies the event, refer to enum kbase_ktrace_code. + * @flags: indicates information about the trace message itself. Used + * during dumping of the message. + * @id: ID of the KCPU queue. + * @extra_info_val: value specific to the type of KCPU event being traced. + * Refer to the KPU specific code in enum kbase_ktrace_code in + * mali_kbase_debug_ktrace_codes_csf.h + * @group_handle: Handle identifying the associated queue group. Only valid + * when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @csg_nr: Number/index of the associated queue group's CS + * group to which it is mapped, or negative if none associated. + * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @slot_prio: The priority of the slot for the associated group, if it was + * scheduled. Hence, only valid when @csg_nr >=0 and @flags + * contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @csi_index: ID of the associated queue's CS HW interface. + * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_QUEUE. */ -struct kbase_ktrace_backend { + +union kbase_ktrace_backend { /* Place 64 and 32-bit members together */ /* Pack smaller members together */ - kbase_ktrace_code_t code; - kbase_ktrace_flag_t flags; - u8 group_handle; - s8 csg_nr; - u8 slot_prio; - s8 csi_index; + struct { + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 id; + u64 extra_info_val; + } kcpu; + + struct { + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 group_handle; + s8 csg_nr; + u8 slot_prio; + s8 csi_index; + } gpu; }; #endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h index ea8e01a..4b4cefc 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ @@ -71,7 +90,9 @@ | KBASE_KTRACE_FLAG_JM_ATOM) /** - * struct kbase_ktrace_backend - backend specific part of a trace message + * union kbase_ktrace_backend - backend specific part of a trace message + * Contains only a struct but is a union such that it is compatible with + * generic JM and CSF KTrace calls. * * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags @@ -86,16 +107,20 @@ * @refcount: reference count for the context, valid for certain events * related to scheduler core and policy. */ -struct kbase_ktrace_backend { - /* Place 64 and 32-bit members together */ - u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ - u64 gpu_addr; - int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ - /* Pack smaller members together */ - kbase_ktrace_code_t code; - kbase_ktrace_flag_t flags; - u8 jobslot; - u8 refcount; +union kbase_ktrace_backend { + struct { + /* Place 64 and 32-bit members together */ + u64 atom_udata[2]; /* Only valid for + * KBASE_KTRACE_FLAG_JM_ATOM + */ + u64 gpu_addr; + int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + /* Pack smaller members together */ + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 jobslot; + u8 refcount; + } gpu; }; #endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c index 1b82128..55e4391 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <mali_kbase.h> #include "debug/mali_kbase_debug_ktrace_internal.h" #include "debug/backend/mali_kbase_debug_ktrace_jm.h" @@ -35,38 +37,39 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *buffer, int sz, s32 *written) { /* katom */ - if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM) + if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "atom %d (ud: 0x%llx 0x%llx)", - trace_msg->backend.atom_number, - trace_msg->backend.atom_udata[0], - trace_msg->backend.atom_udata[1]), 0); + trace_msg->backend.gpu.atom_number, + trace_msg->backend.gpu.atom_udata[0], + trace_msg->backend.gpu.atom_udata[1]), 0); /* gpu_addr */ - if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND) + if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - ",%.8llx,", trace_msg->backend.gpu_addr), 0); + ",%.8llx,", trace_msg->backend.gpu.gpu_addr), + 0); else *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ",,"), 0); /* jobslot */ - if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) + if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "%d", trace_msg->backend.jobslot), 0); + "%d", trace_msg->backend.gpu.jobslot), 0); *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ","), 0); /* refcount */ - if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) + if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "%d", trace_msg->backend.refcount), 0); + "%d", trace_msg->backend.gpu.refcount), 0); } void kbasep_ktrace_add_jm(struct kbase_device *kbdev, @@ -83,30 +86,31 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, /* Reserve and update indices */ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - /* Fill the common part of the message (including backend.flags) */ + /* Fill the common part of the message (including backend.gpu.flags) */ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, info_val); /* Indicate to the common code that backend-specific parts will be * valid */ - trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; /* Fill the JM-specific parts of the message */ if (katom) { - trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM; + trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_JM_ATOM; - trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom); - trace_msg->backend.atom_udata[0] = katom->udata.blob[0]; - trace_msg->backend.atom_udata[1] = katom->udata.blob[1]; + trace_msg->backend.gpu.atom_number = + kbase_jd_atom_id(katom->kctx, katom); + trace_msg->backend.gpu.atom_udata[0] = katom->udata.blob[0]; + trace_msg->backend.gpu.atom_udata[1] = katom->udata.blob[1]; } - trace_msg->backend.gpu_addr = gpu_addr; - trace_msg->backend.jobslot = jobslot; + trace_msg->backend.gpu.gpu_addr = gpu_addr; + trace_msg->backend.gpu.jobslot = jobslot; /* Clamp refcount */ - trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF); + trace_msg->backend.gpu.refcount = MIN((unsigned int)refcount, 0xFF); - WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); + WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); /* Done */ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h index bf258d6..43f5526 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_JM_H_ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_csf.h index d103e57..bb5db30 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_csf.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -144,4 +163,50 @@ DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_STOP); #undef DEFINE_MALI_CSF_GRP_Q_EVENT +/* + * KCPU queue events + */ +DECLARE_EVENT_CLASS(mali_csf_kcpu_queue_template, + TP_PROTO(struct kbase_kcpu_command_queue *queue, + u64 info_val1, u64 info_val2), + TP_ARGS(queue, info_val1, info_val2), + TP_STRUCT__entry( + __field(u64, info_val1) + __field(u64, info_val2) + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(u8, id) + ), + TP_fast_assign( + { + __entry->info_val1 = info_val1; + __entry->info_val2 = info_val2; + __entry->kctx_id = queue->kctx->id; + __entry->kctx_tgid = queue->kctx->tgid; + __entry->id = queue->id; + } + + ), + TP_printk("kctx=%d_%u id=%u info_val1=0x%llx info_val2=0x%llx", + __entry->kctx_tgid, __entry->kctx_id, __entry->id, + __entry->info_val1, __entry->info_val2) +); + +#define DEFINE_MALI_CSF_KCPU_EVENT(name) \ + DEFINE_EVENT(mali_csf_kcpu_queue_template, mali_##name, \ + TP_PROTO(struct kbase_kcpu_command_queue *queue, \ + u64 info_val1, u64 info_val2), \ + TP_ARGS(queue, info_val1, info_val2)) + +DEFINE_MALI_CSF_KCPU_EVENT(KCPU_QUEUE_NEW); +DEFINE_MALI_CSF_KCPU_EVENT(KCPU_QUEUE_DESTROY); +DEFINE_MALI_CSF_KCPU_EVENT(CQS_SET); +DEFINE_MALI_CSF_KCPU_EVENT(CQS_WAIT_START); +DEFINE_MALI_CSF_KCPU_EVENT(CQS_WAIT_END); +DEFINE_MALI_CSF_KCPU_EVENT(FENCE_SIGNAL); +DEFINE_MALI_CSF_KCPU_EVENT(FENCE_WAIT_START); +DEFINE_MALI_CSF_KCPU_EVENT(FENCE_WAIT_END); + +#undef DEFINE_MALI_CSF_KCPU_EVENT + #endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h index 037b1ed..b0722f9 100644 --- a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +++ b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.c b/mali_kbase/debug/mali_kbase_debug_ktrace.c index a13c0ba..a38a656 100644 --- a/mali_kbase/debug/mali_kbase_debug_ktrace.c +++ b/mali_kbase/debug/mali_kbase_debug_ktrace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <mali_kbase.h> #include "debug/mali_kbase_debug_ktrace_internal.h" @@ -93,7 +95,8 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, - kbasep_ktrace_code_string[trace_msg->backend.code]), 0); + kbasep_ktrace_code_string[trace_msg->backend.gpu.code]), + 0); /* kctx part: */ if (trace_msg->kctx_tgid) { @@ -171,8 +174,8 @@ void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, trace_msg->kctx_id = 0; } trace_msg->info_val = info_val; - trace_msg->backend.code = code; - trace_msg->backend.flags = flags; + trace_msg->backend.gpu.code = code; + trace_msg->backend.gpu.flags = flags; } void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, @@ -189,7 +192,7 @@ void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, /* Reserve and update indices */ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - /* Fill the common part of the message (including backend.flags) */ + /* Fill the common part of the message (including backend.gpu.flags) */ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, info_val); diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.h b/mali_kbase/debug/mali_kbase_debug_ktrace.h index 7263e0c..71f0fd7 100644 --- a/mali_kbase/debug/mali_kbase_debug_ktrace.h +++ b/mali_kbase/debug/mali_kbase_debug_ktrace.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h index dd0ad06..fa842f9 100644 --- a/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -146,7 +165,14 @@ int dummy_array[] = { KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), /* info_val == kctx->refcount */ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), - +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* + * Arbitration events + */ + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_EVT), +#endif #if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h index 3b7db2e..841f11b 100644 --- a/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_DEFS_H_ @@ -78,12 +97,18 @@ typedef u8 kbase_ktrace_code_t; */ /* - * struct kbase_ktrace_backend - backend specific part of a trace message - * - * At the very least, this must contain a kbase_ktrace_code_t 'code' member and - * a kbase_ktrace_flag_t 'flags' member + * union kbase_ktrace_backend - backend specific part of a trace message. + * At the very least, this must contain a kbase_ktrace_code_t 'code' member + * and a kbase_ktrace_flag_t 'flags' inside a "gpu" sub-struct. Should a + * backend need several sub structs in its union to optimize the data storage + * for different message types, then it can use a "common initial sequence" to + * allow 'flags' and 'code' to pack optimally without corrupting them. + * Different backends need not share common initial sequences between them, they + * only need to ensure they have gpu.flags and gpu.code members, it + * is up to the backend then how to order these. */ -struct kbase_ktrace_backend; +union kbase_ktrace_backend; + #endif /* KBASE_KTRACE_TARGET_RBUF */ #if MALI_USE_CSF @@ -145,9 +170,9 @@ enum kbase_ktrace_code { * @kctx_id: Unique identifier of the &kbase_context associated with the * message. Only valid if @kctx_tgid != 0. * @info_val: value specific to the type of event being traced. Refer to the - * specific code in enum kbase_ktrace_code + * specific code in enum kbase_ktrace_code. * @backend: backend-specific trace information. All backends must implement - * a minimum common set of members + * a minimum common set of members. */ struct kbase_ktrace_msg { struct timespec64 timestamp; @@ -156,8 +181,7 @@ struct kbase_ktrace_msg { pid_t kctx_tgid; u32 kctx_id; u64 info_val; - - struct kbase_ktrace_backend backend; + union kbase_ktrace_backend backend; }; struct kbase_ktrace { diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h index e450760..02b90ab 100644 --- a/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_ diff --git a/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h index a7310eb..86efdc4 100644 --- a/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h +++ b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -98,7 +117,13 @@ DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE); DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); +#ifdef CONFIG_MALI_ARBITER_SUPPORT +DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); +DEFINE_MALI_ADD_EVENT(ARB_VM_STATE); +DEFINE_MALI_ADD_EVENT(ARB_VM_EVT); + +#endif #if MALI_USE_CSF #include "mali_kbase_debug_linux_ktrace_csf.h" #else diff --git a/mali_kbase/device/backend/mali_kbase_device_csf.c b/mali_kbase/device/backend/mali_kbase_device_csf.c index 7cd3e4a..c70df70 100644 --- a/mali_kbase/device/backend/mali_kbase_device_csf.c +++ b/mali_kbase/device/backend/mali_kbase_device_csf.c @@ -129,6 +129,8 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + /* kbase_pm_context_idle is called after the boot of firmware */ + return 0; fail_update_l2_features: @@ -146,6 +148,7 @@ fail_interrupt_test: kbase_backend_timer_term(kbdev); fail_timer: + kbase_pm_context_idle(kbdev); kbase_hwaccess_pm_halt(kbdev); fail_pm_powerup: kbase_reset_gpu_term(kbdev); @@ -206,20 +209,26 @@ static const struct kbase_device_init dev_init[] = { {kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term, "Clock rate trace manager initialization failed"}, - {kbase_device_hwcnt_backend_jm_init, - kbase_device_hwcnt_backend_jm_term, + {kbase_device_hwcnt_backend_csf_if_init, + kbase_device_hwcnt_backend_csf_if_term, + "GPU hwcnt backend CSF interface creation failed"}, + {kbase_device_hwcnt_backend_csf_init, + kbase_device_hwcnt_backend_csf_term, "GPU hwcnt backend creation failed"}, {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, "GPU hwcnt context initialization failed"}, + {kbase_backend_late_init, kbase_backend_late_term, + "Late backend initialization failed"}, + {kbase_device_csf_firmware_init, kbase_device_csf_firmware_term, + "Firmware initialization failed"}, + {kbase_device_hwcnt_backend_csf_metadata_init, + kbase_device_hwcnt_backend_csf_metadata_term, + "GPU hwcnt backend metadata creation failed"}, {kbase_device_hwcnt_virtualizer_init, kbase_device_hwcnt_virtualizer_term, "GPU hwcnt virtualizer initialization failed"}, {kbase_device_vinstr_init, kbase_device_vinstr_term, "Virtual instrumentation initialization failed"}, - {kbase_backend_late_init, kbase_backend_late_term, - "Late backend initialization failed"}, - {kbase_device_csf_firmware_init, kbase_device_csf_firmware_term, - "Firmware initialization failed"}, #ifdef MALI_KBASE_BUILD {kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed"}, diff --git a/mali_kbase/device/backend/mali_kbase_device_hw_csf.c b/mali_kbase/device/backend/mali_kbase_device_hw_csf.c index ff899be..1ae311a 100644 --- a/mali_kbase/device/backend/mali_kbase_device_hw_csf.c +++ b/mali_kbase/device/backend/mali_kbase_device_hw_csf.c @@ -71,6 +71,15 @@ static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev) if (!as_valid || (as_nr == MCU_AS_NR)) { kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); + /* MCU bus fault could mean hardware counters will stop + * working. + * Put the backend into the unrecoverable error state to + * cause current and subsequent counter operations to + * immediately fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error( + &kbdev->hwcnt_gpu_iface); + dev_err(kbdev->dev, "GPU bus fault triggering gpu-reset ...\n"); if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); @@ -120,12 +129,21 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) } } }; scheduler->active_protm_grp->faulted = true; - kbase_csf_add_fatal_error_to_kctx( + kbase_csf_add_group_fatal_error( scheduler->active_protm_grp, &err_payload); kbase_event_wakeup(scheduler->active_protm_grp->kctx); } kbase_csf_scheduler_spin_unlock(kbdev, flags); + /* Protected fault means we're unlikely to have the counter + * operations we might do during reset acknowledged. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error( + &kbdev->hwcnt_gpu_iface); + if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } diff --git a/mali_kbase/device/backend/mali_kbase_device_jm.c b/mali_kbase/device/backend/mali_kbase_device_jm.c index 2a45a33..0d8de3c 100644 --- a/mali_kbase/device/backend/mali_kbase_device_jm.c +++ b/mali_kbase/device/backend/mali_kbase_device_jm.c @@ -96,9 +96,6 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) if (err) goto fail_devfreq_init; - /* Idle the GPU and/or cores, if the policy wants it to */ - kbase_pm_context_idle(kbdev); - /* Update gpuprops with L2_FEATURES if applicable */ err = kbase_gpuprops_update_l2_features(kbdev); if (err) @@ -106,6 +103,9 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + /* Idle the GPU and/or cores, if the policy wants it to */ + kbase_pm_context_idle(kbdev); + return 0; fail_update_l2_features: @@ -121,6 +121,7 @@ fail_interrupt_test: kbase_backend_timer_term(kbdev); fail_timer: + kbase_pm_context_idle(kbdev); kbase_hwaccess_pm_halt(kbdev); fail_pm_powerup: kbase_reset_gpu_term(kbdev); diff --git a/mali_kbase/device/mali_kbase_device.c b/mali_kbase/device/mali_kbase_device.c index 76fb33a..6178cc8 100644 --- a/mali_kbase/device/mali_kbase_device.c +++ b/mali_kbase/device/mali_kbase_device.c @@ -41,10 +41,14 @@ #include <tl/mali_kbase_timeline.h> #include "mali_kbase_vinstr.h" +#if MALI_USE_CSF +#include <mali_kbase_hwcnt_backend_csf_if_fw.h> +#endif #include "mali_kbase_hwcnt_context.h" #include "mali_kbase_hwcnt_virtualizer.h" #include "mali_kbase_device.h" +#include "mali_kbase_device_internal.h" #include "backend/gpu/mali_kbase_pm_internal.h" #include "backend/gpu/mali_kbase_irq_internal.h" #include "mali_kbase_regs_history_debugfs.h" @@ -112,6 +116,9 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) #ifdef CONFIG_ARM64 struct device_node *np = NULL; #endif /* CONFIG_ARM64 */ +#ifdef CONFIG_OF + struct device_node *prio_ctrl_node = NULL; +#endif spin_lock_init(&kbdev->mmu_mask_change); mutex_init(&kbdev->mmu_hw_mutex); @@ -136,6 +143,34 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) } } #endif /* CONFIG_ARM64 */ + kbdev->pcm_dev = NULL; +#ifdef CONFIG_OF + /* Check to see whether or not a platform specific priority control manager + * is available. + */ + prio_ctrl_node = of_parse_phandle(kbdev->dev->of_node, + "priority-control-manager", 0); + if (!prio_ctrl_node) { + dev_info(kbdev->dev, + "No priority control manager is configured\n"); + } else { + struct platform_device *const pdev = + of_find_device_by_node(prio_ctrl_node); + if (!pdev) { + dev_err(kbdev->dev, + "The configured priority control manager was not found\n"); + } else { + kbdev->pcm_dev = platform_get_drvdata(pdev); + if (!kbdev->pcm_dev) { + dev_info(kbdev->dev, + "Priority control manager is not ready\n"); + err = -EPROBE_DEFER; + } + } + of_node_put(prio_ctrl_node); + } +#endif /* CONFIG_OF */ + /* Get the list of workarounds for issues on the current HW * (identified by the GPU_ID register) */ @@ -152,11 +187,6 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) if (err) goto fail; - /* On Linux 4.0+, dma coherency is determined from device tree */ -#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) - set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); -#endif - /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our * device structure was created by device-tree */ @@ -179,7 +209,9 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) if (err) goto dma_set_mask_failed; +#if !MALI_USE_CSF spin_lock_init(&kbdev->hwcnt.lock); +#endif err = kbase_ktrace_init(kbdev); if (err) @@ -191,9 +223,11 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) atomic_set(&kbdev->ctx_num, 0); +#if !MALI_USE_CSF err = kbase_instr_backend_init(kbdev); if (err) goto term_trace; +#endif kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; @@ -210,8 +244,12 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) spin_lock_init(&kbdev->hwaccess_lock); return 0; + +#if !MALI_USE_CSF term_trace: kbase_ktrace_term(kbdev); +#endif + term_as: kbase_device_all_as_term(kbdev); dma_set_mask_failed: @@ -229,7 +267,9 @@ void kbase_device_misc_term(struct kbase_device *kbdev) kbase_debug_assert_register_hook(NULL, NULL); #endif +#if !MALI_USE_CSF kbase_instr_backend_term(kbdev); +#endif kbase_ktrace_term(kbdev); @@ -253,6 +293,49 @@ void kbase_increment_device_id(void) kbase_dev_nr++; } +#if MALI_USE_CSF + +int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev) +{ + return kbase_hwcnt_backend_csf_if_fw_create( + kbdev, &kbdev->hwcnt_backend_csf_if_fw); +} + +void kbase_device_hwcnt_backend_csf_if_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_if_fw_destroy(&kbdev->hwcnt_backend_csf_if_fw); +} + +int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev) +{ + return kbase_hwcnt_backend_csf_create( + &kbdev->hwcnt_backend_csf_if_fw, + KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT, + &kbdev->hwcnt_gpu_iface); +} + +void kbase_device_hwcnt_backend_csf_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_destroy(&kbdev->hwcnt_gpu_iface); +} + +int kbase_device_hwcnt_backend_csf_metadata_init(struct kbase_device *kbdev) +{ + /* For CSF GPUs, HWC metadata needs to query informatoin from CSF + * firmware, so the initialization of HWC metadata only can be called + * after firmware initialised, but firmware initialization depends on + * HWC backend initialization, so we need to separate HWC backend + * metadata initialization from HWC backend initialization. + */ + return kbase_hwcnt_backend_csf_metadata_init(&kbdev->hwcnt_gpu_iface); +} + +void kbase_device_hwcnt_backend_csf_metadata_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface); +} +#else + int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) { return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_iface); @@ -262,6 +345,7 @@ void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) { kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_iface); } +#endif /* MALI_USE_CSF */ int kbase_device_hwcnt_context_init(struct kbase_device *kbdev) { diff --git a/mali_kbase/device/mali_kbase_device.h b/mali_kbase/device/mali_kbase_device.h index 33264bc..32e85fd 100644 --- a/mali_kbase/device/mali_kbase_device.h +++ b/mali_kbase/device/mali_kbase_device.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #include <mali_kbase.h> diff --git a/mali_kbase/device/mali_kbase_device_internal.h b/mali_kbase/device/mali_kbase_device_internal.h index 5464458..b07b22c 100644 --- a/mali_kbase/device/mali_kbase_device_internal.h +++ b/mali_kbase/device/mali_kbase_device_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #include <mali_kbase.h> @@ -43,8 +62,17 @@ void kbase_device_vinstr_term(struct kbase_device *kbdev); int kbase_device_timeline_init(struct kbase_device *kbdev); void kbase_device_timeline_term(struct kbase_device *kbdev); +#if MALI_USE_CSF +int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_backend_csf_term(struct kbase_device *kbdev); +int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_backend_csf_if_term(struct kbase_device *kbdev); +int kbase_device_hwcnt_backend_csf_metadata_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_backend_csf_metadata_term(struct kbase_device *kbdev); +#else int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev); void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev); +#endif int kbase_device_hwcnt_context_init(struct kbase_device *kbdev); void kbase_device_hwcnt_context_term(struct kbase_device *kbdev); diff --git a/mali_kbase/gpu/backend/mali_kbase_gpu_fault_csf.c b/mali_kbase/gpu/backend/mali_kbase_gpu_fault_csf.c index 8dde2b1..a9f9cb8 100644 --- a/mali_kbase/gpu/backend/mali_kbase_gpu_fault_csf.c +++ b/mali_kbase/gpu/backend/mali_kbase_gpu_fault_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/gpu/backend/mali_kbase_gpu_fault_jm.c b/mali_kbase/gpu/backend/mali_kbase_gpu_fault_jm.c index 56f5415..6015062 100644 --- a/mali_kbase/gpu/backend/mali_kbase_gpu_fault_jm.c +++ b/mali_kbase/gpu/backend/mali_kbase_gpu_fault_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_csf.h b/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_csf.h index 4a5f97f..04ee612 100644 --- a/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_csf.h +++ b/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_REGMAP_CSF_H_ @@ -64,8 +83,6 @@ #include "csf/mali_gpu_csf_control_registers.h" -#define GPU_CONTROL_MCU_REG(r) (GPU_CONTROL_MCU + (r)) - /* Set to implementation defined, outer caching */ #define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull @@ -111,6 +128,7 @@ #define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 /* GPU control registers */ +#define CORE_FEATURES 0x008 /* () Shader Core Features */ #define MCU_CONTROL 0x700 #define MCU_STATUS 0x704 @@ -281,9 +299,12 @@ #define GPU_FAULTSTATUS_ACCESS_TYPE_WRITE 0x3 /* End of GPU_FAULTSTATUS_ACCESS_TYPE values */ -/* TODO: Remove once 10.x.6 headers became available */ +/* Implementation-dependent exception codes used to indicate CSG + * and CS errors that are not specified in the specs. + */ #define GPU_EXCEPTION_TYPE_SW_FAULT_0 ((u8)0x70) #define GPU_EXCEPTION_TYPE_SW_FAULT_1 ((u8)0x71) +#define GPU_EXCEPTION_TYPE_SW_FAULT_2 ((u8)0x72) /* GPU_FAULTSTATUS_EXCEPTION_TYPE values */ #define GPU_FAULTSTATUS_EXCEPTION_TYPE_OK 0x00 diff --git a/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_jm.h b/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_jm.h index c9c2fbd..57d567b 100644 --- a/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_jm.h +++ b/mali_kbase/gpu/backend/mali_kbase_gpu_regmap_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_REGMAP_JM_H_ @@ -139,8 +158,8 @@ #define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ #define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ #define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ -#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job - slot n */ +/* (RO) Extended affinity mask for job slot n*/ +#define JS_XAFFINITY 0x1C #define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ #define JS_STATUS 0x24 /* (RO) Status register for job slot n */ @@ -151,8 +170,8 @@ #define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ #define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ #define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ -#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for - job slot n */ +/* (RW) Next extended affinity mask for job slot n */ +#define JS_XAFFINITY_NEXT 0x5C #define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ diff --git a/mali_kbase/gpu/mali_kbase_gpu.c b/mali_kbase/gpu/mali_kbase_gpu.c index 3128db4..4cc8750 100644 --- a/mali_kbase/gpu/mali_kbase_gpu.c +++ b/mali_kbase/gpu/mali_kbase_gpu.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,6 +23,7 @@ #include <mali_kbase.h> #include <mali_kbase_defs.h> +#include <gpu/mali_kbase_gpu_fault.h> const char *kbase_gpu_access_type_name(u32 fault_status) { diff --git a/mali_kbase/gpu/mali_kbase_gpu.h b/mali_kbase/gpu/mali_kbase_gpu.h index 9516e56..0594cd3 100644 --- a/mali_kbase/gpu/mali_kbase_gpu.h +++ b/mali_kbase/gpu/mali_kbase_gpu.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_H_ diff --git a/mali_kbase/gpu/mali_kbase_gpu_coherency.h b/mali_kbase/gpu/mali_kbase_gpu_coherency.h index bb2b161..00991dd 100644 --- a/mali_kbase/gpu/mali_kbase_gpu_coherency.h +++ b/mali_kbase/gpu/mali_kbase_gpu_coherency.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_COHERENCY_H_ diff --git a/mali_kbase/gpu/mali_kbase_gpu_fault.h b/mali_kbase/gpu/mali_kbase_gpu_fault.h index e63c388..a09c3ba 100644 --- a/mali_kbase/gpu/mali_kbase_gpu_fault.h +++ b/mali_kbase/gpu/mali_kbase_gpu_fault.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_FAULT_H_ diff --git a/mali_kbase/gpu/mali_kbase_gpu_id.h b/mali_kbase/gpu/mali_kbase_gpu_id.h index 446b83a..fda52e6 100644 --- a/mali_kbase/gpu/mali_kbase_gpu_id.h +++ b/mali_kbase/gpu/mali_kbase_gpu_id.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_ID_H_ @@ -53,7 +72,8 @@ GPU_ID2_VERSION_STATUS) /* Helper macro to create a partial GPU_ID (new format) that defines - a product ignoring its version. */ + * a product ignoring its version. + */ #define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ (((u32)arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ @@ -61,7 +81,8 @@ (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) /* Helper macro to create a partial GPU_ID (new format) that specifies the - revision (major, minor, status) of a product */ + * revision (major, minor, status) of a product + */ #define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ ((((u32)version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ (((u32)version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ @@ -76,14 +97,16 @@ version_status)) /* Helper macro to create a partial GPU_ID (new format) that identifies - a particular GPU model by its arch_major and product_major. */ + * a particular GPU model by its arch_major and product_major. + */ #define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) /* Strip off the non-relevant bits from a product_id value and make it suitable - for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU - model. */ + * for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU + * model. + */ #define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ ((((u32)product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ GPU_ID2_PRODUCT_MODEL) @@ -108,7 +131,8 @@ #define GPU_ID2_PRODUCT_LTUX GPU_ID2_MODEL_MAKE(11, 3) /* Helper macro to create a GPU_ID assuming valid values for id, major, - minor, status */ + * minor, status + */ #define GPU_ID_MAKE(id, major, minor, status) \ ((((u32)id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ (((u32)major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ diff --git a/mali_kbase/gpu/mali_kbase_gpu_regmap.h b/mali_kbase/gpu/mali_kbase_gpu_regmap.h index acd6baa..8c1e41c 100644 --- a/mali_kbase/gpu/mali_kbase_gpu_regmap.h +++ b/mali_kbase/gpu/mali_kbase_gpu_regmap.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_GPU_REGMAP_H_ @@ -66,7 +85,8 @@ #define PWR_KEY 0x050 /* (WO) Power manager key register */ #define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ #define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ - +#define GPU_FEATURES_LO 0x060 /* (RO) GPU features, low word */ +#define GPU_FEATURES_HI 0x064 /* (RO) GPU features, high word */ #define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ #define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ #define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ @@ -142,6 +162,10 @@ #define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ #define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ +#define ASN_HASH_0 0x02C0 /* (RW) ASN hash function argument 0 */ +#define ASN_HASH(n) (ASN_HASH_0 + (n)*4) +#define ASN_HASH_COUNT 3 + #define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ #define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ @@ -241,7 +265,8 @@ #define MMU_PAGE_FAULT_FLAGS 16 /* Macros returning a bitmask to retrieve page fault or bus error flags from - * MMU registers */ + * MMU registers + */ #define MMU_PAGE_FAULT(n) (1UL << (n)) #define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) @@ -329,11 +354,16 @@ #define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ #define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ #define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ -#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs - (deprecated - only for use with T60x) */ -#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ -#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then - flush all L2 caches then issue a flush region command to all MMUs */ +/* Flush all L2 caches then issue a flush region command to all MMUs + * (deprecated - only for use with T60x) + */ +#define AS_COMMAND_FLUSH 0x04 +/* Flush all L2 caches then issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH_PT 0x04 +/* Wait for memory accesses to complete, flush all the L1s cache then flush all + * L2 caches then issue a flush region command to all MMUs + */ +#define AS_COMMAND_FLUSH_MEM 0x05 /* GPU_STATUS values */ #define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ @@ -419,6 +449,8 @@ #define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) #define L2_CONFIG_HASH_SHIFT 24 #define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT) +#define L2_CONFIG_ASN_HASH_ENABLE_SHIFT 24 +#define L2_CONFIG_ASN_HASH_ENABLE_MASK (1ul << L2_CONFIG_ASN_HASH_ENABLE_SHIFT) /* End L2_CONFIG register */ /* IDVS_GROUP register */ diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.c b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.c index 2d8f963..b7c861c 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.c +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -31,6 +32,25 @@ */ #define DEFAULT_MIN_SAMPLE_CYCLES 10000 +/* Typical value for the sampling interval is expected to be less than 100ms, + * So 5 seconds is a reasonable upper limit for the time gap between the + * 2 samples. + */ +#define MAX_SAMPLE_INTERVAL_MS ((s64)5000) + +/* Maximum increment that is expected for a counter value during a sampling + * interval is derived assuming + * - max sampling interval of 1 second. + * - max GPU frequency of 2 GHz. + * - max number of cores as 32. + * - max increment of 4 in per core counter value at every clock cycle. + * + * So max increment = 2 * 10^9 * 32 * 4 = ~2^38. + * If a counter increases by an amount greater than this value, then an error + * will be returned and the simple power model will be used. + */ +#define MAX_COUNTER_INCREMENT (((u64)1 << 38) - 1) + static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) { s64 rtn; @@ -47,11 +67,9 @@ static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) static s64 kbase_ipa_group_energy(s32 coeff, u64 counter_value) { - /* Range: 0 < counter_value < 2^27 */ - if (counter_value > U32_MAX) - counter_value = U32_MAX; + /* Range: 0 < counter_value < 2^38 */ - /* Range: -2^49 < ret < 2^49 */ + /* Range: -2^59 < ret < 2^59 (as -2^21 < coeff < 2^21) */ return counter_value * (s64)coeff; } @@ -70,12 +88,18 @@ kbase_ipa_attach_ipa_control(struct kbase_ipa_counter_model_data *model_data) { struct kbase_device *kbdev = model_data->kbdev; struct kbase_ipa_control_perf_counter *perf_counters; - size_t num_counters = model_data->counters_def_num; + u32 cnt_idx = 0; int err; size_t i; - perf_counters = - kcalloc(num_counters, sizeof(*perf_counters), GFP_KERNEL); + /* Value for GPU_ACTIVE counter also needs to be queried. It is required + * for the normalization of top-level and shader core counters. + */ + model_data->num_counters = 1 + model_data->num_top_level_cntrs + + model_data->num_shader_cores_cntrs; + + perf_counters = kcalloc(model_data->num_counters, + sizeof(*perf_counters), GFP_KERNEL); if (!perf_counters) { dev_err(kbdev->dev, @@ -83,18 +107,41 @@ kbase_ipa_attach_ipa_control(struct kbase_ipa_counter_model_data *model_data) return -ENOMEM; } - for (i = 0; i < num_counters; ++i) { + /* Fill in the description for GPU_ACTIVE counter which is always + * needed, as mentioned above, regardless of the energy model used + * by the CSF GPUs. + */ + perf_counters[cnt_idx].type = KBASE_IPA_CORE_TYPE_CSHW; + perf_counters[cnt_idx].idx = GPU_ACTIVE_CNT_IDX; + perf_counters[cnt_idx].gpu_norm = false; + perf_counters[cnt_idx].scaling_factor = 1; + cnt_idx++; + + for (i = 0; i < model_data->num_top_level_cntrs; ++i) { const struct kbase_ipa_counter *counter = - &model_data->counters_def[i]; + &model_data->top_level_cntrs_def[i]; - perf_counters[i].type = counter->counter_block_type; - perf_counters[i].idx = counter->counter_block_offset; - perf_counters[i].gpu_norm = false; - perf_counters[i].scaling_factor = 1; + perf_counters[cnt_idx].type = counter->counter_block_type; + perf_counters[cnt_idx].idx = counter->counter_block_offset; + perf_counters[cnt_idx].gpu_norm = false; + perf_counters[cnt_idx].scaling_factor = 1; + cnt_idx++; } - err = kbase_ipa_control_register(kbdev, perf_counters, num_counters, - &model_data->ipa_control_cli); + for (i = 0; i < model_data->num_shader_cores_cntrs; ++i) { + const struct kbase_ipa_counter *counter = + &model_data->shader_cores_cntrs_def[i]; + + perf_counters[cnt_idx].type = counter->counter_block_type; + perf_counters[cnt_idx].idx = counter->counter_block_offset; + perf_counters[cnt_idx].gpu_norm = false; + perf_counters[cnt_idx].scaling_factor = 1; + cnt_idx++; + } + + err = kbase_ipa_control_register(kbdev, perf_counters, + model_data->num_counters, + &model_data->ipa_control_client); if (err) dev_err(kbdev->dev, "Failed to register IPA with kbase_ipa_control"); @@ -110,98 +157,87 @@ kbase_ipa_attach_ipa_control(struct kbase_ipa_counter_model_data *model_data) static void kbase_ipa_detach_ipa_control(struct kbase_ipa_counter_model_data *model_data) { - if (model_data->ipa_control_cli) { + if (model_data->ipa_control_client) { kbase_ipa_control_unregister(model_data->kbdev, - model_data->ipa_control_cli); - model_data->ipa_control_cli = NULL; + model_data->ipa_control_client); + model_data->ipa_control_client = NULL; } } -int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) +static int calculate_coeff(struct kbase_ipa_counter_model_data *model_data, + const struct kbase_ipa_counter *const cnt_defs, + size_t num_counters, s32 *counter_coeffs, + u64 *counter_values, u32 active_cycles, u32 *coeffp) { - struct kbase_ipa_counter_model_data *model_data = - (struct kbase_ipa_counter_model_data *)model->model_data; - s64 energy = 0; - size_t i; u64 coeff = 0, coeff_mul = 0; - u32 active_cycles; - u64 ret; - - /* The last argument is supposed to be a pointer to the location that - * will store the time for which GPU has been in protected mode since - * last query. This can be passed as NULL as counter model itself will - * not be used when GPU enters protected mode, as IPA is supposed to - * switch to the simple power model. - */ - ret = kbase_ipa_control_query(model->kbdev, model_data->ipa_control_cli, - model_data->counter_values, - model_data->counters_def_num, NULL); - if (WARN_ON(ret)) - return ret; - - /* Range: 0 (GPU not used at all), to the max sampling interval, say - * 1s, * max GPU frequency (GPU 100% utilized). - * 0 <= active_cycles <= 1 * ~2GHz - * 0 <= active_cycles < 2^31 - */ - active_cycles = model_data->get_active_cycles(model_data); - - /* If the value of the active_cycles is less than the threshold, then - * return an error so that IPA framework can approximate using the - * cached simple model results instead. This may be more accurate - * than extrapolating using a very small counter dump. - */ - if (active_cycles < (u32)max(model_data->min_sample_cycles, 0)) - return -ENODATA; - - /* Range: 1 <= active_cycles < 2^31 */ - active_cycles = max(1u, active_cycles); + s64 total_energy = 0; + size_t i; - /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around - * -2^57 < energy < 2^57 + /* Range for the 'counter_value' is [0, 2^38) + * Range for the 'coeff' is [-2^21, 2^21] + * So range for the 'group_energy' is [-2^59, 2^59) and range for the + * 'total_energy' is +/- 2^59 * number of IPA groups (~16), i.e. + * [-2^63, 2^63). */ - for (i = 0; i < model_data->counters_def_num; i++) { - s32 coeff = model_data->counter_coeffs[i]; - u64 counter_value = model_data->counter_values[i]; + for (i = 0; i < num_counters; i++) { + s32 coeff = counter_coeffs[i]; + u64 counter_value = counter_values[i]; s64 group_energy = kbase_ipa_group_energy(coeff, counter_value); - energy = kbase_ipa_add_saturate(energy, group_energy); - } + if (counter_value > MAX_COUNTER_INCREMENT) { + dev_warn(model_data->kbdev->dev, + "Increment in counter %s more than expected", + cnt_defs[i].name); + return -ERANGE; + } - /* Range: 0 <= coeff < 2^57 */ - if (energy > 0) - coeff = energy; + total_energy = + kbase_ipa_add_saturate(total_energy, group_energy); + } - /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this - * can be constrained further: Counter values can only be increased by - * a theoretical maximum of about 64k per clock cycle. Beyond this, - * we'd have to sample every 1ms to avoid them overflowing at the - * lowest clock frequency (say 100MHz). Therefore, we can write the - * range of 'coeff' in terms of active_cycles: + /* Range: 0 <= coeff < 2^63 */ + if (total_energy >= 0) + coeff = total_energy; + else + dev_dbg(model_data->kbdev->dev, + "Energy value came negative as %lld", total_energy); + + /* Range: 0 <= coeff < 2^63 (because active_cycles >= 1). However, this + * can be constrained further: the value of counters that are being + * used for dynamic power estimation can only increment by about 128 + * maximum per clock cycle. This is because max number of shader + * cores is expected to be 32 (max number of L2 slices is expected to + * be 8) and some counters (per shader core) like SC_BEATS_RD_TEX_EXT & + * SC_EXEC_STARVE_ARITH can increment by 4 every clock cycle. + * Each "beat" is defined as 128 bits and each shader core can + * (currently) do 512 bits read and 512 bits write to/from the L2 + * cache per cycle, so the SC_BEATS_RD_TEX_EXT counter can increment + * [0, 4] per shader core per cycle. + * We can thus write the range of 'coeff' in terms of active_cycles: * * coeff = SUM(coeffN * counterN * num_cores_for_counterN) - * coeff <= SUM(coeffN * counterN) * max_num_cores - * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores - * (substitute max_counter = 2^16 * active_cycles) - * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores - * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 - * coeff <= 2^46 * active_cycles + * coeff <= SUM(coeffN * counterN) * max_cores + * coeff <= num_IPA_groups * max_coeff * max_counter * max_cores + * (substitute max_counter = 2^2 * active_cycles) + * coeff <= num_IPA_groups * max_coeff * 2^2 * active_cycles * max_cores + * coeff <= 2^4 * 2^21 * 2^2 * active_cycles * 2^5 + * coeff <= 2^32 * active_cycles * - * So after the division: 0 <= coeff <= 2^46 + * So after the division: 0 <= coeff <= 2^32 */ coeff = div_u64(coeff, active_cycles); /* Not all models were derived at the same reference voltage. Voltage * scaling is done by multiplying by V^2, so we need to *divide* by * Vref^2 here. - * Range: 0 <= coeff <= 2^49 + * Range: 0 <= coeff <= 2^35 */ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); - /* Range: 0 <= coeff <= 2^52 */ + /* Range: 0 <= coeff <= 2^38 */ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); /* Scale by user-specified integer factor. - * Range: 0 <= coeff_mul < 2^57 + * Range: 0 <= coeff_mul < 2^43 */ coeff_mul = coeff * model_data->scaling_factor; @@ -210,28 +246,133 @@ int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) * becomes fW/(Hz V^2), which are the units of coeff_mul. However, * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide * by 1000. - * Range: 0 <= coeff_mul < 2^47 + * Range: 0 <= coeff_mul < 2^33 */ coeff_mul = div_u64(coeff_mul, 1000u); /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ *coeffp = clamp(coeff_mul, (u64)0, (u64)1 << 16); + return 0; } -int kbase_ipa_counter_common_model_init( - struct kbase_ipa_model *model, - const struct kbase_ipa_counter *ipa_counters_def, - size_t ipa_num_counters, - kbase_ipa_get_active_cycles_callback get_active_cycles, - s32 reference_voltage) +int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ + struct kbase_ipa_counter_model_data *model_data = + (struct kbase_ipa_counter_model_data *)model->model_data; + struct kbase_device *kbdev = model->kbdev; + s32 *counter_coeffs_p = model_data->counter_coeffs; + u64 *cnt_values_p = model_data->counter_values; + const u64 num_counters = model_data->num_counters; + u32 active_cycles; + ktime_t now, diff; + s64 diff_ms; + int ret; + + lockdep_assert_held(&kbdev->ipa.lock); + + /* The last argument is supposed to be a pointer to the location that + * will store the time for which GPU has been in protected mode since + * last query. This can be passed as NULL as counter model itself will + * not be used when GPU enters protected mode, as IPA is supposed to + * switch to the simple power model. + */ + ret = kbase_ipa_control_query(kbdev, + model_data->ipa_control_client, + cnt_values_p, num_counters, NULL); + if (WARN_ON(ret)) + return ret; + + now = ktime_get(); + diff = ktime_sub(now, kbdev->ipa.last_sample_time); + diff_ms = ktime_to_ms(diff); + + kbdev->ipa.last_sample_time = now; + + /* The counter values cannot be relied upon if the sampling interval was + * too long. Typically this will happen when the polling is started + * after the temperature has risen above a certain trip point. After + * that regular calls every 25-100 ms interval are expected. + */ + if (diff_ms > MAX_SAMPLE_INTERVAL_MS) { + dev_dbg(kbdev->dev, + "Last sample was taken %lld milli seconds ago", + diff_ms); + return -EOVERFLOW; + } + + /* Range: 0 (GPU not used at all), to the max sampling interval, say + * 1 seconds, * max GPU frequency (GPU 100% utilized). + * 0 <= active_cycles <= 1 * ~2GHz + * 0 <= active_cycles < 2^31 + */ + if (*cnt_values_p > U32_MAX) { + dev_warn(kbdev->dev, + "Increment in GPU_ACTIVE counter more than expected"); + return -ERANGE; + } + + active_cycles = (u32)*cnt_values_p; + + /* If the value of the active_cycles is less than the threshold, then + * return an error so that IPA framework can approximate using the + * cached simple model results instead. This may be more accurate + * than extrapolating using a very small counter dump. + */ + if (active_cycles < (u32)max(model_data->min_sample_cycles, 0)) + return -ENODATA; + + /* Range: 1 <= active_cycles < 2^31 */ + active_cycles = max(1u, active_cycles); + + cnt_values_p++; + ret = calculate_coeff(model_data, model_data->top_level_cntrs_def, + model_data->num_top_level_cntrs, + counter_coeffs_p, cnt_values_p, active_cycles, + &coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); + if (ret) + return ret; + + cnt_values_p += model_data->num_top_level_cntrs; + counter_coeffs_p += model_data->num_top_level_cntrs; + ret = calculate_coeff(model_data, model_data->shader_cores_cntrs_def, + model_data->num_shader_cores_cntrs, + counter_coeffs_p, cnt_values_p, active_cycles, + &coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); + + return ret; +} + +void kbase_ipa_counter_reset_data(struct kbase_ipa_model *model) +{ + struct kbase_ipa_counter_model_data *model_data = + (struct kbase_ipa_counter_model_data *)model->model_data; + u64 *cnt_values_p = model_data->counter_values; + const u64 num_counters = model_data->num_counters; + int ret; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + ret = kbase_ipa_control_query(model->kbdev, + model_data->ipa_control_client, + cnt_values_p, num_counters, NULL); + WARN_ON(ret); +} + +int kbase_ipa_counter_common_model_init(struct kbase_ipa_model *model, + const struct kbase_ipa_counter *top_level_cntrs_def, + size_t num_top_level_cntrs, + const struct kbase_ipa_counter *shader_cores_cntrs_def, + size_t num_shader_cores_cntrs, + s32 reference_voltage) { + struct kbase_ipa_counter_model_data *model_data; + s32 *counter_coeffs_p; int err = 0; size_t i; - struct kbase_ipa_counter_model_data *model_data; - if (!model || !ipa_counters_def || !ipa_num_counters || - !get_active_cycles) + if (!model || !top_level_cntrs_def || !shader_cores_cntrs_def || + !num_top_level_cntrs || !num_shader_cores_cntrs) return -EINVAL; model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); @@ -239,22 +380,43 @@ int kbase_ipa_counter_common_model_init( return -ENOMEM; model_data->kbdev = model->kbdev; - model_data->counters_def = ipa_counters_def; - model_data->counters_def_num = ipa_num_counters; - model_data->get_active_cycles = get_active_cycles; + + model_data->top_level_cntrs_def = top_level_cntrs_def; + model_data->num_top_level_cntrs = num_top_level_cntrs; + + model_data->shader_cores_cntrs_def = shader_cores_cntrs_def; + model_data->num_shader_cores_cntrs = num_shader_cores_cntrs; model->model_data = (void *)model_data; - for (i = 0; i < model_data->counters_def_num; ++i) { + counter_coeffs_p = model_data->counter_coeffs; + + for (i = 0; i < model_data->num_top_level_cntrs; ++i) { + const struct kbase_ipa_counter *counter = + &model_data->top_level_cntrs_def[i]; + + *counter_coeffs_p = counter->coeff_default_value; + + err = kbase_ipa_model_add_param_s32( + model, counter->name, counter_coeffs_p, 1, false); + if (err) + goto exit; + + counter_coeffs_p++; + } + + for (i = 0; i < model_data->num_shader_cores_cntrs; ++i) { const struct kbase_ipa_counter *counter = - &model_data->counters_def[i]; + &model_data->shader_cores_cntrs_def[i]; + + *counter_coeffs_p = counter->coeff_default_value; - model_data->counter_coeffs[i] = counter->coeff_default_value; err = kbase_ipa_model_add_param_s32( - model, counter->name, &model_data->counter_coeffs[i], 1, - false); + model, counter->name, counter_coeffs_p, 1, false); if (err) goto exit; + + counter_coeffs_p++; } model_data->scaling_factor = DEFAULT_SCALING_FACTOR; diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.h b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.h index 064c101..9383dc2 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.h +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_csf.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IPA_COUNTER_COMMON_CSF_H_ @@ -27,26 +46,25 @@ #include "csf/ipa_control/mali_kbase_csf_ipa_control.h" /* Maximum number of HW counters used by the IPA counter model. */ -#define KBASE_IPA_MAX_COUNTER_DEF_NUM 16 +#define KBASE_IPA_MAX_COUNTER_DEF_NUM 24 struct kbase_ipa_counter_model_data; -typedef u32 (*kbase_ipa_get_active_cycles_callback)( - struct kbase_ipa_counter_model_data *); - /** * struct kbase_ipa_counter_model_data - IPA counter model context per device * @kbdev: Pointer to kbase device - * @ipa_control_cli: Handle returned on registering IPA counter model as a + * @ipa_control_client: Handle returned on registering IPA counter model as a * client of kbase_ipa_control. - * @counters_def: Array of description of HW counters used by the IPA - * counter model. - * @counters_def_num: Number of elements in the array of HW counters. - * @get_active_cycles: Callback to return number of active cycles during - * counter sample period. + * @top_level_cntrs_def: Array of description of HW counters used by the IPA + * counter model for top-level. + * @num_top_level_cntrs: Number of elements in @top_level_cntrs_def array. + * @shader_cores_cntrs_def: Array of description of HW counters used by the IPA + * counter model for shader cores. + * @num_shader_cores_cntrs: Number of elements in @shader_cores_cntrs_def array. * @counter_coeffs: Buffer to store coefficient value used for HW counters * @counter_values: Buffer to store the accumulated value of HW counters * retreived from kbase_ipa_control. + * @num_counters: Number of counters queried from kbase_ipa_control. * @reference_voltage: voltage, in mV, of the operating point used when * deriving the power model coefficients. Range approx * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 @@ -64,12 +82,14 @@ typedef u32 (*kbase_ipa_get_active_cycles_callback)( */ struct kbase_ipa_counter_model_data { struct kbase_device *kbdev; - void *ipa_control_cli; - const struct kbase_ipa_counter *counters_def; - size_t counters_def_num; - kbase_ipa_get_active_cycles_callback get_active_cycles; + void *ipa_control_client; + const struct kbase_ipa_counter *top_level_cntrs_def; + size_t num_top_level_cntrs; + const struct kbase_ipa_counter *shader_cores_cntrs_def; + size_t num_shader_cores_cntrs; s32 counter_coeffs[KBASE_IPA_MAX_COUNTER_DEF_NUM]; u64 counter_values[KBASE_IPA_MAX_COUNTER_DEF_NUM]; + u64 num_counters; s32 reference_voltage; s32 scaling_factor; s32 min_sample_cycles; @@ -97,7 +117,7 @@ struct kbase_ipa_counter { * kbase_ipa_counter_dynamic_coeff() - calculate dynamic power based on HW counters * @model: pointer to instantiated model * @coeffp: pointer to location where calculated power, in - * pW/(Hz V^2), is stored. + * pW/(Hz V^2), is stored for top level and shader cores. * * This is a GPU-agnostic implementation of the get_dynamic_coeff() * function of an IPA model. It relies on the model being populated @@ -108,31 +128,45 @@ struct kbase_ipa_counter { int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); /** + * kbase_ipa_counter_reset_data() - Reset the counters data used for dynamic + * power estimation + * @model: pointer to instantiated model + * + * Retrieve the accumulated value of HW counters from the kbase_ipa_control + * component, without doing any processing, which is effectively a reset as the + * next call to kbase_ipa_counter_dynamic_coeff() will see the increment in + * counter values from this point onwards. + */ +void kbase_ipa_counter_reset_data(struct kbase_ipa_model *model); + +/** * kbase_ipa_counter_common_model_init() - initialize ipa power model - * @model: ipa power model to initialize - * @ipa_counters_def: Array corresponding to the HW counters used in the - * IPA counter model, contains the counter index, default - * value of the coefficient. - * @ipa_num_counters: number of elements in the array @ipa_counters_def - * @get_active_cycles: callback to return the number of cycles the GPU was - * active during the counter sample period. - * @reference_voltage: voltage, in mV, of the operating point used when - * deriving the power model coefficients. - * - * This initialization function performs initialization steps common - * for ipa models based on counter values. In each call, the model - * passes its specific coefficient values per ipa counter group via - * @ipa_counters_def array. + * @model: Pointer to the ipa power model to initialize + * @top_level_cntrs_def: Array corresponding to the HW counters used in the + * top level counter model, contains the counter index, + * default value of the coefficient. + * @num_top_level_cntrs: Number of elements in the array @top_level_cntrs_def + * @shader_cores_cntrs_def: Array corresponding to the HW counters used in the + * shader cores counter model, contains the counter index, + * default value of the coefficient. + * @num_shader_cores_cntrs: Number of elements in the array + * @shader_cores_cntrs_def. + * @reference_voltage: voltage, in mV, of the operating point used when + * deriving the power model coefficients. + * + * This function performs initialization steps common for ipa counter based + * model of all CSF GPUs. The set of counters and their respective weights + * could be different for each GPU. The tuple of counter index and weight + * is passed via @top_level_cntrs_def and @shader_cores_cntrs_def array. * * Return: 0 on success, error code otherwise */ -int kbase_ipa_counter_common_model_init( - struct kbase_ipa_model *model, - const struct kbase_ipa_counter *ipa_counters_def, - size_t ipa_num_counters, - kbase_ipa_get_active_cycles_callback get_active_cycles, - s32 reference_voltage); - +int kbase_ipa_counter_common_model_init(struct kbase_ipa_model *model, + const struct kbase_ipa_counter *top_level_cntrs_def, + size_t num_top_level_cntrs, + const struct kbase_ipa_counter *shader_cores_cntrs_def, + size_t num_shader_cores_cntrs, + s32 reference_voltage); /** * kbase_ipa_counter_common_model_term() - terminate ipa power model * @model: ipa power model to terminate diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.c b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.c index ef1e526..fe01f78 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.c +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. @@ -273,6 +274,12 @@ err0: return err; } +void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model) +{ + /* Currently not implemented */ + WARN_ON_ONCE(1); +} + int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, const struct kbase_ipa_group *ipa_groups_def, size_t ipa_group_size, diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.h b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.h index 02e735f..0147f18 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.h +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_common_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IPA_COUNTER_COMMON_JM_H_ @@ -182,6 +201,19 @@ void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); /** + * kbase_ipa_vinstr_reset_data() - Reset the counters data used for dynamic + * power estimation + * @model: pointer to instantiated model + * + * Currently it is not implemented for JM GPUs. + * When implemented it is expected to retrieve the accumulated value of HW + * counters from the Vinstr component, without doing any processing, which is + * effectively a reset as the next call to kbase_ipa_counter_dynamic_coeff() + * will see the increment in counter values from this point onwards. + */ +void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model); + +/** * kbase_ipa_vinstr_common_model_init() - initialize ipa power model * @model: ipa power model to initialize * @ipa_groups_def: array of ipa groups which sets coefficients for diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_csf.c b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_csf.c index 8cd47a1..1f21a85 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_csf.c +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -24,77 +25,79 @@ #include "mali_kbase.h" /* CSHW counter block offsets */ -#define GPU_ACTIVE (4) +#define MESSAGES_RECEIVED (9) +#define CEU_ACTIVE (40) /* MEMSYS counter block offsets */ -#define MEMSYS_L2_ANY_LOOKUP (25) +#define L2_RD_MSG_IN (16) +#define L2_WR_MSG_IN_STALL (19) +#define L2_SNP_MSG_IN (20) +#define L2_ANY_LOOKUP (25) +#define L2_EXT_READ_BEATS (32) +#define L2_EXT_AR_CNT_Q3 (36) +#define L2_EXT_AW_CNT_Q2 (50) /* SC counter block offsets */ -#define SC_EXEC_INSTR_FMA (27) -#define SC_EXEC_INSTR_MSG (30) -#define SC_TEX_FILT_NUM_OPERATIONS (39) +#define FRAG_FPK_ACTIVE (7) +#define COMPUTE_ACTIVE (22) +#define EXEC_CORE_ACTIVE (26) +#define EXEC_STARVE_ARITH (33) +#define TEX_FILT_NUM_OPS (39) +#define BEATS_RD_TEX_EXT (59) + +/* Tiler counter block offsets */ +#define PRIM_SAT_CULLED (14) + +#define COUNTER_DEF(cnt_name, coeff, cnt_idx, block_type) \ + { \ + .name = cnt_name, \ + .coeff_default_value = coeff, \ + .counter_block_offset = cnt_idx, \ + .counter_block_type = block_type, \ + } -/** - * get_active_cycles() - return the GPU_ACTIVE counter - * @model_data: Pointer to GPU model data. - * - * Return: the number of cycles the GPU was active during the counter sampling - * period. - */ -static u32 kbase_csf_get_active_cycles( - struct kbase_ipa_counter_model_data *model_data) -{ - size_t i; +#define CSHW_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ + COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_CSHW) - for (i = 0; i < model_data->counters_def_num; ++i) { - const struct kbase_ipa_counter *counter = - &model_data->counters_def[i]; +#define MEMSYS_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ + COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_MEMSYS) - if (!strcmp(counter->name, "gpu_active")) - return model_data->counter_values[i]; - } +#define SC_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ + COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_SHADER) - WARN_ON_ONCE(1); - - return 0; -} +#define TILER_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ + COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_TILER) /** Table of description of HW counters used by IPA counter model. * * This table provides a description of each performance counter - * used by the IPA counter model for energy estimation. + * used by the top level counter model for energy estimation. */ -static const struct kbase_ipa_counter ipa_counters_def_todx[] = { - { - .name = "l2_access", - .coeff_default_value = 599800, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - .counter_block_type = KBASE_IPA_CORE_TYPE_MEMSYS, - }, - { - .name = "exec_instr_msg", - .coeff_default_value = 1830200, - .counter_block_offset = SC_EXEC_INSTR_MSG, - .counter_block_type = KBASE_IPA_CORE_TYPE_SHADER, - }, - { - .name = "exec_instr_fma", - .coeff_default_value = 407300, - .counter_block_offset = SC_EXEC_INSTR_FMA, - .counter_block_type = KBASE_IPA_CORE_TYPE_SHADER, - }, - { - .name = "tex_filt_num_operations", - .coeff_default_value = 224500, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - .counter_block_type = KBASE_IPA_CORE_TYPE_SHADER, - }, - { - .name = "gpu_active", - .coeff_default_value = 153800, - .counter_block_offset = GPU_ACTIVE, - .counter_block_type = KBASE_IPA_CORE_TYPE_CSHW, - }, +static const struct kbase_ipa_counter ipa_top_level_cntrs_def_todx[] = { + CSHW_COUNTER_DEF("messages_received", 925749, MESSAGES_RECEIVED), + CSHW_COUNTER_DEF("ceu_active", 25611, CEU_ACTIVE), + + MEMSYS_COUNTER_DEF("l2_ext_read_beats", 3413, L2_EXT_READ_BEATS), + MEMSYS_COUNTER_DEF("l2_ext_ar_cnt_q3", 8141, L2_EXT_AR_CNT_Q3), + MEMSYS_COUNTER_DEF("l2_rd_msg_in", 3231, L2_RD_MSG_IN), + MEMSYS_COUNTER_DEF("l2_ext_aw_cnt_q2", 21714, L2_EXT_AW_CNT_Q2), + MEMSYS_COUNTER_DEF("l2_any_lookup", 110567, L2_ANY_LOOKUP), + MEMSYS_COUNTER_DEF("l2_wr_msg_in_stall", -370971, L2_WR_MSG_IN_STALL), + MEMSYS_COUNTER_DEF("l2_snp_msg_in", 270337, L2_SNP_MSG_IN), + + TILER_COUNTER_DEF("prim_sat_culled", -1094458, PRIM_SAT_CULLED), +}; + + /* This table provides a description of each performance counter + * used by the shader cores counter model for energy estimation. + */ +static const struct kbase_ipa_counter ipa_shader_core_cntrs_def_todx[] = { + SC_COUNTER_DEF("frag_fpk_active", -91312, FRAG_FPK_ACTIVE), + SC_COUNTER_DEF("exec_core_active", 485012, EXEC_CORE_ACTIVE), + SC_COUNTER_DEF("beats_rd_tex_ext", 174174, BEATS_RD_TEX_EXT), + SC_COUNTER_DEF("tex_filt_num_operations", 164419, TEX_FILT_NUM_OPS), + SC_COUNTER_DEF("exec_starve_arith", -59107, EXEC_STARVE_ARITH), + SC_COUNTER_DEF("compute_active", -277940, COMPUTE_ACTIVE), }; #define IPA_POWER_MODEL_OPS(gpu, init_token) \ @@ -103,6 +106,7 @@ static const struct kbase_ipa_counter ipa_counters_def_todx[] = { .init = kbase_ ## init_token ## _power_model_init, \ .term = kbase_ipa_counter_common_model_term, \ .get_dynamic_coeff = kbase_ipa_counter_dynamic_coeff, \ + .reset_counter_data = kbase_ipa_counter_reset_data, \ }; \ KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) @@ -110,13 +114,16 @@ static const struct kbase_ipa_counter ipa_counters_def_todx[] = { static int kbase_ ## gpu ## _power_model_init(\ struct kbase_ipa_model *model) \ { \ - BUILD_BUG_ON(ARRAY_SIZE(ipa_counters_def_ ## gpu) > \ - KBASE_IPA_MAX_COUNTER_DEF_NUM); \ + BUILD_BUG_ON((1 + \ + ARRAY_SIZE(ipa_top_level_cntrs_def_ ## gpu) +\ + ARRAY_SIZE(ipa_shader_core_cntrs_def_ ## gpu)) > \ + KBASE_IPA_MAX_COUNTER_DEF_NUM); \ return kbase_ipa_counter_common_model_init(model, \ - ipa_counters_def_ ## gpu, \ - ARRAY_SIZE(ipa_counters_def_ ## gpu), \ - kbase_csf_get_active_cycles, \ - (reference_voltage)); \ + ipa_top_level_cntrs_def_ ## gpu, \ + ARRAY_SIZE(ipa_top_level_cntrs_def_ ## gpu), \ + ipa_shader_core_cntrs_def_ ## gpu, \ + ARRAY_SIZE(ipa_shader_core_cntrs_def_ ## gpu), \ + (reference_voltage)); \ } \ IPA_POWER_MODEL_OPS(gpu, gpu) @@ -124,10 +131,9 @@ static const struct kbase_ipa_counter ipa_counters_def_todx[] = { #define ALIAS_POWER_MODEL(gpu, as_gpu) \ IPA_POWER_MODEL_OPS(gpu, as_gpu) -/* Currently tBEx energy model is being used, for which reference voltage - * value is 1000 mV. +/* Reference voltage value is 750 mV. */ -STANDARD_POWER_MODEL(todx, 1000); +STANDARD_POWER_MODEL(todx, 750); /* Assuming LODX is an alias of TODX for IPA */ ALIAS_POWER_MODEL(lodx, todx); diff --git a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_jm.c b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_jm.c index 6799d8e..233e6d5 100644 --- a/mali_kbase/ipa/backend/mali_kbase_ipa_counter_jm.c +++ b/mali_kbase/ipa/backend/mali_kbase_ipa_counter_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <linux/thermal.h> #include "mali_kbase_ipa_counter_common_jm.h" @@ -455,6 +457,7 @@ static const struct kbase_ipa_group ipa_groups_def_tbax[] = { .init = kbase_ ## init_token ## _power_model_init, \ .term = kbase_ipa_vinstr_common_model_term, \ .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ + .reset_counter_data = kbase_ipa_vinstr_reset_data, \ }; \ KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) diff --git a/mali_kbase/ipa/mali_kbase_ipa.c b/mali_kbase/ipa/mali_kbase_ipa.c index 5946e4a..848ea93 100644 --- a/mali_kbase/ipa/mali_kbase_ipa.c +++ b/mali_kbase/ipa/mali_kbase_ipa.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include <linux/thermal.h> #include <linux/devfreq_cooling.h> #include <linux/of.h> @@ -27,18 +29,20 @@ #include "mali_kbase_ipa_debugfs.h" #include "mali_kbase_ipa_simple.h" #include "backend/gpu/mali_kbase_pm_internal.h" - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include "backend/gpu/mali_kbase_devfreq.h" #include <linux/pm_opp.h> -#else -#include <linux/opp.h> -#define dev_pm_opp_find_freq_exact opp_find_freq_exact -#define dev_pm_opp_get_voltage opp_get_voltage -#define dev_pm_opp opp -#endif #define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" +/* Polling by thermal governor starts when the temperature exceeds the certain + * trip point. In order to have meaningful value for the counters, when the + * polling starts and first call to kbase_get_real_power() is made, it is + * required to reset the counter values every now and then. + * It is reasonable to do the reset every second if no polling is being done, + * the counter model implementation also assumes max sampling interval of 1 sec. + */ +#define RESET_INTERVAL_MS ((s64)1000) + int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) { int err = 0; @@ -322,6 +326,8 @@ int kbase_ipa_init(struct kbase_device *kbdev) kbdev->ipa.configured_model = default_model; } + kbdev->ipa.last_sample_time = ktime_get(); + end: if (err) kbase_ipa_term_locked(kbdev); @@ -376,7 +382,8 @@ static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, const u32 v2f = v2f_big / 1000; /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. - * Must be < 2^42 to avoid overflowing the return value. */ + * Must be < 2^42 to avoid overflowing the return value. + */ const u64 v2fc = (u64) c * (u64) v2f; /* Range: 0 < v2fc / 1000 < 2^13 mW */ @@ -472,8 +479,8 @@ static u32 get_static_power_locked(struct kbase_device *kbdev, return power; } -#if defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE static unsigned long kbase_get_static_power(struct devfreq *df, unsigned long voltage) #else @@ -482,8 +489,8 @@ static unsigned long kbase_get_static_power(unsigned long voltage) { struct kbase_ipa_model *model; u32 power = 0; -#if defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE struct kbase_device *kbdev = dev_get_drvdata(&df->dev); #else struct kbase_device *kbdev = kbase_find_device(-1); @@ -499,16 +506,64 @@ static unsigned long kbase_get_static_power(unsigned long voltage) mutex_unlock(&kbdev->ipa.lock); -#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) +#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) kbase_release_device(kbdev); #endif return power; } -#if defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +/** + * opp_translate_freq_voltage() - Translate nominal OPP frequency from + * devicetree into the real frequency for + * top-level and shader cores. + * @kbdev: Device pointer + * @nominal_freq: Nominal frequency in Hz. + * @nominal_voltage: Nominal voltage, in mV. + * @freqs: Pointer to array of real frequency values. + * @volts: Pointer to array of voltages. + * + * If there are 2 clock domains, then top-level and shader cores can operate + * at different frequency and voltage level. The nominal frequency ("opp-hz") + * used by devfreq from the devicetree may not be same as the real frequency + * at which top-level and shader cores are operating, so a translation is + * needed. + * Nominal voltage shall always be same as the real voltage for top-level. + */ +static void opp_translate_freq_voltage(struct kbase_device *kbdev, + unsigned long nominal_freq, + unsigned long nominal_voltage, + unsigned long *freqs, + unsigned long *volts) +{ +#ifndef CONFIG_MALI_NO_MALI + u64 core_mask; + + kbase_devfreq_opp_translate(kbdev, nominal_freq, &core_mask, + freqs, volts); + CSTD_UNUSED(core_mask); + + if (kbdev->nr_clocks == 1) { + freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = + freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; + volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = + volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; + } +#else + /* An arbitrary voltage and frequency value can be chosen for testing + * in no mali configuration which may not match with any OPP level. + */ + freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = nominal_freq; + volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = nominal_voltage; + + freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = nominal_freq; + volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = nominal_voltage; +#endif +} + +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE static unsigned long kbase_get_dynamic_power(struct devfreq *df, unsigned long freq, unsigned long voltage) @@ -518,10 +573,13 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, #endif { struct kbase_ipa_model *model; - u32 power_coeff = 0, power = 0; + unsigned long freqs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; + unsigned long volts[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; + u32 power_coeffs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; + u32 power = 0; int err = 0; -#if defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE struct kbase_device *kbdev = dev_get_drvdata(&df->dev); #else struct kbase_device *kbdev = kbase_find_device(-1); @@ -534,19 +592,34 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, model = kbdev->ipa.fallback_model; - err = model->ops->get_dynamic_coeff(model, &power_coeff); - - if (!err) - power = kbase_scale_dynamic_power(power_coeff, freq, voltage); - else + err = model->ops->get_dynamic_coeff(model, power_coeffs); + + if (!err) { + opp_translate_freq_voltage(kbdev, freq, voltage, freqs, volts); + + power = kbase_scale_dynamic_power( + power_coeffs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], + freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], + volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); + + /* Here unlike kbase_get_real_power(), shader core frequency is + * used for the scaling as simple power model is used to obtain + * the value of dynamic coefficient (which is is a fixed value + * retrieved from the device tree). + */ + power += kbase_scale_dynamic_power( + power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], + freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], + volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); + } else dev_err_ratelimited(kbdev->dev, "Model %s returned error code %d\n", model->ops->name, err); mutex_unlock(&kbdev->ipa.lock); -#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) +#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) kbase_release_device(kbdev); #endif @@ -558,10 +631,13 @@ int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, unsigned long voltage) { struct kbase_ipa_model *model; - u32 power_coeff = 0; - int err = 0; + unsigned long freqs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; + unsigned long volts[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; + u32 power_coeffs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; struct kbasep_pm_metrics diff; u64 total_time; + bool skip_utilization_scaling = false; + int err = 0; lockdep_assert_held(&kbdev->ipa.lock); @@ -569,30 +645,62 @@ int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, model = get_current_model(kbdev); - err = model->ops->get_dynamic_coeff(model, &power_coeff); + err = model->ops->get_dynamic_coeff(model, power_coeffs); /* If the counter model returns an error (e.g. switching back to * protected mode and failing to read counters, or a counter sample * with too few cycles), revert to the fallback model. */ if (err && model != kbdev->ipa.fallback_model) { + /* No meaningful scaling for GPU utilization can be done if + * the sampling interval was too long. This is equivalent to + * assuming GPU was busy throughout (similar to what is done + * during protected mode). + */ + if (err == -EOVERFLOW) + skip_utilization_scaling = true; + model = kbdev->ipa.fallback_model; - err = model->ops->get_dynamic_coeff(model, &power_coeff); + err = model->ops->get_dynamic_coeff(model, power_coeffs); } - if (err) + if (WARN_ON(err)) return err; - *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + opp_translate_freq_voltage(kbdev, freq, voltage, freqs, volts); - /* time_busy / total_time cannot be >1, so assigning the 64-bit - * result of div_u64 to *power cannot overflow. - */ - total_time = diff.time_busy + (u64) diff.time_idle; - *power = div_u64(*power * (u64) diff.time_busy, - max(total_time, 1ull)); + *power = kbase_scale_dynamic_power( + power_coeffs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], + freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], + volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); + + if (power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]) { + unsigned long freq = freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]; + + /* As per the HW team, the top-level frequency needs to be used + * for the scaling if the counter based model was used as + * counter values are normalized with the GPU_ACTIVE counter + * value, which increments at the rate of top-level frequency. + */ + if (model != kbdev->ipa.fallback_model) + freq = freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; + + *power += kbase_scale_dynamic_power( + power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], + freq, volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); + } + + if (!skip_utilization_scaling) { + /* time_busy / total_time cannot be >1, so assigning the 64-bit + * result of div_u64 to *power cannot overflow. + */ + total_time = diff.time_busy + (u64) diff.time_idle; + *power = div_u64(*power * (u64) diff.time_busy, + max(total_time, 1ull)); + } - *power += get_static_power_locked(kbdev, model, voltage); + *power += get_static_power_locked(kbdev, model, + volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); return err; } @@ -616,16 +724,40 @@ int kbase_get_real_power(struct devfreq *df, u32 *power, } KBASE_EXPORT_TEST_API(kbase_get_real_power); -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -struct devfreq_cooling_ops kbase_ipa_power_model_ops = { -#else struct devfreq_cooling_power kbase_ipa_power_model_ops = { -#endif .get_static_power = &kbase_get_static_power, .get_dynamic_power = &kbase_get_dynamic_power, -#if defined(CONFIG_MALI_PWRSOFT_765) || \ - LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE .get_real_power = &kbase_get_real_power, #endif }; KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); + +void kbase_ipa_reset_data(struct kbase_device *kbdev) +{ + ktime_t now, diff; + s64 elapsed_time; + + mutex_lock(&kbdev->ipa.lock); + + now = ktime_get(); + diff = ktime_sub(now, kbdev->ipa.last_sample_time); + elapsed_time = ktime_to_ms(diff); + + if (elapsed_time > RESET_INTERVAL_MS) { + struct kbasep_pm_metrics diff; + struct kbase_ipa_model *model; + + kbase_pm_get_dvfs_metrics( + kbdev, &kbdev->ipa.last_metrics, &diff); + + model = get_current_model(kbdev); + if (model != kbdev->ipa.fallback_model) + model->ops->reset_counter_data(model); + + kbdev->ipa.last_sample_time = ktime_get(); + } + + mutex_unlock(&kbdev->ipa.lock); +} diff --git a/mali_kbase/ipa/mali_kbase_ipa.h b/mali_kbase/ipa/mali_kbase_ipa.h index 5f04169..d979540 100644 --- a/mali_kbase/ipa/mali_kbase_ipa.h +++ b/mali_kbase/ipa/mali_kbase_ipa.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IPA_H_ @@ -28,6 +47,20 @@ struct devfreq; /** + * enum kbase_ipa_block_type - Type of block for which power estimation is done. + * + * @KBASE_IPA_BLOCK_TYPE_TOP_LEVEL: Top-level block, that covers CSHW, + * MEMSYS, Tiler. + * @KBASE_IPA_BLOCK_TYPE_SHADER_CORES: All Shader cores. + * @KBASE_IPA_BLOCK_TYPE_NUM: Number of blocks. + */ +enum kbase_ipa_block_type { + KBASE_IPA_BLOCK_TYPE_TOP_LEVEL, + KBASE_IPA_BLOCK_TYPE_SHADER_CORES, + KBASE_IPA_BLOCK_TYPE_NUM +}; + +/** * struct kbase_ipa_model - Object describing a particular IPA model. * @kbdev: pointer to kbase device * @model_data: opaque pointer to model specific data, accessed @@ -89,7 +122,8 @@ struct kbase_ipa_model_ops { int (*init)(struct kbase_ipa_model *model); /* Called immediately after init(), or when a parameter is changed, so * that any coefficients derived from model parameters can be - * recalculated. */ + * recalculated + */ int (*recalculate)(struct kbase_ipa_model *model); void (*term)(struct kbase_ipa_model *model); /* @@ -101,7 +135,9 @@ struct kbase_ipa_model_ops { * is then scaled by the IPA framework according to the current OPP's * frequency and voltage. * - * Return: 0 on success, or an error code. + * Return: 0 on success, or an error code. -EOVERFLOW error code will + * indicate that sampling interval was too large and no meaningful + * scaling for GPU utiliation can be done. */ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp); /* @@ -115,6 +151,18 @@ struct kbase_ipa_model_ops { * Return: 0 on success, or an error code. */ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); + + /* + * reset_counter_data() - Reset the HW counter data used for calculating + * dynamic power coefficient + * @model: pointer to model + * + * This method is currently applicable only to the counter based model. + * The next call to get_dynamic_coeff() will have to calculate the + * dynamic power coefficient based on the HW counter data generated + * from this point onwards. + */ + void (*reset_counter_data)(struct kbase_ipa_model *model); }; /** @@ -248,11 +296,20 @@ int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, unsigned long voltage); #endif /* MALI_UNIT_TEST */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; -#else extern struct devfreq_cooling_power kbase_ipa_power_model_ops; -#endif + +/** + * kbase_ipa_reset_data() - Reset the data required for power estimation. + * @kbdev: Pointer to kbase device. + * + * This function is called to ensure a meaningful baseline for + * kbase_get_real_power(), when thermal governor starts the polling, and + * that is achieved by updating the GPU utilization metrics and retrieving + * the accumulated value of HW counters. + * Basically this function collects all the data required for power estimation + * but does not process it. + */ +void kbase_ipa_reset_data(struct kbase_device *kbdev); #else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c index 267a586..82288fd 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.c +++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. @@ -28,7 +29,7 @@ #include "mali_kbase_ipa.h" #include "mali_kbase_ipa_debugfs.h" -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) #define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE #endif @@ -160,7 +161,8 @@ int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, return -ENOMEM; /* 'name' is stack-allocated for array elements, so copy it into - * heap-allocated storage */ + * heap-allocated storage + */ param->name = kstrdup(name, GFP_KERNEL); if (!param->name) { diff --git a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h index a983d9c..6641f1e 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_debugfs.h +++ b/mali_kbase/ipa/mali_kbase_ipa_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IPA_DEBUGFS_H_ diff --git a/mali_kbase/ipa/mali_kbase_ipa_simple.c b/mali_kbase/ipa/mali_kbase_ipa_simple.c index 852559e..c55daa1 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_simple.c +++ b/mali_kbase/ipa/mali_kbase_ipa_simple.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2016-2018, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,20 +35,18 @@ #include "mali_kbase_ipa_simple.h" #include "mali_kbase_ipa_debugfs.h" -#if MALI_UNIT_TEST - -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) -static unsigned long dummy_temp; +/* This is used if the dynamic power for top-level is estimated separately + * through the counter model. To roughly match the contribution of top-level + * power in the total dynamic power, when calculated through counter model, + * this scalar is used for the dynamic coefficient specified in the device tree + * for simple power model. This value was provided by the HW team after + * taking all the power data collected and dividing top level power by shader + * core power and then averaging it across all samples. + */ +#define TOP_LEVEL_DYN_COEFF_SCALER (3) -static int kbase_simple_power_model_get_dummy_temp( - struct thermal_zone_device *tz, - unsigned long *temp) -{ - *temp = READ_ONCE(dummy_temp); - return 0; -} +#if MALI_UNIT_TEST -#else static int dummy_temp; static int kbase_simple_power_model_get_dummy_temp( @@ -57,7 +56,6 @@ static int kbase_simple_power_model_get_dummy_temp( *temp = READ_ONCE(dummy_temp); return 0; } -#endif /* Intercept calls to the kernel function using a macro */ #ifdef thermal_zone_get_temp @@ -143,16 +141,13 @@ static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) /* We can't call thermal_zone_get_temp() directly in model_static_coeff(), * because we don't know if tz->lock is held in the same thread. So poll it in - * a separate thread to get around this. */ + * a separate thread to get around this. + */ static int poll_temperature(void *data) { struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *) data; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) - unsigned long temp; -#else int temp; -#endif while (!kthread_should_stop()) { struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); @@ -208,7 +203,21 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *) model->model_data; +#if MALI_USE_CSF + /* On CSF GPUs, the dynamic power for top-level and shader cores is + * estimated separately. Currently there is a single dynamic + * coefficient value provided in the device tree for simple model. + * As per the discussion with HW team the coefficient value needs to + * be scaled down for top-level to limit its contribution in the + * total dyanmic power. + */ + coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = + model_data->dynamic_coefficient / TOP_LEVEL_DYN_COEFF_SCALER; + coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = + model_data->dynamic_coefficient; +#else *coeffp = model_data->dynamic_coefficient; +#endif return 0; } diff --git a/mali_kbase/ipa/mali_kbase_ipa_simple.h b/mali_kbase/ipa/mali_kbase_ipa_simple.h index fed67d5..f5f8609 100644 --- a/mali_kbase/ipa/mali_kbase_ipa_simple.h +++ b/mali_kbase/ipa/mali_kbase_ipa_simple.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IPA_SIMPLE_H_ diff --git a/mali_kbase/jm/mali_base_jm_kernel.h b/mali_kbase/jm/mali_base_jm_kernel.h index ddac90a..de317eb 100644 --- a/mali_kbase/jm/mali_base_jm_kernel.h +++ b/mali_kbase/jm/mali_base_jm_kernel.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,7 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ + #ifndef _BASE_JM_KERNEL_H_ #define _BASE_JM_KERNEL_H_ @@ -752,11 +772,15 @@ typedef u8 base_jd_prio; #define BASE_JD_PRIO_HIGH ((base_jd_prio)1) /* Low atom priority. */ #define BASE_JD_PRIO_LOW ((base_jd_prio)2) +/* Real-Time atom priority. This is a priority higher than BASE_JD_PRIO_HIGH, + * BASE_JD_PRIO_MEDIUM, and BASE_JD_PRIO_LOW + */ +#define BASE_JD_PRIO_REALTIME ((base_jd_prio)3) /* Count of the number of priority levels. This itself is not a valid * base_jd_prio setting */ -#define BASE_JD_NR_PRIO_LEVELS 3 +#define BASE_JD_NR_PRIO_LEVELS 4 /** * struct base_jd_atom_v2 - Node of a dependency graph used to submit a diff --git a/mali_kbase/jm/mali_kbase_jm_defs.h b/mali_kbase/jm/mali_kbase_jm_defs.h index 22e3fb3..2aeedad 100644 --- a/mali_kbase/jm/mali_kbase_jm_defs.h +++ b/mali_kbase/jm/mali_kbase_jm_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Definitions (types, defines, etcs) specific to Job Manager Kbase. * They are placed here to allow the hierarchy of header files to work. @@ -781,6 +798,7 @@ struct kbase_jd_renderpass { * @jit_pending_alloc: A list of just-in-time memory allocation * soft-jobs which will be reattempted after the * impending free of other active allocations. + * @max_priority: Max priority level allowed for this context. */ struct kbase_jd_context { struct mutex lock; @@ -801,6 +819,7 @@ struct kbase_jd_context { struct list_head jit_atoms_head; struct list_head jit_pending_alloc; + int max_priority; }; /** diff --git a/mali_kbase/jm/mali_kbase_jm_ioctl.h b/mali_kbase/jm/mali_kbase_jm_ioctl.h index 76ac278..7e2515b 100644 --- a/mali_kbase/jm/mali_kbase_jm_ioctl.h +++ b/mali_kbase/jm/mali_kbase_jm_ioctl.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_JM_IOCTL_H_ @@ -114,9 +133,12 @@ * - Added kernel side cache ops needed hint * 11.29: * - Reserve ioctl 52 + * 11.30: + * - Add a new priority level BASE_JD_PRIO_REALTIME + * - Add ioctl 54: This controls the priority setting. */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 29 +#define BASE_UK_VERSION_MINOR 30 /** * struct kbase_ioctl_version_check - Check version compatibility between @@ -133,8 +155,6 @@ struct kbase_ioctl_version_check { #define KBASE_IOCTL_VERSION_CHECK \ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) -#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ - _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) /** * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel @@ -213,4 +233,8 @@ union kbase_kinstr_jm_fd { #define KBASE_IOCTL_KINSTR_JM_FD \ _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) + +#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ + _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) + #endif /* _KBASE_JM_IOCTL_H_ */ diff --git a/mali_kbase/jm/mali_kbase_jm_js.h b/mali_kbase/jm/mali_kbase_jm_js.h index 6c222ce..6978b56 100644 --- a/mali_kbase/jm/mali_kbase_jm_js.h +++ b/mali_kbase/jm/mali_kbase_jm_js.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -584,7 +603,6 @@ void kbase_js_set_timeouts(struct kbase_device *kbdev); */ void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); - /** * kbase_js_update_ctx_priority - update the context priority * @@ -889,4 +907,17 @@ static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) return kbasep_js_relative_priority_to_atom[prio_idx]; } +/** + * kbase_js_priority_check - Check the priority requested + * + * @kbdev: Device pointer + * @priority: Requested priority + * + * This will determine whether the requested priority can be satisfied. + * + * Return: The same or lower priority than requested. + */ + +base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority); + #endif /* _KBASE_JM_JS_H_ */ diff --git a/mali_kbase/jm/mali_kbase_js_defs.h b/mali_kbase/jm/mali_kbase_js_defs.h index 0b48615..7a68f7a 100644 --- a/mali_kbase/jm/mali_kbase_js_defs.h +++ b/mali_kbase/jm/mali_kbase_js_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /** * @file mali_kbase_js.h * Job Scheduler Type Definitions @@ -51,7 +69,8 @@ struct kbase_jd_atom; typedef u32 kbase_context_flags; /** Callback function run on all of a context's jobs registered with the Job - * Scheduler */ + * Scheduler + */ typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); /** @@ -168,7 +187,8 @@ enum { * Internal atom priority defines for kbase_jd_atom::sched_prio */ enum { - KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, + KBASE_JS_ATOM_SCHED_PRIO_REALTIME = 0, + KBASE_JS_ATOM_SCHED_PRIO_HIGH, KBASE_JS_ATOM_SCHED_PRIO_MED, KBASE_JS_ATOM_SCHED_PRIO_LOW, KBASE_JS_ATOM_SCHED_PRIO_COUNT, @@ -198,7 +218,8 @@ enum { */ struct kbasep_js_device_data { /* Sub-structure to collect together Job Scheduling data used in IRQ - * context. The hwaccess_lock must be held when accessing. */ + * context. The hwaccess_lock must be held when accessing. + */ struct runpool_irq { /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. * When bit 'N' is set in this, it indicates whether the context bound to address space @@ -219,14 +240,17 @@ struct kbasep_js_device_data { * - error detection in debug builds * - Optimization: it is undefined for a signed int to overflow, and so * the compiler can optimize for that never happening (thus, no masking - * is required on updating the variable) */ + * is required on updating the variable) + */ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; /* * Affinity management and tracking */ /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates - * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ + * that slot 'n' is using core i (i.e. + * slot_affinity_refcount[n][i] > 0) + */ u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; /** Refcount for each core owned by each slot. Used to generate the * slot_affinities array of bitvectors @@ -234,7 +258,8 @@ struct kbasep_js_device_data { * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, * because it is refcounted only when a job is definitely about to be * submitted to a slot, and is de-refcounted immediately after a job - * finishes */ + * finishes + */ s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; } runpool_irq; @@ -260,7 +285,9 @@ struct kbasep_js_device_data { s8 nr_all_contexts_running; /** Core Requirements to match up with base_js_atom's core_req memeber - * @note This is a write-once member, and so no locking is required to read */ + * @note This is a write-once member, and so no locking is required to + * read + */ base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ @@ -282,16 +309,19 @@ struct kbasep_js_device_data { bool softstop_always; #endif /* CONFIG_MALI_DEBUG */ - /** The initalized-flag is placed at the end, to avoid cache-pollution (we should - * only be using this during init/term paths). - * @note This is a write-once member, and so no locking is required to read */ + /** The initialized-flag is placed at the end, to avoid cache-pollution + * (we should only be using this during init/term paths). + * @note This is a write-once member, and so no locking is required to + * read + */ int init_status; /* Number of contexts that can currently be pulled from */ u32 nr_contexts_pullable; /* Number of contexts that can either be pulled from or are currently - * running */ + * running + */ atomic_t nr_contexts_runnable; /** Value for JS_SOFT_JOB_TIMEOUT */ @@ -339,9 +369,10 @@ struct kbasep_js_kctx_info { struct kbase_jsctx { struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ - /** Number of jobs <b>ready to run</b> - does \em not include the jobs waiting in - * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr - * for such jobs*/ + /** Number of jobs <b>ready to run</b> - does \em not include + * the jobs waiting in the dispatcher, and dependency-only + * jobs. See kbase_jd_context::job_nr for such jobs + */ u32 nr_jobs; /** Context Attributes: @@ -351,7 +382,7 @@ struct kbasep_js_kctx_info { /** * Wait queue to wait for KCTX_SHEDULED flag state changes. - * */ + */ wait_queue_head_t is_scheduled_wait; /** Link implementing JS queues. Context can be present on one @@ -361,13 +392,15 @@ struct kbasep_js_kctx_info { } ctx; /* The initalized-flag is placed at the end, to avoid cache-pollution (we should - * only be using this during init/term paths) */ + * only be using this during init/term paths) + */ int init_status; }; /** Subset of atom state that can be available after jd_done_nolock() is called * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), - * because the original atom could disappear. */ + * because the original atom could disappear. + */ struct kbasep_js_atom_retained_state { /** Event code - to determine whether the atom has finished */ enum base_jd_event_code event_code; diff --git a/mali_kbase/mali_base_hwconfig_features.h b/mali_kbase/mali_base_hwconfig_features.h index c0ce65f..ebbe6bf 100644 --- a/mali_kbase/mali_base_hwconfig_features.h +++ b/mali_kbase/mali_base_hwconfig_features.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, @@ -56,6 +75,7 @@ enum base_hw_feature { BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_ASN_HASH, BASE_HW_FEATURE_END }; @@ -480,6 +500,7 @@ static const enum base_hw_feature base_hw_features_tTUx[] = { BASE_HW_FEATURE_AARCH64_MMU, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_ASN_HASH, BASE_HW_FEATURE_END }; diff --git a/mali_kbase/mali_base_hwconfig_issues.h b/mali_kbase/mali_base_hwconfig_issues.h index d0a3545..603c8cd 100644 --- a/mali_kbase/mali_base_hwconfig_issues.h +++ b/mali_kbase/mali_base_hwconfig_issues.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, @@ -612,7 +631,6 @@ static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; @@ -621,7 +639,6 @@ static const enum base_hw_issue base_hw_issues_model_tGRx[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; @@ -629,7 +646,6 @@ static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; @@ -638,7 +654,6 @@ static const enum base_hw_issue base_hw_issues_model_tVAx[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; @@ -646,7 +661,6 @@ static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; @@ -655,7 +669,6 @@ static const enum base_hw_issue base_hw_issues_model_tTUx[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_END }; diff --git a/mali_kbase/mali_base_kernel.h b/mali_kbase/mali_base_kernel.h index aa6fb9f..259922b 100644 --- a/mali_kbase/mali_base_kernel.h +++ b/mali_kbase/mali_base_kernel.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Base structures shared with the kernel. */ @@ -646,7 +663,6 @@ struct gpu_raw_gpu_props { u64 tiler_present; u64 l2_present; u64 stack_present; - u32 l2_features; u32 core_features; u32 mem_features; @@ -673,6 +689,7 @@ struct gpu_raw_gpu_props { u32 coherency_mode; u32 thread_tls_alloc; + u64 gpu_features; }; /** diff --git a/mali_kbase/mali_base_mem_priv.h b/mali_kbase/mali_base_mem_priv.h index 844a025..fe193c0 100644 --- a/mali_kbase/mali_base_mem_priv.h +++ b/mali_kbase/mali_base_mem_priv.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _BASE_MEM_PRIV_H_ #define _BASE_MEM_PRIV_H_ diff --git a/mali_kbase/mali_kbase.h b/mali_kbase/mali_kbase.h index fd176a5..9981991 100644 --- a/mali_kbase/mali_kbase.h +++ b/mali_kbase/mali_kbase.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _KBASE_H_ #define _KBASE_H_ @@ -38,7 +55,7 @@ #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/sched.h> -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) #include <linux/sched/mm.h> #endif #include <linux/slab.h> @@ -456,7 +473,11 @@ static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, bool gpu_lost) { - atomic_set(&kbdev->pm.gpu_lost, (gpu_lost ? 1 : 0)); + const int new_val = (gpu_lost ? 1 : 0); + const int cur_val = atomic_xchg(&kbdev->pm.gpu_lost, new_val); + + if (new_val != cur_val) + KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, new_val); } #endif diff --git a/mali_kbase/mali_kbase_as_fault_debugfs.c b/mali_kbase/mali_kbase_as_fault_debugfs.c index 2e2e394..7851b6d 100644 --- a/mali_kbase/mali_kbase_as_fault_debugfs.c +++ b/mali_kbase/mali_kbase_as_fault_debugfs.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_as_fault_debugfs.h b/mali_kbase/mali_kbase_as_fault_debugfs.h index 496d8b1..19a3e17 100644 --- a/mali_kbase/mali_kbase_as_fault_debugfs.h +++ b/mali_kbase/mali_kbase_as_fault_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_AS_FAULT_DEBUG_FS_H diff --git a/mali_kbase/mali_kbase_bits.h b/mali_kbase/mali_kbase_bits.h index 2c11093..e1625c6 100644 --- a/mali_kbase/mali_kbase_bits.h +++ b/mali_kbase/mali_kbase_bits.h @@ -21,12 +21,22 @@ *//* SPDX-License-Identifier: GPL-2.0 */ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_BITS_H_ diff --git a/mali_kbase/mali_kbase_cache_policy.c b/mali_kbase/mali_kbase_cache_policy.c index 27a03cf..1cc8738 100644 --- a/mali_kbase/mali_kbase_cache_policy.c +++ b/mali_kbase/mali_kbase_cache_policy.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2018, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - /* * Cache Policy API. */ diff --git a/mali_kbase/mali_kbase_cache_policy.h b/mali_kbase/mali_kbase_cache_policy.h index 8a1e529..85c328a 100644 --- a/mali_kbase/mali_kbase_cache_policy.h +++ b/mali_kbase/mali_kbase_cache_policy.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2013, 2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * Cache Policy API. */ diff --git a/mali_kbase/mali_kbase_caps.h b/mali_kbase/mali_kbase_caps.h index b201a60..1901c89 100644 --- a/mali_kbase/mali_kbase_caps.h +++ b/mali_kbase/mali_kbase_caps.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /** * @file mali_kbase_caps.h * diff --git a/mali_kbase/mali_kbase_ccswe.c b/mali_kbase/mali_kbase_ccswe.c index 87d5aaa..c48c4aa 100644 --- a/mali_kbase/mali_kbase_ccswe.c +++ b/mali_kbase/mali_kbase_ccswe.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -50,7 +51,6 @@ void kbase_ccswe_init(struct kbase_ccswe *self) spin_lock_init(&self->access); } -KBASE_EXPORT_TEST_API(kbase_ccswe_init); u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns) { @@ -63,7 +63,6 @@ u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns) return result; } -KBASE_EXPORT_TEST_API(kbase_ccswe_cycle_at); void kbase_ccswe_freq_change( struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq) @@ -87,7 +86,6 @@ void kbase_ccswe_freq_change( exit: spin_unlock_irqrestore(&self->access, flags); } -KBASE_EXPORT_TEST_API(kbase_ccswe_freq_change); void kbase_ccswe_reset(struct kbase_ccswe *self) { @@ -102,4 +100,3 @@ void kbase_ccswe_reset(struct kbase_ccswe *self) spin_unlock_irqrestore(&self->access, flags); } - diff --git a/mali_kbase/mali_kbase_ccswe.h b/mali_kbase/mali_kbase_ccswe.h index 3a7cf73..77769f8 100644 --- a/mali_kbase/mali_kbase_ccswe.h +++ b/mali_kbase/mali_kbase_ccswe.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CCSWE_H_ diff --git a/mali_kbase/mali_kbase_config.c b/mali_kbase/mali_kbase_config.c index ce7070d..a4bd10f 100644 --- a/mali_kbase/mali_kbase_config.c +++ b/mali_kbase/mali_kbase_config.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2015, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - #include <mali_kbase.h> #include <mali_kbase_defs.h> #include <mali_kbase_config_defaults.h> diff --git a/mali_kbase/mali_kbase_config.h b/mali_kbase/mali_kbase_config.h index 57456e2..4ff7f8c 100644 --- a/mali_kbase/mali_kbase_config.h +++ b/mali_kbase/mali_kbase_config.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2017, 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2017, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_config.h * Configuration API and Attributes for KBase diff --git a/mali_kbase/mali_kbase_config_defaults.h b/mali_kbase/mali_kbase_config_defaults.h index 48c728e..972ea77 100644 --- a/mali_kbase/mali_kbase_config_defaults.h +++ b/mali_kbase/mali_kbase_config_defaults.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_core_linux.c b/mali_kbase/mali_kbase_core_linux.c index dcdac19..97cd3cb 100644 --- a/mali_kbase/mali_kbase_core_linux.c +++ b/mali_kbase/mali_kbase_core_linux.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -110,12 +111,7 @@ #include <mali_kbase_config.h> -#if (KERNEL_VERSION(3, 13, 0) <= LINUX_VERSION_CODE) #include <linux/pm_opp.h> -#else -#include <linux/opp.h> -#endif - #include <linux/pm_runtime.h> #include <tl/mali_kbase_timeline.h> @@ -144,25 +140,6 @@ #define KBASE_API_MAJ(api_version) ((api_version >> 20) & 0xFFF) /** - * mali_kbase_api_version_to_maj_min - convert an api_version to a min/maj pair - * - * @api_version: API version to convert - * @major: Major version number (must not exceed 12 bits) - * @minor: Major version number (must not exceed 12 bits) - */ -void mali_kbase_api_version_to_maj_min(unsigned long api_version, u16 *maj, u16 *min) -{ - if (WARN_ON(!maj)) - return; - - if (WARN_ON(!min)) - return; - - *maj = KBASE_API_MAJ(api_version); - *min = KBASE_API_MIN(api_version); -} - -/** * kbase capabilities table */ typedef struct mali_kbase_capability_def { @@ -434,25 +411,6 @@ static int kbase_api_handshake_dummy(struct kbase_file *kfile, return -EPERM; } -/** - * enum mali_error - Mali error codes shared with userspace - * - * This is subset of those common Mali errors that can be returned to userspace. - * Values of matching user and kernel space enumerators MUST be the same. - * MALI_ERROR_NONE is guaranteed to be 0. - * - * @MALI_ERROR_NONE: Success - * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver - * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure - * @MALI_ERROR_FUNCTION_FAILED: Generic error code - */ -enum mali_error { - MALI_ERROR_NONE = 0, - MALI_ERROR_OUT_OF_GPU_MEMORY, - MALI_ERROR_OUT_OF_MEMORY, - MALI_ERROR_FUNCTION_FAILED, -}; - static struct kbase_device *to_kbase_device(struct device *dev) { return dev_get_drvdata(dev); @@ -530,9 +488,9 @@ void kbase_release_device(struct kbase_device *kbdev) EXPORT_SYMBOL(kbase_release_device); #ifdef CONFIG_DEBUG_FS -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ - !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ - LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && \ + !(KERNEL_VERSION(4, 4, 28) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE) /* * Older versions, before v4.6, of the kernel doesn't have * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 @@ -1585,52 +1543,80 @@ static int kbasep_ioctl_cs_cpu_queue_dump(struct kbase_context *kctx, #endif /* MALI_USE_CSF */ -#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ - do { \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ - return function(arg); \ +static int kbasep_ioctl_context_priority_check(struct kbase_context *kctx, + struct kbase_ioctl_context_priority_check *priority_check) +{ +#if MALI_USE_CSF + priority_check->priority = kbase_csf_priority_check(kctx->kbdev, priority_check->priority); +#else + base_jd_prio req_priority = (base_jd_prio)priority_check->priority; + + priority_check->priority = (u8)kbase_js_priority_check(kctx->kbdev, req_priority); +#endif + return 0; +} + +#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ + do { \ + int ret; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + ret = function(arg); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ + #function); \ + return ret; \ } while (0) -#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ - do { \ - type param; \ - int err; \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - return function(arg, ¶m); \ +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(arg, ¶m); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ + #function); \ + return ret; \ } while (0) -#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - memset(¶m, 0, sizeof(param)); \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - return ret; \ +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + memset(¶m, 0, sizeof(param)); \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ + #function); \ + return ret; \ } while (0) -#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - return ret; \ +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ + #function); \ + return ret; \ } while (0) static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) @@ -2007,6 +1993,12 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) kctx); break; #endif /* MALI_UNIT_TEST */ + case KBASE_IOCTL_CONTEXT_PRIORITY_CHECK: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CONTEXT_PRIORITY_CHECK, + kbasep_ioctl_context_priority_check, + struct kbase_ioctl_context_priority_check, + kctx); + break; } dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); @@ -2123,7 +2115,8 @@ static unsigned int kbase_poll(struct file *filp, poll_table *wait) void kbase_event_wakeup(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx); - + dev_dbg(kctx->kbdev->dev, "Waking event queue for context %p\n", + (void *)kctx); wake_up_interruptible(&kctx->event_queue); } @@ -2817,7 +2810,8 @@ static ssize_t set_js_scheduling_period(struct device *dev, /* If no contexts have been scheduled since js_timeouts was last written * to, the new timeouts might not have been latched yet. So check if an - * update is pending and use the new values if necessary. */ + * update is pending and use the new values if necessary. + */ /* Use previous 'new' scheduling period as a base if present. */ old_period = js_data->scheduling_period_ns; @@ -4374,6 +4368,8 @@ int kbase_device_pm_init(struct kbase_device *kbdev) } } } else { + kbdev->arb.arb_if = NULL; + kbdev->arb.arb_dev = NULL; err = power_control_init(kbdev); } #else @@ -4398,7 +4394,7 @@ void kbase_device_pm_term(struct kbase_device *kbdev) int power_control_init(struct kbase_device *kbdev) { -#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) +#ifndef CONFIG_OF /* Power control initialization requires at least the capability to get * regulators and clocks from the device tree, as well as parsing * arrays of unsigned integer values. @@ -4493,12 +4489,6 @@ int power_control_init(struct kbase_device *kbdev) * on the device tree of the platform shouldn't prevent the driver * from completing its initialization. */ -#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ - !defined(LSK_OPPV2_BACKPORT)) - err = of_init_opp_table(kbdev->dev); - CSTD_UNUSED(err); -#else - #if defined(CONFIG_PM_OPP) #if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ defined(CONFIG_REGULATOR)) @@ -4510,8 +4500,6 @@ int power_control_init(struct kbase_device *kbdev) err = dev_pm_opp_of_add_table(kbdev->dev); CSTD_UNUSED(err); #endif /* CONFIG_PM_OPP */ - -#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ return 0; clocks_probe_defer: @@ -4520,20 +4508,13 @@ clocks_probe_defer: regulator_put(kbdev->regulators[i]); #endif return err; -#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ +#endif /* CONFIG_OF */ } void power_control_term(struct kbase_device *kbdev) { unsigned int i; -#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ - !defined(LSK_OPPV2_BACKPORT)) -#if KERNEL_VERSION(3, 19, 0) <= LINUX_VERSION_CODE - of_free_opp_table(kbdev->dev); -#endif -#else - #if defined(CONFIG_PM_OPP) dev_pm_opp_of_remove_table(kbdev->dev); #if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ @@ -4543,8 +4524,6 @@ void power_control_term(struct kbase_device *kbdev) #endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ #endif /* CONFIG_PM_OPP */ -#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ - for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { if (kbdev->clocks[i]) { if (__clk_is_enabled(kbdev->clocks[i])) @@ -4555,15 +4534,14 @@ void power_control_term(struct kbase_device *kbdev) break; } -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ - && defined(CONFIG_REGULATOR) +#if defined(CONFIG_OF) && defined(CONFIG_REGULATOR) for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { if (kbdev->regulators[i]) { regulator_put(kbdev->regulators[i]); kbdev->regulators[i] = NULL; } } -#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ +#endif } #ifdef MALI_KBASE_BUILD @@ -4765,7 +4743,8 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) kbase_instr_backend_debugfs_init(kbdev); #endif /* fops_* variables created by invocations of macro - * MAKE_QUIRK_ACCESSORS() above. */ + * MAKE_QUIRK_ACCESSORS() above. + */ debugfs_create_file("quirks_sc", 0644, kbdev->mali_debugfs_directory, kbdev, &fops_sc_quirks); @@ -4931,6 +4910,149 @@ void buslog_term(struct kbase_device *kbdev) } #endif +#if MALI_USE_CSF +/** + * csg_scheduling_period_store - Store callback for the csg_scheduling_period + * sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the csg_scheduling_period sysfs file is written + * to. It checks the data written, and if valid updates the reset timeout. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t csg_scheduling_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int csg_scheduling_period; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtouint(buf, 0, &csg_scheduling_period); + if (ret || csg_scheduling_period == 0) { + dev_err(kbdev->dev, + "Couldn't process csg_scheduling_period write operation.\n" + "Use format 'csg_scheduling_period_ms', and csg_scheduling_period_ms > 0\n"); + return -EINVAL; + } + + kbase_csf_scheduler_lock(kbdev); + kbdev->csf.scheduler.csg_scheduling_period_ms = csg_scheduling_period; + dev_dbg(kbdev->dev, "CSG scheduling period: %ums\n", + csg_scheduling_period); + kbase_csf_scheduler_unlock(kbdev); + + return count; +} + +/** + * csg_scheduling_period_show - Show callback for the csg_scheduling_period + * sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current reset timeout. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t csg_scheduling_period_show(struct device *dev, + struct device_attribute *attr, + char *const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + kbdev->csf.scheduler.csg_scheduling_period_ms); + + return ret; +} + +static DEVICE_ATTR(csg_scheduling_period, 0644, csg_scheduling_period_show, + csg_scheduling_period_store); + +/** + * fw_timeout_store - Store callback for the fw_timeout sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the fw_timeout sysfs file is written to. It + * checks the data written, and if valid updates the reset timeout. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t fw_timeout_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int fw_timeout; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtouint(buf, 0, &fw_timeout); + if (ret || fw_timeout == 0) { + dev_err(kbdev->dev, "%s\n%s\n%u", + "Couldn't process fw_timeout write operation.", + "Use format 'fw_timeout_ms', and fw_timeout_ms > 0", + FIRMWARE_PING_INTERVAL_MS); + return -EINVAL; + } + + kbase_csf_scheduler_lock(kbdev); + kbdev->csf.fw_timeout_ms = fw_timeout; + kbase_csf_scheduler_unlock(kbdev); + dev_dbg(kbdev->dev, "Firmware timeout: %ums\n", fw_timeout); + + return count; +} + +/** + * fw_timeout_show - Show callback for the firmware timeout sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current reset timeout. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t fw_timeout_show(struct device *dev, + struct device_attribute *attr, char *const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->csf.fw_timeout_ms); + + return ret; +} + +static DEVICE_ATTR(fw_timeout, 0644, fw_timeout_show, fw_timeout_store); +#endif /* MALI_USE_CSF */ + static struct attribute *kbase_scheduling_attrs[] = { #if !MALI_USE_CSF &dev_attr_serialize_jobs.attr, @@ -4958,6 +5080,9 @@ static struct attribute *kbase_attrs[] = { &dev_attr_reset_timeout.attr, #if !MALI_USE_CSF &dev_attr_js_scheduling_period.attr, +#else + &dev_attr_csg_scheduling_period.attr, + &dev_attr_fw_timeout.attr, #endif /* !MALI_USE_CSF */ &dev_attr_power_policy.attr, &dev_attr_core_mask.attr, @@ -5136,8 +5261,11 @@ static int kbase_device_suspend(struct device *dev) kbase_pm_suspend(kbdev); -#if defined(CONFIG_MALI_DEVFREQ) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbase_pm_metrics_stop(kbdev); +#endif + +#ifdef CONFIG_MALI_DEVFREQ dev_dbg(dev, "Callback %s\n", __func__); if (kbdev->devfreq) { kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); @@ -5165,8 +5293,11 @@ static int kbase_device_resume(struct device *dev) kbase_pm_resume(kbdev); -#if defined(CONFIG_MALI_DEVFREQ) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbase_pm_metrics_start(kbdev); +#endif + +#ifdef CONFIG_MALI_DEVFREQ dev_dbg(dev, "Callback %s\n", __func__); if (kbdev->devfreq) { mutex_lock(&kbdev->pm.lock); @@ -5199,8 +5330,12 @@ static int kbase_device_runtime_suspend(struct device *dev) return -ENODEV; dev_dbg(dev, "Callback %s\n", __func__); -#if defined(CONFIG_MALI_DEVFREQ) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbase_pm_metrics_stop(kbdev); +#endif + +#ifdef CONFIG_MALI_DEVFREQ if (kbdev->devfreq) kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); #endif @@ -5238,8 +5373,11 @@ static int kbase_device_runtime_resume(struct device *dev) dev_dbg(dev, "runtime resume\n"); } -#if defined(CONFIG_MALI_DEVFREQ) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbase_pm_metrics_start(kbdev); +#endif + +#ifdef CONFIG_MALI_DEVFREQ if (kbdev->devfreq) kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); #endif diff --git a/mali_kbase/mali_kbase_cs_experimental.h b/mali_kbase/mali_kbase_cs_experimental.h index caba2cd..8a3daa0 100644 --- a/mali_kbase/mali_kbase_cs_experimental.h +++ b/mali_kbase/mali_kbase_cs_experimental.h @@ -19,14 +19,23 @@ * SPDX-License-Identifier: GPL-2.0 * *//* SPDX-License-Identifier: GPL-2.0 */ - /* + * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/mali_kbase/mali_kbase_ctx_sched.c b/mali_kbase/mali_kbase_ctx_sched.c index 70b498a..953db9e 100644 --- a/mali_kbase/mali_kbase_ctx_sched.c +++ b/mali_kbase/mali_kbase_ctx_sched.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. @@ -44,7 +45,8 @@ int kbase_ctx_sched_init(struct kbase_device *kbdev) int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; /* These two must be recalculated if nr_hw_address_spaces changes - * (e.g. for HW workarounds) */ + * (e.g. for HW workarounds) + */ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; kbdev->as_free = as_present; /* All ASs initially free */ @@ -259,7 +261,7 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( found_kctx = kbdev->as_to_kctx[as_nr]; - if (found_kctx != NULL) + if (!WARN_ON(found_kctx == NULL)) kbase_ctx_sched_retain_ctx_refcount(found_kctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -273,23 +275,35 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, unsigned long flags; struct kbase_context *found_kctx; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +struct kbase_context *kbase_ctx_sched_as_to_ctx_nolock( + struct kbase_device *kbdev, size_t as_nr) +{ + struct kbase_context *found_kctx; + if (WARN_ON(kbdev == NULL)) return NULL; if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) return NULL; - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + lockdep_assert_held(&kbdev->hwaccess_lock); found_kctx = kbdev->as_to_kctx[as_nr]; if (found_kctx) { - if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0)) + if (atomic_read(&found_kctx->refcount) <= 0) found_kctx = NULL; } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return found_kctx; } @@ -351,3 +365,45 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); } + +#if MALI_USE_CSF +bool kbase_ctx_sched_refcount_mmu_flush(struct kbase_context *kctx, + bool sync) +{ + struct kbase_device *kbdev; + bool added_ref = false; + unsigned long flags; + + if (WARN_ON(kctx == NULL)) + return added_ref; + + kbdev = kctx->kbdev; + + if (WARN_ON(kbdev == NULL)) + return added_ref; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + added_ref = kbase_ctx_sched_inc_refcount_nolock(kctx); + + WARN_ON(added_ref && + (kctx->mmu_flush_pend_state != KCTX_MMU_FLUSH_NOT_PEND)); + + if (!added_ref && (kctx->as_nr != KBASEP_AS_NR_INVALID)) { + enum kbase_ctx_mmu_flush_pending_state new_state = + sync ? KCTX_MMU_FLUSH_PEND_SYNC : + KCTX_MMU_FLUSH_PEND_NO_SYNC; + + WARN_ON(kctx != kbdev->as_to_kctx[kctx->as_nr]); + + if (kctx->mmu_flush_pend_state != KCTX_MMU_FLUSH_PEND_SYNC) + kctx->mmu_flush_pend_state = new_state; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + return added_ref; +} +#endif diff --git a/mali_kbase/mali_kbase_ctx_sched.h b/mali_kbase/mali_kbase_ctx_sched.h index 1affa71..5e5e242 100644 --- a/mali_kbase/mali_kbase_ctx_sched.h +++ b/mali_kbase/mali_kbase_ctx_sched.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_CTX_SCHED_H_ @@ -168,6 +187,21 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, size_t as_nr); /** + * kbase_ctx_sched_as_to_ctx_nolock - Lookup a context based on its current + * address space. + * @kbdev: The device for which the returned context must belong + * @as_nr: address space assigned to the context of interest + * + * The following lock must be held by the caller: + * * kbase_device::hwaccess_lock + * + * Return: a valid struct kbase_context on success or NULL on failure, + * indicating that no context was found in as_nr. + */ +struct kbase_context *kbase_ctx_sched_as_to_ctx_nolock( + struct kbase_device *kbdev, size_t as_nr); + +/** * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy, * preventing it from being scheduled out. * @kctx: Context to be refcounted @@ -206,4 +240,25 @@ bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); */ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); +#if MALI_USE_CSF +/** + * kbase_ctx_sched_refcount_mmu_flush - Refcount the context for the MMU flush + * operation. + * + * @kctx: Context to be refcounted. + * @sync: Flag passed to the caller function kbase_mmu_flush_invalidate(). + * + * This function takes a reference on the context for the MMU flush operation. + * The refcount is taken only if the context is busy/active. + * If the context isn't active but has a GPU address space slot assigned to it + * then a flag is set to indicate that MMU flush operation is pending, which + * will be performed when the context becomes active. + * + * Return: true if refcount succeeded and the address space slot will not be + * reassigned, false if the refcount failed (because the context was inactive) + */ +bool kbase_ctx_sched_refcount_mmu_flush(struct kbase_context *kctx, + bool sync); +#endif + #endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/mali_kbase/mali_kbase_debug.c b/mali_kbase/mali_kbase_debug.c index 118f787..82d709d 100644 --- a/mali_kbase/mali_kbase_debug.c +++ b/mali_kbase/mali_kbase_debug.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2014, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - #include <mali_kbase.h> static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { diff --git a/mali_kbase/mali_kbase_debug.h b/mali_kbase/mali_kbase_debug.h index 2fdb72d..75e4ed1 100644 --- a/mali_kbase/mali_kbase_debug.h +++ b/mali_kbase/mali_kbase_debug.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2015, 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _KBASE_DEBUG_H #define _KBASE_DEBUG_H diff --git a/mali_kbase/mali_kbase_debug_job_fault.c b/mali_kbase/mali_kbase_debug_job_fault.c index dbc774d..5a1bc2d 100644 --- a/mali_kbase/mali_kbase_debug_job_fault.c +++ b/mali_kbase/mali_kbase_debug_job_fault.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2016, 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -518,23 +519,24 @@ void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) /* * Initialize the relevant data structure per context */ -void kbase_debug_job_fault_context_init(struct kbase_context *kctx) +int kbase_debug_job_fault_context_init(struct kbase_context *kctx) { /* We need allocate double size register range * Because this memory will keep the register address and value */ kctx->reg_dump = vmalloc(0x4000 * 2); - if (kctx->reg_dump == NULL) - return; - - if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { - vfree(kctx->reg_dump); - kctx->reg_dump = NULL; + if (kctx->reg_dump != NULL) { + if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == + false) { + vfree(kctx->reg_dump); + kctx->reg_dump = NULL; + } + INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); + atomic_set(&kctx->job_fault_count, 0); } - INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); - atomic_set(&kctx->job_fault_count, 0); + return 0; } /* diff --git a/mali_kbase/mali_kbase_debug_job_fault.h b/mali_kbase/mali_kbase_debug_job_fault.h index ef69627..c147153 100644 --- a/mali_kbase/mali_kbase_debug_job_fault.h +++ b/mali_kbase/mali_kbase_debug_job_fault.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_JOB_FAULT_H @@ -54,8 +73,9 @@ void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); * kbase_debug_job_fault_context_init - Initialize the relevant * data structure per context * @kctx: KBase context pointer + * @return 0 on success */ -void kbase_debug_job_fault_context_init(struct kbase_context *kctx); +int kbase_debug_job_fault_context_init(struct kbase_context *kctx); /** * kbase_debug_job_fault_context_term - Release the relevant diff --git a/mali_kbase/mali_kbase_debug_mem_view.c b/mali_kbase/mali_kbase_debug_mem_view.c index 4788137..305007c 100644 --- a/mali_kbase/mali_kbase_debug_mem_view.c +++ b/mali_kbase/mali_kbase_debug_mem_view.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -32,10 +33,6 @@ #ifdef CONFIG_DEBUG_FS -#if (KERNEL_VERSION(4, 1, 0) > LINUX_VERSION_CODE) -#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) -#endif - struct debug_mem_mapping { struct list_head node; @@ -179,6 +176,13 @@ static int debug_mem_zone_open(struct rb_root *rbtree, /* Empty region - ignore */ continue; + if (reg->flags & KBASE_REG_PROTECTED) { + /* CPU access to protected memory is forbidden - so + * skip this GPU virtual region. + */ + continue; + } + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); if (!mapping) { ret = -ENOMEM; @@ -222,19 +226,19 @@ static int debug_mem_open(struct inode *i, struct file *file) kbase_gpu_vm_lock(kctx); ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); - if (0 != ret) { + if (ret != 0) { kbase_gpu_vm_unlock(kctx); goto out; } ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); - if (0 != ret) { + if (ret != 0) { kbase_gpu_vm_unlock(kctx); goto out; } ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); - if (0 != ret) { + if (ret != 0) { kbase_gpu_vm_unlock(kctx); goto out; } diff --git a/mali_kbase/mali_kbase_debug_mem_view.h b/mali_kbase/mali_kbase_debug_mem_view.h index b948b7c..bee7a53 100644 --- a/mali_kbase/mali_kbase_debug_mem_view.h +++ b/mali_kbase/mali_kbase_debug_mem_view.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2015, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2013-2015, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUG_MEM_VIEW_H diff --git a/mali_kbase/mali_kbase_debugfs_helper.c b/mali_kbase/mali_kbase_debugfs_helper.c index 37e507b..c48af37 100644 --- a/mali_kbase/mali_kbase_debugfs_helper.c +++ b/mali_kbase/mali_kbase_debugfs_helper.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -90,6 +91,59 @@ static int set_attr_from_string( return err; } +int kbase_debugfs_string_validator(char *const buf) +{ + size_t index; + int err = 0; + char *ptr = buf; + + for (index = 0; *ptr; ++index) { + unsigned long test_number; + size_t len; + + /* Drop leading spaces */ + while (*ptr == ' ') + ptr++; + + /* Strings passed into the validator will be NULL terminated + * by nature, so here strcspn only needs to delimit by + * newlines, spaces and NULL terminator (delimited natively). + */ + len = strcspn(ptr, "\n "); + if (len == 0) { + /* No more values (allow this) */ + break; + } + + /* Substitute a nul terminator for a space character to make + * the substring valid for kstrtoul, and then replace it back. + */ + if (ptr[len] == ' ') { + ptr[len] = '\0'; + err = kstrtoul(ptr, 0, &test_number); + ptr[len] = ' '; + + /* len should only be incremented if there is a valid + * number to follow - otherwise this will skip over + * the NULL terminator in cases with no ending newline + */ + len++; + } else { + /* This would occur at the last element before a space + * or a NULL terminator. + */ + err = kstrtoul(ptr, 0, &test_number); + } + + if (err) + break; + /* Skip the substring (including any premature nul terminator) + */ + ptr += len; + } + return err; +} + int kbase_debugfs_helper_set_attr_from_string( const char *const buf, void *const array, size_t const nelems, kbase_debugfs_helper_set_attr_fn const set_attr_fn) @@ -100,6 +154,13 @@ int kbase_debugfs_helper_set_attr_from_string( if (!wbuf) return -ENOMEM; + /* validate string before actually writing values */ + err = kbase_debugfs_string_validator(wbuf); + if (err) { + kfree(wbuf); + return err; + } + err = set_attr_from_string(wbuf, array, nelems, set_attr_fn); @@ -154,6 +215,14 @@ int kbase_debugfs_helper_seq_write(struct file *const file, } buf[count] = '\0'; + + /* validate string before actually writing values */ + err = kbase_debugfs_string_validator(buf); + if (err) { + kfree(buf); + return err; + } + err = set_attr_from_string(buf, array, nelems, set_attr_fn); kfree(buf); diff --git a/mali_kbase/mali_kbase_debugfs_helper.h b/mali_kbase/mali_kbase_debugfs_helper.h index c3c9efa..c1e1689 100644 --- a/mali_kbase/mali_kbase_debugfs_helper.h +++ b/mali_kbase/mali_kbase_debugfs_helper.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DEBUGFS_HELPER_H_ @@ -59,6 +78,29 @@ int kbase_debugfs_helper_set_attr_from_string( kbase_debugfs_helper_set_attr_fn set_attr_fn); /** + * kbase_debugfs_string_validator - Validate a string to be written to a + * debugfs file for any incorrect formats + * or wrong values. + * + * This function is to be used before any writes to debugfs values are done + * such that any strings with erroneous values (such as octal 09 or + * hexadecimal 0xGH are fully ignored) - without this validation, any correct + * values before the first incorrect one will still be entered into the + * debugfs file. This essentially iterates the values through kstrtoul to see + * if it is valid. + * + * It is largely similar to set_attr_from_string to iterate through the values + * of the input string. This function also requires the input string to be + * writable. + * + * @buf: Null-terminated string to validate. + * + * Return: 0 with no error, else -22 (the invalid return value of kstrtoul) if + * any value in the string was wrong or with an incorrect format. + */ +int kbase_debugfs_string_validator(char *const buf); + +/** * typedef kbase_debugfs_helper_get_attr_fn - Type of function to get an * attribute value from an array * diff --git a/mali_kbase/mali_kbase_defs.h b/mali_kbase/mali_kbase_defs.h index ab1f380..b4cca7a 100644 --- a/mali_kbase/mali_kbase_defs.h +++ b/mali_kbase/mali_kbase_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_defs.h * @@ -40,7 +57,11 @@ #include <mali_kbase_instr_defs.h> #include <mali_kbase_pm.h> #include <mali_kbase_gpuprops_types.h> +#if MALI_USE_CSF +#include <mali_kbase_hwcnt_backend_csf.h> +#else #include <mali_kbase_hwcnt_backend_jm.h> +#endif #include <protected_mode_switcher.h> #include <linux/atomic.h> @@ -75,8 +96,7 @@ #include <linux/regulator/consumer.h> #include <linux/memory_group_manager.h> -#if defined(CONFIG_PM_RUNTIME) || \ - (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM) #define KBASE_PM_RUNTIME 1 #endif @@ -138,17 +158,22 @@ */ #define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) +#if MALI_USE_CSF +/* The buffer count of CSF hwcnt backend ring buffer, which is used when CSF + * hwcnt backend allocate the ring buffer to communicate with CSF firmware for + * HWC dump samples. + * To meet the hardware requirement, this number MUST be power of 2, otherwise, + * CSF hwcnt backend creation will be failed. + */ +#define KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT (128) +#endif + /* Maximum number of clock/regulator pairs that may be referenced by * the device node. * This is dependent on support for of_property_read_u64_array() in the * kernel. */ -#if (KERNEL_VERSION(4, 0, 0) <= LINUX_VERSION_CODE) || \ - defined(LSK_OPPV2_BACKPORT) #define BASE_MAX_NR_CLOCKS_REGULATORS (2) -#else -#define BASE_MAX_NR_CLOCKS_REGULATORS (1) -#endif /* Forward declarations */ struct kbase_context; @@ -178,11 +203,7 @@ struct kbase_io_access { * @buf: array of kbase_io_access */ struct kbase_io_history { -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) bool enabled; -#else - u32 enabled; -#endif spinlock_t lock; size_t count; @@ -429,6 +450,12 @@ struct kbase_pm_device_data { * The state of the arbiter VM machine */ struct kbase_arbiter_vm_state *arb_vm_state; + + /** + * Used by virtualization to notify the arbiter that there are users + * waiting for the GPU so that it can request and resume the driver. + */ + atomic_t gpu_users_waiting; #endif /* CONFIG_MALI_ARBITER_SUPPORT */ /** @@ -882,6 +909,8 @@ struct kbase_process { * Job Scheduler * @l2_size_override: Used to set L2 cache size via device tree blob * @l2_hash_override: Used to set L2 cache hash via device tree blob + * @l2_hash_values_override: true if @l2_hash_values is valid. + * @l2_hash_values: Used to set L2 asn_hash via device tree blob * @process_root: rb_tree root node for maintaining a rb_tree of * kbase_process based on key tgid(thread group ID). * @dma_buf_root: rb_tree root node for maintaining a rb_tree of @@ -896,6 +925,7 @@ struct kbase_process { * @gpu_mem_usage_lock: This spinlock should be held while accounting * @total_gpu_pages for both native and dma-buf imported * allocations. + * @pcm_dev: The priority control manager device. */ struct kbase_device { u32 hw_quirks_sc; @@ -964,6 +994,9 @@ struct kbase_device { s8 nr_hw_address_spaces; s8 nr_user_address_spaces; +#if MALI_USE_CSF + struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; +#else struct kbase_hwcnt { /* The lock should be used when accessing any of the following members */ spinlock_t lock; @@ -974,6 +1007,7 @@ struct kbase_device { struct kbase_instr_backend backend; } hwcnt; +#endif struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; struct kbase_hwcnt_context *hwcnt_gpu_ctx; @@ -1007,17 +1041,10 @@ struct kbase_device { struct kbase_devfreq_opp *devfreq_table; int num_opps; struct kbasep_pm_metrics last_devfreq_metrics; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) struct kbase_devfreq_queue_info devfreq_queue; -#endif #ifdef CONFIG_DEVFREQ_THERMAL -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) - struct devfreq_cooling_device *devfreq_cooling; -#else struct thermal_cooling_device *devfreq_cooling; -#endif bool ipa_protection_mode_switched; struct { /* Access to this struct must be with ipa.lock held */ @@ -1033,6 +1060,10 @@ struct kbase_device { /* true if use of fallback model has been forced by the User */ bool force_fallback_model; + /* Records the time when counters, used for dynamic energy + * estimation, were last sampled. + */ + ktime_t last_sample_time; } ipa; #endif /* CONFIG_DEVFREQ_THERMAL */ #endif /* CONFIG_MALI_DEVFREQ */ @@ -1057,7 +1088,7 @@ struct kbase_device { #if !MALI_CUSTOMER_RELEASE struct { - u16 reg_offset; + u32 reg_offset; } regs_dump_debugfs_data; #endif /* !MALI_CUSTOMER_RELEASE */ #endif /* CONFIG_DEBUG_FS */ @@ -1074,8 +1105,7 @@ struct kbase_device { bool poweroff_pending; - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) +#if (KERNEL_VERSION(4, 4, 0) <= LINUX_VERSION_CODE) bool infinite_cache_active_default; #else u32 infinite_cache_active_default; @@ -1118,6 +1148,8 @@ struct kbase_device { u8 l2_size_override; u8 l2_hash_override; + bool l2_hash_values_override; + u32 l2_hash_values[ASN_HASH_COUNT]; #if MALI_USE_CSF /* CSF object for the GPU device. */ @@ -1155,6 +1187,8 @@ struct kbase_device { /* Pointer to the arbiter device */ struct kbase_arbiter_device arb; #endif + /* Priority Control Manager device */ + struct priority_control_manager_device *pcm_dev; }; /** @@ -1295,6 +1329,30 @@ enum kbase_context_flags { #endif /* !MALI_JIT_PRESSURE_LIMIT_BASE */ }; +#if MALI_USE_CSF +/** + * enum kbase_ctx_mmu_flush_pending_state - State for the pending mmu flush + * operation for a kbase context. + * + * @KCTX_MMU_FLUSH_NOT_PEND: Set when there is no MMU flush operation pending + * for a kbase context or deferred flush operation + * is performed. + * + * @KCTX_MMU_FLUSH_PEND_NO_SYNC: Set when the MMU flush operation is deferred + * for a kbase context when it is inactive and + * the sync flag passed is 0. + * + * @KCTX_MMU_FLUSH_PEND_SYNC: Set when the MMU flush operation is deferred + * for a kbase context when it is inactive and + * the sync flag passed is 1. + */ +enum kbase_ctx_mmu_flush_pending_state { + KCTX_MMU_FLUSH_NOT_PEND, + KCTX_MMU_FLUSH_PEND_NO_SYNC, + KCTX_MMU_FLUSH_PEND_SYNC, +}; +#endif + struct kbase_sub_alloc { struct list_head link; struct page *page; @@ -1555,6 +1613,8 @@ struct kbase_sub_alloc { * @atoms_count: Number of GPU atoms currently in use, per priority * @create_flags: Flags used in context creation. * @kinstr_jm: Kernel job manager instrumentation context handle + * @tl_kctx_list_node: List item into the device timeline's list of + * contexts, for timeline summarization. * * A kernel base context is an entity among which the GPU is scheduled. * Each context has its own GPU address space. @@ -1701,6 +1761,16 @@ struct kbase_context { #if !MALI_USE_CSF struct kbase_kinstr_jm *kinstr_jm; #endif + struct list_head tl_kctx_list_node; + +#if MALI_USE_CSF + /* Tracks if the MMU flush operations are pending for the context. + * The flush required due to unmap is also tracked. + * It is supposed to be in KCTX_MMU_FLUSH_NOT_PEND state whilst a + * context is active and shall be updated with mmu_hw_mutex lock held. + */ + enum kbase_ctx_mmu_flush_pending_state mmu_flush_pend_state; +#endif }; #ifdef CONFIG_MALI_CINSTR_GWT diff --git a/mali_kbase/mali_kbase_disjoint_events.c b/mali_kbase/mali_kbase_disjoint_events.c index b5ac414..ea41565 100644 --- a/mali_kbase/mali_kbase_disjoint_events.c +++ b/mali_kbase/mali_kbase_disjoint_events.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_dma_fence.c b/mali_kbase/mali_kbase_dma_fence.c index 7665d8d..027e530 100644 --- a/mali_kbase/mali_kbase_dma_fence.c +++ b/mali_kbase/mali_kbase_dma_fence.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2016, 2020 ARM Limited. All rights reserved. @@ -56,7 +57,7 @@ static int kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, struct ww_acquire_ctx *ctx) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) struct reservation_object *content_res = NULL; #else struct dma_resv *content_res = NULL; @@ -206,7 +207,7 @@ out: } static void -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) #else kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) @@ -226,7 +227,7 @@ kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) kbase_dma_fence_queue_work(katom); } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) static int kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, struct reservation_object *resv, @@ -238,7 +239,7 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, bool exclusive) #endif { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *excl_fence = NULL; struct fence **shared_fences = NULL; #else @@ -302,7 +303,7 @@ out: return err; } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) void kbase_dma_fence_add_reservation(struct reservation_object *resv, struct kbase_dma_fence_resv_info *info, bool exclusive) @@ -331,7 +332,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, struct kbase_dma_fence_resv_info *info) { int err, i; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; @@ -360,7 +361,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, } for (i = 0; i < info->dma_fence_resv_count; i++) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) struct reservation_object *obj = info->resv_objs[i]; #else struct dma_resv *obj = info->resv_objs[i]; diff --git a/mali_kbase/mali_kbase_dma_fence.h b/mali_kbase/mali_kbase_dma_fence.h index f439895..b53ead1 100644 --- a/mali_kbase/mali_kbase_dma_fence.h +++ b/mali_kbase/mali_kbase_dma_fence.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DMA_FENCE_H_ @@ -44,7 +63,7 @@ struct kbase_context; * reservation objects. */ struct kbase_dma_fence_resv_info { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) struct reservation_object **resv_objs; #else struct dma_resv **resv_objs; @@ -63,7 +82,7 @@ struct kbase_dma_fence_resv_info { * reservation_objects. At the same time keeps track of which objects require * exclusive access in dma_fence_excl_bitmap. */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) void kbase_dma_fence_add_reservation(struct reservation_object *resv, struct kbase_dma_fence_resv_info *info, bool exclusive); diff --git a/mali_kbase/mali_kbase_dummy_job_wa.c b/mali_kbase/mali_kbase_dummy_job_wa.c index a5a7ad7..ac6d953 100644 --- a/mali_kbase/mali_kbase_dummy_job_wa.c +++ b/mali_kbase/mali_kbase_dummy_job_wa.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_dummy_job_wa.h b/mali_kbase/mali_kbase_dummy_job_wa.h index e194950..d480291 100644 --- a/mali_kbase/mali_kbase_dummy_job_wa.h +++ b/mali_kbase/mali_kbase_dummy_job_wa.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DUMMY_JOB_WORKAROUND_ diff --git a/mali_kbase/mali_kbase_dvfs_debugfs.c b/mali_kbase/mali_kbase_dvfs_debugfs.c index 438b528..3c7930b 100644 --- a/mali_kbase/mali_kbase_dvfs_debugfs.c +++ b/mali_kbase/mali_kbase_dvfs_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -38,9 +39,16 @@ static int kbasep_dvfs_utilization_debugfs_show(struct seq_file *file, void *dat { struct kbase_device *kbdev = file->private; +#if MALI_USE_CSF + seq_printf(file, "busy_time: %u idle_time: %u protm_time: %u\n", + kbdev->pm.backend.metrics.values.time_busy, + kbdev->pm.backend.metrics.values.time_idle, + kbdev->pm.backend.metrics.values.time_in_protm); +#else seq_printf(file, "busy_time: %u idle_time: %u\n", kbdev->pm.backend.metrics.values.time_busy, kbdev->pm.backend.metrics.values.time_idle); +#endif return 0; } diff --git a/mali_kbase/mali_kbase_dvfs_debugfs.h b/mali_kbase/mali_kbase_dvfs_debugfs.h index 4f9e3fc..cd17ac2 100644 --- a/mali_kbase/mali_kbase_dvfs_debugfs.h +++ b/mali_kbase/mali_kbase_dvfs_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_DVFS_DEBUGFS_H_ diff --git a/mali_kbase/mali_kbase_event.c b/mali_kbase/mali_kbase_event.c index 5adb80f..f00cef2 100644 --- a/mali_kbase/mali_kbase_event.c +++ b/mali_kbase/mali_kbase_event.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2016,2018-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - #include <mali_kbase.h> #include <mali_kbase_debug.h> #include <tl/mali_kbase_tracepoints.h> @@ -154,7 +153,8 @@ static int kbase_event_coalesce(struct kbase_context *kctx) const int event_count = kctx->event_coalesce_count; /* Join the list of pending events onto the tail of the main list - and reset it */ + * and reset it + */ list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); kctx->event_coalesce_count = 0; @@ -237,7 +237,7 @@ int kbase_event_init(struct kbase_context *kctx) kctx->event_coalesce_count = 0; kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); - if (NULL == kctx->event_workq) + if (kctx->event_workq == NULL) return -EINVAL; return 0; diff --git a/mali_kbase/mali_kbase_fence.c b/mali_kbase/mali_kbase_fence.c index 5e04acf..edf1d52 100644 --- a/mali_kbase/mali_kbase_fence.c +++ b/mali_kbase/mali_kbase_fence.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. @@ -29,7 +30,7 @@ /* Spin lock protecting all Mali fences as fence->lock. */ static DEFINE_SPINLOCK(kbase_fence_lock); -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence * kbase_fence_out_new(struct kbase_jd_atom *katom) #else @@ -37,7 +38,7 @@ struct dma_fence * kbase_fence_out_new(struct kbase_jd_atom *katom) #endif { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; @@ -98,7 +99,7 @@ kbase_fence_free_callbacks(struct kbase_jd_atom *katom) return res; } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) int kbase_fence_add_callback(struct kbase_jd_atom *katom, struct fence *fence, diff --git a/mali_kbase/mali_kbase_fence.h b/mali_kbase/mali_kbase_fence.h index e12f1f9..0705598 100644 --- a/mali_kbase/mali_kbase_fence.h +++ b/mali_kbase/mali_kbase_fence.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_FENCE_H_ @@ -35,7 +54,7 @@ #include "mali_kbase_fence_defs.h" #include "mali_kbase.h" -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) extern const struct fence_ops kbase_fence_ops; #else extern const struct dma_fence_ops kbase_fence_ops; @@ -49,7 +68,7 @@ extern const struct dma_fence_ops kbase_fence_ops; * @node: List head for linking this callback to the katom */ struct kbase_fence_cb { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence_cb fence_cb; struct fence *fence; #else @@ -66,7 +85,7 @@ struct kbase_fence_cb { * * return: A new fence object on success, NULL on failure. */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); #else struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); @@ -169,7 +188,7 @@ static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, * Return: 0 on success: fence was either already signaled, or callback was * set up. Negative error code is returned on error. */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) int kbase_fence_add_callback(struct kbase_jd_atom *katom, struct fence *fence, fence_func_t callback); diff --git a/mali_kbase/mali_kbase_fence_defs.h b/mali_kbase/mali_kbase_fence_defs.h index 3030296..d8f1193 100644 --- a/mali_kbase/mali_kbase_fence_defs.h +++ b/mali_kbase/mali_kbase_fence_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_FENCE_DEFS_H_ @@ -30,7 +49,7 @@ #include <linux/version.h> -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) #include <linux/fence.h> @@ -53,7 +72,7 @@ #include <linux/dma-fence.h> -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) +#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) #define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \ (a)->status ?: 1 \ : 0) diff --git a/mali_kbase/mali_kbase_fence_ops.c b/mali_kbase/mali_kbase_fence_ops.c index c470374..d0d1672 100644 --- a/mali_kbase/mali_kbase_fence_ops.c +++ b/mali_kbase/mali_kbase_fence_ops.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -26,7 +27,7 @@ #include <mali_kbase.h> static const char * -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) kbase_fence_get_driver_name(struct fence *fence) #else kbase_fence_get_driver_name(struct dma_fence *fence) @@ -36,7 +37,7 @@ kbase_fence_get_driver_name(struct dma_fence *fence) } static const char * -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) kbase_fence_get_timeline_name(struct fence *fence) #else kbase_fence_get_timeline_name(struct dma_fence *fence) @@ -46,7 +47,7 @@ kbase_fence_get_timeline_name(struct dma_fence *fence) } static bool -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) kbase_fence_enable_signaling(struct fence *fence) #else kbase_fence_enable_signaling(struct dma_fence *fence) @@ -56,7 +57,7 @@ kbase_fence_enable_signaling(struct dma_fence *fence) } static void -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) kbase_fence_fence_value_str(struct fence *fence, char *str, int size) #else kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) @@ -69,7 +70,7 @@ kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) #endif } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) const struct fence_ops kbase_fence_ops = { .wait = fence_default_wait, #else diff --git a/mali_kbase/mali_kbase_gator.h b/mali_kbase/mali_kbase_gator.h index 6428f08..f1ee3fc 100644 --- a/mali_kbase/mali_kbase_gator.h +++ b/mali_kbase/mali_kbase_gator.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* NB taken from gator */ diff --git a/mali_kbase/mali_kbase_gpu_memory_debugfs.c b/mali_kbase/mali_kbase_gpu_memory_debugfs.c index 569abd9..84837f4 100644 --- a/mali_kbase/mali_kbase_gpu_memory_debugfs.c +++ b/mali_kbase/mali_kbase_gpu_memory_debugfs.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2017, 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -54,7 +55,8 @@ static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) mutex_lock(&kbdev->kctx_list_lock); list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { /* output the memory usage and cap for each kctx - * opened on this device */ + * opened on this device + */ seq_printf(sfile, " %s-0x%p %10u\n", "kctx", kctx, diff --git a/mali_kbase/mali_kbase_gpu_memory_debugfs.h b/mali_kbase/mali_kbase_gpu_memory_debugfs.h index a45dabb..9b1aaa0 100644 --- a/mali_kbase/mali_kbase_gpu_memory_debugfs.h +++ b/mali_kbase/mali_kbase_gpu_memory_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2014, 2016, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2014, 2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_gpu_memory_debugfs.h * Header file for gpu_memory entry in debugfs diff --git a/mali_kbase/mali_kbase_gpuprops.c b/mali_kbase/mali_kbase_gpuprops.c index 81e0395..5eae532 100644 --- a/mali_kbase/mali_kbase_gpuprops.c +++ b/mali_kbase/mali_kbase_gpuprops.c @@ -21,8 +21,6 @@ * */ - - /* * Base kernel property query APIs */ @@ -48,7 +46,7 @@ static void kbase_gpuprops_construct_coherent_groups( u64 first_set, first_set_prev; u32 num_groups = 0; - KBASE_DEBUG_ASSERT(NULL != props); + KBASE_DEBUG_ASSERT(props != NULL); props->coherency_info.coherency = props->raw_props.mem_features; props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); @@ -124,8 +122,8 @@ static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, int i; int err; - KBASE_DEBUG_ASSERT(NULL != kbdev); - KBASE_DEBUG_ASSERT(NULL != gpu_props); + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(gpu_props != NULL); /* Dump relevant registers */ err = kbase_backend_gpuprops_get(kbdev, ®dump); @@ -166,6 +164,10 @@ static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, gpu_props->raw_props.thread_features = regdump.thread_features; gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc; + gpu_props->raw_props.gpu_features = + ((u64) regdump.gpu_features_hi << 32) + + regdump.gpu_features_lo; + return 0; } @@ -217,7 +219,8 @@ static void kbase_gpuprops_calculate_props( /* Field with number of l2 slices is added to MEM_FEATURES register * since t76x. Below code assumes that for older GPU reserved bits will - * be read as zero. */ + * be read as zero. + */ gpu_props->l2_props.num_l2_slices = KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; @@ -305,7 +308,7 @@ void kbase_gpuprops_set(struct kbase_device *kbdev) struct kbase_gpu_props *gpu_props; struct gpu_raw_gpu_props *raw; - KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(kbdev != NULL); gpu_props = &kbdev->gpu_props; raw = &gpu_props->props.raw_props; @@ -365,13 +368,26 @@ int kbase_gpuprops_set_features(struct kbase_device *kbdev) * in sysfs. */ static u8 override_l2_size; -module_param(override_l2_size, byte, 0); +module_param(override_l2_size, byte, 0000); MODULE_PARM_DESC(override_l2_size, "Override L2 size config for testing"); static u8 override_l2_hash; -module_param(override_l2_hash, byte, 0); +module_param(override_l2_hash, byte, 0000); MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); +static u32 l2_hash_values[ASN_HASH_COUNT] = { + 0, +}; +static int num_override_l2_hash_values; +module_param_array(l2_hash_values, uint, &num_override_l2_hash_values, 0000); +MODULE_PARM_DESC(l2_hash_values, "Override L2 hash values config for testing"); + +enum l2_config_override_result { + L2_CONFIG_OVERRIDE_FAIL = -1, + L2_CONFIG_OVERRIDE_NONE, + L2_CONFIG_OVERRIDE_OK, +}; + /** * kbase_read_l2_config_from_dt - Read L2 configuration * @kbdev: The kbase device for which to get the L2 configuration. @@ -380,15 +396,17 @@ MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); * Override values in module parameters take priority over override values in * device tree. * - * Return: true if either size or hash was overridden, false if no overrides - * were found. + * Return: L2_CONFIG_OVERRIDE_OK if either size or hash, or both was properly + * overridden, L2_CONFIG_OVERRIDE_NONE if no overrides are provided. + * L2_CONFIG_OVERRIDE_FAIL otherwise. */ -static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) +static enum l2_config_override_result +kbase_read_l2_config_from_dt(struct kbase_device *const kbdev) { struct device_node *np = kbdev->dev->of_node; if (!np) - return false; + return L2_CONFIG_OVERRIDE_NONE; if (override_l2_size) kbdev->l2_size_override = override_l2_size; @@ -400,10 +418,41 @@ static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) else if (of_property_read_u8(np, "l2-hash", &kbdev->l2_hash_override)) kbdev->l2_hash_override = 0; - if (kbdev->l2_size_override || kbdev->l2_hash_override) - return true; + kbdev->l2_hash_values_override = false; + if (num_override_l2_hash_values) { + int i; + + kbdev->l2_hash_values_override = true; + for (i = 0; i < num_override_l2_hash_values; i++) + kbdev->l2_hash_values[i] = l2_hash_values[i]; + } else if (!of_property_read_u32_array(np, "l2-hash-values", + kbdev->l2_hash_values, + ASN_HASH_COUNT)) + kbdev->l2_hash_values_override = true; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH) && + (kbdev->l2_hash_override)) { + dev_err(kbdev->dev, "l2-hash not supported\n"); + return L2_CONFIG_OVERRIDE_FAIL; + } + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH) && + (kbdev->l2_hash_values_override)) { + dev_err(kbdev->dev, "l2-hash-values not supported\n"); + return L2_CONFIG_OVERRIDE_FAIL; + } - return false; + if (kbdev->l2_hash_override && kbdev->l2_hash_values_override) { + dev_err(kbdev->dev, + "both l2-hash & l2-hash-values not supported\n"); + return L2_CONFIG_OVERRIDE_FAIL; + } + + if (kbdev->l2_size_override || kbdev->l2_hash_override || + kbdev->l2_hash_values_override) + return L2_CONFIG_OVERRIDE_OK; + + return L2_CONFIG_OVERRIDE_NONE; } int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) @@ -415,8 +464,15 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; /* Check for L2 cache size & hash overrides */ - if (!kbase_read_l2_config_from_dt(kbdev)) - return 0; + switch (kbase_read_l2_config_from_dt(kbdev)) { + case L2_CONFIG_OVERRIDE_FAIL: + err = -EIO; + goto exit; + case L2_CONFIG_OVERRIDE_NONE: + goto exit; + default: + break; + } /* Need L2 to get powered to reflect to L2_FEATURES */ kbase_pm_context_active(kbdev); @@ -440,8 +496,9 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) idle_gpu: /* Let GPU idle */ kbase_pm_context_idle(kbdev); - } +} +exit: return err; } @@ -521,7 +578,7 @@ static struct { PROP(RAW_THREAD_FEATURES, raw_props.thread_features), PROP(RAW_THREAD_TLS_ALLOC, raw_props.thread_tls_alloc), PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), - + PROP(RAW_GPU_FEATURES, raw_props.gpu_features), PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), PROP(COHERENCY_COHERENCY, coherency_info.coherency), diff --git a/mali_kbase/mali_kbase_gpuprops.h b/mali_kbase/mali_kbase_gpuprops.h index 5eee794..e634922 100644 --- a/mali_kbase/mali_kbase_gpuprops.h +++ b/mali_kbase/mali_kbase_gpuprops.h @@ -26,16 +26,19 @@ * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ - - /** * @file mali_kbase_gpuprops.h * Base kernel property query APIs diff --git a/mali_kbase/mali_kbase_gpuprops_types.h b/mali_kbase/mali_kbase_gpuprops_types.h index ec6f1c3..d96243d 100644 --- a/mali_kbase/mali_kbase_gpuprops_types.h +++ b/mali_kbase/mali_kbase_gpuprops_types.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_gpuprops_types.h * Base kernel property query APIs @@ -60,6 +77,8 @@ struct kbase_gpuprops_regdump { u32 stack_present_lo; u32 stack_present_hi; u32 coherency_features; + u32 gpu_features_lo; + u32 gpu_features_hi; }; struct kbase_gpu_cache_props { diff --git a/mali_kbase/mali_kbase_gwt.c b/mali_kbase/mali_kbase_gwt.c index 91dc4db..42ddf9b 100644 --- a/mali_kbase/mali_kbase_gwt.c +++ b/mali_kbase/mali_kbase_gwt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_gwt.h b/mali_kbase/mali_kbase_gwt.h index 7e7746e..4510497 100644 --- a/mali_kbase/mali_kbase_gwt.h +++ b/mali_kbase/mali_kbase_gwt.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #if !defined(_KBASE_GWT_H) diff --git a/mali_kbase/mali_kbase_hw.c b/mali_kbase/mali_kbase_hw.c index 386fb9e..512213c 100644 --- a/mali_kbase/mali_kbase_hw.c +++ b/mali_kbase/mali_kbase_hw.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /* * Run-time work-arounds helpers */ @@ -237,7 +236,7 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( {U32_MAX, NULL} } }, {GPU_ID2_PRODUCT_LTUX, - {{GPU_ID2_VERSION_MAKE(3, 0, 0), base_hw_issues_tTUx_r0p0}, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTUx_r0p0}, {U32_MAX, NULL} } }, }; @@ -272,8 +271,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( } /* Check whether this is a candidate for most recent - known version not later than the actual - version. */ + * known version not later than the actual version. + */ if ((version > product->map[v].version) && (product->map[v].version >= fallback_version)) { #if MALI_CUSTOMER_RELEASE @@ -290,7 +289,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( if ((issues == NULL) && (fallback_issues != NULL)) { /* Fall back to the issue set of the most recent known - version not later than the actual version. */ + * version not later than the actual version. + */ issues = fallback_issues; #if MALI_CUSTOMER_RELEASE @@ -343,7 +343,8 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) #if !MALI_CUSTOMER_RELEASE /* The GPU ID might have been replaced with the last - known version of the same GPU. */ + * known version of the same GPU. + */ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; #endif } else { diff --git a/mali_kbase/mali_kbase_hw.h b/mali_kbase/mali_kbase_hw.h index f386b16..afc56a6 100644 --- a/mali_kbase/mali_kbase_hw.h +++ b/mali_kbase/mali_kbase_hw.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file * Run-time work-arounds helpers diff --git a/mali_kbase/mali_kbase_hwaccess_backend.h b/mali_kbase/mali_kbase_hwaccess_backend.h index 89df251..ce57991 100644 --- a/mali_kbase/mali_kbase_hwaccess_backend.h +++ b/mali_kbase/mali_kbase_hwaccess_backend.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /* * HW access backend common APIs */ diff --git a/mali_kbase/mali_kbase_hwaccess_defs.h b/mali_kbase/mali_kbase_hwaccess_defs.h index 3c3dfb0..e21f52d 100644 --- a/mali_kbase/mali_kbase_hwaccess_defs.h +++ b/mali_kbase/mali_kbase_hwaccess_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2016-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2016-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_hwaccess_gpuprops.h b/mali_kbase/mali_kbase_hwaccess_gpuprops.h index 3ae0dbe..edfa02e 100644 --- a/mali_kbase/mali_kbase_hwaccess_gpuprops.h +++ b/mali_kbase/mali_kbase_hwaccess_gpuprops.h @@ -26,15 +26,19 @@ * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ - /** * Base kernel property query backend APIs */ diff --git a/mali_kbase/mali_kbase_hwaccess_instr.h b/mali_kbase/mali_kbase_hwaccess_instr.h index fd17e55..25a4695 100644 --- a/mali_kbase/mali_kbase_hwaccess_instr.h +++ b/mali_kbase/mali_kbase_hwaccess_instr.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2017-2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2017-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /* * HW Access instrumentation common APIs */ diff --git a/mali_kbase/mali_kbase_hwaccess_jm.h b/mali_kbase/mali_kbase_hwaccess_jm.h index f6ce17e..82236fe 100644 --- a/mali_kbase/mali_kbase_hwaccess_jm.h +++ b/mali_kbase/mali_kbase_hwaccess_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /* * HW access job manager common APIs */ diff --git a/mali_kbase/mali_kbase_hwaccess_pm.h b/mali_kbase/mali_kbase_hwaccess_pm.h index 3e223c6..eb6229e 100644 --- a/mali_kbase/mali_kbase_hwaccess_pm.h +++ b/mali_kbase/mali_kbase_hwaccess_pm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_hwaccess_time.h b/mali_kbase/mali_kbase_hwaccess_time.h index 94b7551..d982f2a 100644 --- a/mali_kbase/mali_kbase_hwaccess_time.h +++ b/mali_kbase/mali_kbase_hwaccess_time.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,9 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - /** * */ diff --git a/mali_kbase/mali_kbase_hwcnt.c b/mali_kbase/mali_kbase_hwcnt.c index 2708af7..59bbc9d 100644 --- a/mali_kbase/mali_kbase_hwcnt.c +++ b/mali_kbase/mali_kbase_hwcnt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. @@ -28,9 +29,6 @@ #include "mali_kbase_hwcnt_accumulator.h" #include "mali_kbase_hwcnt_backend.h" #include "mali_kbase_hwcnt_types.h" -#include "mali_malisw.h" -#include "mali_kbase_debug.h" -#include "mali_kbase_linux.h" #include <linux/mutex.h> #include <linux/spinlock.h> @@ -51,6 +49,7 @@ enum kbase_hwcnt_accum_state { /** * struct kbase_hwcnt_accumulator - Hardware counter accumulator structure. + * @metadata: Pointer to immutable hwcnt metadata. * @backend: Pointer to created counter backend. * @state: The current state of the accumulator. * - State transition from disabled->enabled or @@ -89,6 +88,7 @@ enum kbase_hwcnt_accum_state { * accum_lock. */ struct kbase_hwcnt_accumulator { + const struct kbase_hwcnt_metadata *metadata; struct kbase_hwcnt_backend *backend; enum kbase_hwcnt_accum_state state; struct kbase_hwcnt_enable_map enable_map; @@ -117,6 +117,10 @@ struct kbase_hwcnt_accumulator { * state_lock. * - Can be read while holding either lock. * @accum: Hardware counter accumulator structure. + * @wq: Centralized workqueue for users of hardware counters to + * submit async hardware counter related work. Never directly + * called, but it's expected that a lot of the functions in this + * API will end up called from the enqueued async work. */ struct kbase_hwcnt_context { const struct kbase_hwcnt_backend_interface *iface; @@ -125,6 +129,7 @@ struct kbase_hwcnt_context { struct mutex accum_lock; bool accum_inited; struct kbase_hwcnt_accumulator accum; + struct workqueue_struct *wq; }; int kbase_hwcnt_context_init( @@ -138,7 +143,7 @@ int kbase_hwcnt_context_init( hctx = kzalloc(sizeof(*hctx), GFP_KERNEL); if (!hctx) - return -ENOMEM; + goto err_alloc_hctx; hctx->iface = iface; spin_lock_init(&hctx->state_lock); @@ -146,11 +151,21 @@ int kbase_hwcnt_context_init( mutex_init(&hctx->accum_lock); hctx->accum_inited = false; + hctx->wq = + alloc_workqueue("mali_kbase_hwcnt", WQ_HIGHPRI | WQ_UNBOUND, 0); + if (!hctx->wq) + goto err_alloc_workqueue; + *out_hctx = hctx; return 0; + + destroy_workqueue(hctx->wq); +err_alloc_workqueue: + kfree(hctx); +err_alloc_hctx: + return -ENOMEM; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_init); void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) { @@ -159,9 +174,13 @@ void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) /* Make sure we didn't leak the accumulator */ WARN_ON(hctx->accum_inited); + + /* We don't expect any work to be pending on this workqueue. + * Regardless, this will safely drain and complete the work. + */ + destroy_workqueue(hctx->wq); kfree(hctx); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_term); /** * kbasep_hwcnt_accumulator_term() - Terminate the accumulator for the context. @@ -197,22 +216,23 @@ static int kbasep_hwcnt_accumulator_init(struct kbase_hwcnt_context *hctx) if (errcode) goto error; + hctx->accum.metadata = hctx->iface->metadata(hctx->iface->info); hctx->accum.state = ACCUM_STATE_ERROR; - errcode = kbase_hwcnt_enable_map_alloc( - hctx->iface->metadata, &hctx->accum.enable_map); + errcode = kbase_hwcnt_enable_map_alloc(hctx->accum.metadata, + &hctx->accum.enable_map); if (errcode) goto error; hctx->accum.enable_map_any_enabled = false; - errcode = kbase_hwcnt_dump_buffer_alloc( - hctx->iface->metadata, &hctx->accum.accum_buf); + errcode = kbase_hwcnt_dump_buffer_alloc(hctx->accum.metadata, + &hctx->accum.accum_buf); if (errcode) goto error; - errcode = kbase_hwcnt_enable_map_alloc( - hctx->iface->metadata, &hctx->accum.scratch_map); + errcode = kbase_hwcnt_enable_map_alloc(hctx->accum.metadata, + &hctx->accum.scratch_map); if (errcode) goto error; @@ -366,8 +386,8 @@ static int kbasep_hwcnt_accumulator_dump( WARN_ON(!hctx); WARN_ON(!ts_start_ns); WARN_ON(!ts_end_ns); - WARN_ON(dump_buf && (dump_buf->metadata != hctx->iface->metadata)); - WARN_ON(new_map && (new_map->metadata != hctx->iface->metadata)); + WARN_ON(dump_buf && (dump_buf->metadata != hctx->accum.metadata)); + WARN_ON(new_map && (new_map->metadata != hctx->accum.metadata)); WARN_ON(!hctx->accum_inited); lockdep_assert_held(&hctx->accum_lock); @@ -609,7 +629,6 @@ int kbase_hwcnt_accumulator_acquire( return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_acquire); void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) { @@ -644,7 +663,6 @@ void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) spin_unlock_irqrestore(&hctx->state_lock, flags); mutex_unlock(&hctx->accum_lock); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_release); void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) { @@ -663,7 +681,6 @@ void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) mutex_unlock(&hctx->accum_lock); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable); bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) { @@ -692,7 +709,6 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) return atomic_disabled; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable_atomic); void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) { @@ -712,7 +728,6 @@ void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) spin_unlock_irqrestore(&hctx->state_lock, flags); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_enable); const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( struct kbase_hwcnt_context *hctx) @@ -720,9 +735,17 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( if (!hctx) return NULL; - return hctx->iface->metadata; + return hctx->iface->metadata(hctx->iface->info); +} + +bool kbase_hwcnt_context_queue_work(struct kbase_hwcnt_context *hctx, + struct work_struct *work) +{ + if (WARN_ON(!hctx) || WARN_ON(!work)) + return false; + + return queue_work(hctx->wq, work); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_context_metadata); int kbase_hwcnt_accumulator_set_counters( struct kbase_hwcnt_accumulator *accum, @@ -739,8 +762,8 @@ int kbase_hwcnt_accumulator_set_counters( hctx = container_of(accum, struct kbase_hwcnt_context, accum); - if ((new_map->metadata != hctx->iface->metadata) || - (dump_buf && (dump_buf->metadata != hctx->iface->metadata))) + if ((new_map->metadata != hctx->accum.metadata) || + (dump_buf && (dump_buf->metadata != hctx->accum.metadata))) return -EINVAL; mutex_lock(&hctx->accum_lock); @@ -752,7 +775,6 @@ int kbase_hwcnt_accumulator_set_counters( return errcode; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_set_counters); int kbase_hwcnt_accumulator_dump( struct kbase_hwcnt_accumulator *accum, @@ -768,7 +790,7 @@ int kbase_hwcnt_accumulator_dump( hctx = container_of(accum, struct kbase_hwcnt_context, accum); - if (dump_buf && (dump_buf->metadata != hctx->iface->metadata)) + if (dump_buf && (dump_buf->metadata != hctx->accum.metadata)) return -EINVAL; mutex_lock(&hctx->accum_lock); @@ -780,7 +802,6 @@ int kbase_hwcnt_accumulator_dump( return errcode; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_dump); u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum) { diff --git a/mali_kbase/mali_kbase_hwcnt_accumulator.h b/mali_kbase/mali_kbase_hwcnt_accumulator.h index eb82ea4..c3c06b3 100644 --- a/mali_kbase/mali_kbase_hwcnt_accumulator.h +++ b/mali_kbase/mali_kbase_hwcnt_accumulator.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_hwcnt_backend.h b/mali_kbase/mali_kbase_hwcnt_backend.h index 3a921b7..5f3ab08 100644 --- a/mali_kbase/mali_kbase_hwcnt_backend.h +++ b/mali_kbase/mali_kbase_hwcnt_backend.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -41,12 +60,26 @@ struct kbase_hwcnt_dump_buffer; struct kbase_hwcnt_backend_info; /* - * struct kbase_hwcnt_backend_info - Opaque pointer to a hardware counter - * backend, used to perform dumps. + * struct kbase_hwcnt_backend - Opaque pointer to a hardware counter + * backend, used to perform dumps. */ struct kbase_hwcnt_backend; /** + * typedef kbase_hwcnt_backend_metadata_fn - Get the immutable hardware counter + * metadata that describes the layout + * of the counter data structures. + * @info: Non-NULL pointer to backend info. + * + * Multiple calls to this function with the same info are guaranteed to return + * the same metadata object each time. + * + * Return: Non-NULL pointer to immutable hardware counter metadata. + */ +typedef const struct kbase_hwcnt_metadata *(*kbase_hwcnt_backend_metadata_fn)( + const struct kbase_hwcnt_backend_info *info); + +/** * typedef kbase_hwcnt_backend_init_fn - Initialise a counter backend. * @info: Non-NULL pointer to backend info. * @out_backend: Non-NULL pointer to where backend is stored on success. @@ -171,9 +204,9 @@ typedef int (*kbase_hwcnt_backend_dump_wait_fn)( * @accumulate: True if counters should be accumulated into dump_buffer, rather * than copied. * - * If the backend is not enabled, returns an error. - * If a dump is in progress (i.e. dump_wait has not yet returned successfully) - * then the resultant contents of the dump buffer will be undefined. + * The resultant contents of the dump buffer are only well defined if a prior + * call to dump_wait returned successfully, and a new dump has not yet been + * requested by a call to dump_request. * * Return: 0 on success, else error code. */ @@ -186,9 +219,10 @@ typedef int (*kbase_hwcnt_backend_dump_get_fn)( /** * struct kbase_hwcnt_backend_interface - Hardware counter backend virtual * interface. - * @metadata: Immutable hardware counter metadata. * @info: Immutable info used to initialise an instance of the * backend. + * @metadata: Function ptr to get the immutable hardware counter + * metadata. * @init: Function ptr to initialise an instance of the backend. * @term: Function ptr to terminate an instance of the backend. * @timestamp_ns: Function ptr to get the current backend timestamp. @@ -203,8 +237,8 @@ typedef int (*kbase_hwcnt_backend_dump_get_fn)( * buffer. */ struct kbase_hwcnt_backend_interface { - const struct kbase_hwcnt_metadata *metadata; const struct kbase_hwcnt_backend_info *info; + kbase_hwcnt_backend_metadata_fn metadata; kbase_hwcnt_backend_init_fn init; kbase_hwcnt_backend_term_fn term; kbase_hwcnt_backend_timestamp_ns_fn timestamp_ns; diff --git a/mali_kbase/mali_kbase_hwcnt_backend_csf.c b/mali_kbase/mali_kbase_hwcnt_backend_csf.c new file mode 100644 index 0000000..8560fd7 --- /dev/null +++ b/mali_kbase/mali_kbase_hwcnt_backend_csf.c @@ -0,0 +1,1882 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_backend_csf.h" +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_hwcnt_types.h" + +#include <linux/log2.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <linux/completion.h> + +#ifndef BASE_MAX_NR_CLOCKS_REGULATORS +#define BASE_MAX_NR_CLOCKS_REGULATORS 2 +#endif + +/** + * enum kbase_hwcnt_backend_csf_dump_state - HWC CSF backend dumping states. + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE: Initial state, or the state if there is + * an error. + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED: A dump has been requested and we are + * waiting for an ACK, this ACK could come from either PRFCNT_ACK, + * PROTMODE_ENTER_ACK, or if an error occurs. + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT: Checking the insert + * immediately after receiving the ACK, so we know which index corresponds to + * the buffer we requested. + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED: The insert has been saved and + * now we have kicked off the worker. + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING: The insert has been saved and now + * we have kicked off the worker to accumulate up to that insert and then copy + * the delta to the user buffer to prepare for dump_get(). + * + * @KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED: The dump completed successfully. + * + * Valid state transitions: + * IDLE -> REQUESTED (on dump request) + * REQUESTED -> QUERYING_INSERT (on dump ack) + * QUERYING_INSERT -> WORKER_LAUNCHED (on worker submission) + * WORKER_LAUNCHED -> ACCUMULATING (while the worker is accumulating) + * ACCUMULATING -> COMPLETED (on accumulation completion) + * COMPLETED -> REQUESTED (on dump request) + * COMPLETED -> IDLE (on disable) + * ANY -> IDLE (on error) + */ +enum kbase_hwcnt_backend_csf_dump_state { + KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE, + KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED, + KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT, + KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED, + KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING, + KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED, +}; + +/** + * enum kbase_hwcnt_backend_csf_enable_state - HWC CSF backend enable states. + * + * @KBASE_HWCNT_BACKEND_CSF_DISABLED: Initial state, and the state when backend + * is disabled. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: Enable request is in + * progress, waiting for firmware acknowledgment. + * + * @KBASE_HWCNT_BACKEND_CSF_ENABLED: Enable request has been acknowledged, + * enable is done. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: Disable request is in + * progress, waiting for firmware acknowledgment. + * + * @KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: Disable request has been + * acknowledged, waiting for dump workers to be finished. + * + * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: An + * unrecoverable error happened, waiting for dump workers to be finished. + * + * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: An unrecoverable error + * happened, and dump workers have finished, waiting for reset. + * + * Valid state transitions: + * DISABLED -> TRANSITIONING_TO_ENABLED (on enable) + * TRANSITIONING_TO_ENABLED -> ENABLED (on enable ack) + * ENABLED -> TRANSITIONING_TO_DISABLED (on disable) + * TRANSITIONING_TO_DISABLED -> DISABLED_WAIT_FOR_WORKER (on disable ack) + * DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) + * DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) + * ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable + * error) + * UNRECOVERABLE_ERROR -> DISABLED (on before reset) + */ +enum kbase_hwcnt_backend_csf_enable_state { + KBASE_HWCNT_BACKEND_CSF_DISABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED, + KBASE_HWCNT_BACKEND_CSF_ENABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED, + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER, + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER, + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR, +}; + +/** + * struct kbase_hwcnt_backend_csf_info - Information used to create an instance + * of a CSF hardware counter backend. + * @backend: Pointer to access CSF backend. + * @lock: Spinlock protecting backend and its internal + * states. + * @fw_in_protected_mode: True if FW is running in protected mode, else + * false. + * @unrecoverable_error_happened: True if an recoverable error happened, else + * false. + * @csf_if: CSF interface object pointer. Functions inside + * this interface MUST never be called while + * holding the spin lock, as that could cause + * deadlocks. + * @ring_buf_cnt: Dump buffer count in the ring buffer. + * @counter_set: The performance counter set to use. + * @metadata: Hardware counter metadata. + * @dump_bytes: Bytes of GPU memory required to perform a + * hardware counter dump. + * @gpu_info: GPU information to initialise HWC dump memory + * layout. + */ +struct kbase_hwcnt_backend_csf_info { + struct kbase_hwcnt_backend_csf *backend; + spinlock_t lock; + bool fw_in_protected_mode; + bool unrecoverable_error_happened; + struct kbase_hwcnt_backend_csf_if *csf_if; + u32 ring_buf_cnt; + enum kbase_hwcnt_set counter_set; + const struct kbase_hwcnt_metadata *metadata; + size_t dump_bytes; + struct kbase_hwcnt_gpu_info gpu_info; +}; + +/** + * struct kbase_hwcnt_csf_physical_layout - HWC sample memory physical layout + * information. + * @fe_cnt: FroneEnd block count. + * @tiler_cnt: Tiler block count. + * @mmu_l2_cnt: Memory system(MMU and L2 cache) block count. + * @shader_cnt: Shader Core block count. + * @block_cnt: Total block count (sum of all other block counts). + * @shader_avail_mask: Bitmap of all shader cores in the system. + * @offset_enable_mask: Offset of enable mask in the block. + * @headers_per_block: Header size per block. + * @counters_per_block: Counters size per block. + * @values_per_block: Total size per block. + */ +struct kbase_hwcnt_csf_physical_layout { + size_t fe_cnt; + size_t tiler_cnt; + size_t mmu_l2_cnt; + size_t shader_cnt; + size_t block_cnt; + u64 shader_avail_mask; + size_t offset_enable_mask; + size_t headers_per_block; + size_t counters_per_block; + size_t values_per_block; +}; + +/** + * struct kbase_hwcnt_backend_csf - Instance of a CSF hardware counter backend. + * @info: CSF Info used to create the backend. + * @dump_state: The dumping state of the backend. + * @enable_state: The CSF backend internal enabled state. + * @insert_index_to_accumulate: The insert index in the ring buffer which need + * to accumulate up to. + * @enable_state_waitq: Wait queue object used to notify the enable + * changing flag is done. + * @to_user_buf: HWC sample buffer for client user. + * @accum_buf: HWC sample buffer used as an internal + * accumulator. + * @old_sample_buf: HWC sample buffer to save the previous values + * for delta calculation. + * @ring_buf: Opaque pointer for ring buffer object. + * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. + * @clk_enable_map: The enable map specifying enabled clock domains. + * @cycle_count_elapsed: Cycle count elapsed for a given sample period. + * @prev_cycle_count: Previous cycle count to calculate the cycle + * count for sample period. + * @phys_layout: Physical memory layout information of HWC + * sample buffer. + * @dump_completed: Completion signalled by the dump worker when + * it is completed accumulating up to the + * insert_index_to_accumulate. + * Should be initialized to the "complete" state. + * @hwc_dump_workq: Single threaded work queue for HWC workers + * execution. + * @hwc_dump_work: Worker to accumulate samples. + * @hwc_threshold_work: Worker for consuming available samples when + * threshold interrupt raised. + */ +struct kbase_hwcnt_backend_csf { + struct kbase_hwcnt_backend_csf_info *info; + enum kbase_hwcnt_backend_csf_dump_state dump_state; + enum kbase_hwcnt_backend_csf_enable_state enable_state; + u32 insert_index_to_accumulate; + wait_queue_head_t enable_state_waitq; + u32 *to_user_buf; + u32 *accum_buf; + u32 *old_sample_buf; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; + void *ring_buf_cpu_base; + u64 clk_enable_map; + u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; + u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; + struct kbase_hwcnt_csf_physical_layout phys_layout; + struct completion dump_completed; + struct workqueue_struct *hwc_dump_workq; + struct work_struct hwc_dump_work; + struct work_struct hwc_threshold_work; +}; + +bool kbasep_hwcnt_backend_csf_backend_exists( + struct kbase_hwcnt_backend_csf_info *csf_info) +{ + WARN_ON(!csf_info); + lockdep_assert_held(&csf_info->lock); + return (csf_info->backend != NULL); +} + +/** + * kbasep_hwcnt_backend_csf_cc_initial_sample() - Initialize cycle count + * tracking. + * + * @backend_csf: Non-NULL pointer to backend. + * @enable_map: Non-NULL pointer to enable map specifying enabled counters. + */ +static void kbasep_hwcnt_backend_csf_cc_initial_sample( + struct kbase_hwcnt_backend_csf *backend_csf, + const struct kbase_hwcnt_enable_map *enable_map) +{ + u64 clk_enable_map = enable_map->clk_enable_map; + u64 cycle_counts[BASE_MAX_NR_CLOCKS_REGULATORS]; + size_t clk; + + /* Read cycle count from CSF interface for both clock domains. */ + backend_csf->info->csf_if->get_gpu_cycle_count( + backend_csf->info->csf_if->ctx, cycle_counts, clk_enable_map); + + kbase_hwcnt_metadata_for_each_clock(enable_map->metadata, clk) { + if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, clk)) + backend_csf->prev_cycle_count[clk] = cycle_counts[clk]; + } + + /* Keep clk_enable_map for dump_request. */ + backend_csf->clk_enable_map = clk_enable_map; +} + +static void +kbasep_hwcnt_backend_csf_cc_update(struct kbase_hwcnt_backend_csf *backend_csf) +{ + u64 cycle_counts[BASE_MAX_NR_CLOCKS_REGULATORS]; + size_t clk; + + backend_csf->info->csf_if->get_gpu_cycle_count( + backend_csf->info->csf_if->ctx, cycle_counts, + backend_csf->clk_enable_map); + + kbase_hwcnt_metadata_for_each_clock(backend_csf->info->metadata, clk) { + if (kbase_hwcnt_clk_enable_map_enabled( + backend_csf->clk_enable_map, clk)) { + backend_csf->cycle_count_elapsed[clk] = + cycle_counts[clk] - + backend_csf->prev_cycle_count[clk]; + backend_csf->prev_cycle_count[clk] = cycle_counts[clk]; + } + } +} + +/* CSF backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ +static u64 +kbasep_hwcnt_backend_csf_timestamp_ns(struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + + if (!backend_csf || !backend_csf->info || !backend_csf->info->csf_if) + return 0; + + return backend_csf->info->csf_if->timestamp_ns( + backend_csf->info->csf_if->ctx); +} + +/** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to + * guarantee the header is + * enabled, the header will be + * used when do the samples + * delta calculation. + *@phys_enable_map: HWC physical enable map to be processed. + */ +static void kbasep_hwcnt_backend_csf_process_enable_map( + struct kbase_hwcnt_physical_enable_map *phys_enable_map) +{ + WARN_ON(!phys_enable_map); + + /* Enable header if any counter is required from user, the header is + * controlled by bit 0 of the enable mask. + */ + if (phys_enable_map->fe_bm) + phys_enable_map->fe_bm |= 1; + + if (phys_enable_map->tiler_bm) + phys_enable_map->tiler_bm |= 1; + + if (phys_enable_map->mmu_l2_bm) + phys_enable_map->mmu_l2_bm |= 1; + + if (phys_enable_map->shader_bm) + phys_enable_map->shader_bm |= 1; +} + +static void kbasep_hwcnt_backend_csf_init_layout( + const struct kbase_hwcnt_gpu_info *gpu_info, + struct kbase_hwcnt_csf_physical_layout *phys_layout) +{ + WARN_ON(!gpu_info); + WARN_ON(!phys_layout); + + phys_layout->fe_cnt = 1; + phys_layout->tiler_cnt = 1; + phys_layout->mmu_l2_cnt = gpu_info->l2_count; + phys_layout->shader_cnt = fls64(gpu_info->core_mask); + phys_layout->block_cnt = phys_layout->fe_cnt + phys_layout->tiler_cnt + + phys_layout->mmu_l2_cnt + + phys_layout->shader_cnt; + + phys_layout->shader_avail_mask = gpu_info->core_mask; + + phys_layout->headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; + phys_layout->counters_per_block = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; + phys_layout->values_per_block = KBASE_HWCNT_V5_VALUES_PER_BLOCK; + phys_layout->offset_enable_mask = KBASE_HWCNT_V5_PRFCNT_EN_HEADER; +} + +static void kbasep_hwcnt_backend_csf_reset_internal_buffers( + struct kbase_hwcnt_backend_csf *backend_csf) +{ + memset(backend_csf->to_user_buf, 0, backend_csf->info->dump_bytes); + memset(backend_csf->accum_buf, 0, backend_csf->info->dump_bytes); + memset(backend_csf->old_sample_buf, 0, backend_csf->info->dump_bytes); +} + +static void kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( + struct kbase_hwcnt_backend_csf *backend_csf, u32 *sample) +{ + u32 block_idx; + const struct kbase_hwcnt_csf_physical_layout *phys_layout; + u32 *block_buf; + + phys_layout = &backend_csf->phys_layout; + + for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + block_buf = sample + block_idx * phys_layout->values_per_block; + block_buf[phys_layout->offset_enable_mask] = 0; + } +} + +static void kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header( + struct kbase_hwcnt_backend_csf *backend_csf) +{ + u32 idx; + u32 *sample; + char *cpu_dump_base; + + cpu_dump_base = (char *)backend_csf->ring_buf_cpu_base; + + for (idx = 0; idx < backend_csf->info->ring_buf_cnt; idx++) { + sample = (u32 *)&cpu_dump_base[idx * + backend_csf->info->dump_bytes]; + kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( + backend_csf, sample); + } +} + +static void kbasep_hwcnt_backend_csf_update_user_sample( + struct kbase_hwcnt_backend_csf *backend_csf) +{ + /* Copy the data into the sample and wait for the user to get it. */ + memcpy(backend_csf->to_user_buf, backend_csf->accum_buf, + backend_csf->info->dump_bytes); + + /* After copied data into user sample, clear the accumulator values to + * prepare for the next accumulator, such as the next request or + * threshold. + */ + memset(backend_csf->accum_buf, 0, backend_csf->info->dump_bytes); +} + +static void kbasep_hwcnt_backend_csf_accumulate_sample( + const struct kbase_hwcnt_csf_physical_layout *phys_layout, + size_t dump_bytes, u32 *accum_buf, const u32 *old_sample_buf, + const u32 *new_sample_buf) +{ + size_t block_idx, ctr_idx; + const u32 *old_block = old_sample_buf; + const u32 *new_block = new_sample_buf; + u32 *acc_block = accum_buf; + + for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + const u32 new_enable_mask = + new_block[phys_layout->offset_enable_mask]; + + if (new_enable_mask == 0) { + /* Hardware block was unavailable or we didn't turn on + * any counters. Do nothing. + */ + } else { + /* Hardware block was available and it had some counters + * enabled. We need to update the accumulation buffer. + */ + + /* Unconditionally copy the headers. */ + memcpy(acc_block, new_block, + phys_layout->headers_per_block * + KBASE_HWCNT_VALUE_BYTES); + + /* Accumulate the counters. */ + for (ctr_idx = phys_layout->headers_per_block; + ctr_idx < phys_layout->values_per_block; + ctr_idx++) { + acc_block[ctr_idx] += new_block[ctr_idx]; + } + } + old_block += phys_layout->values_per_block; + new_block += phys_layout->values_per_block; + acc_block += phys_layout->values_per_block; + } + + WARN_ON(old_block != + old_sample_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); + WARN_ON(new_block != + new_sample_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); + WARN_ON(acc_block != accum_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); + (void)dump_bytes; +} + +static void kbasep_hwcnt_backend_csf_accumulate_samples( + struct kbase_hwcnt_backend_csf *backend_csf, u32 extract_index_to_start, + u32 insert_index_to_stop) +{ + u32 raw_idx; + u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base; + const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt; + const size_t buf_dump_bytes = backend_csf->info->dump_bytes; + u32 *old_sample_buf = backend_csf->old_sample_buf; + u32 *new_sample_buf; + + if (extract_index_to_start == insert_index_to_stop) + /* No samples to accumulate. Early out. */ + return; + + /* Sync all the buffers to CPU side before read the data. */ + backend_csf->info->csf_if->ring_buf_sync( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf, + extract_index_to_start, (insert_index_to_stop - 1), true); + + /* Consider u32 wrap case, '!=' is used here instead of '<' operator */ + for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; + raw_idx++) { + /* The logical "&" acts as a modulo operation since buf_count + * must be a power of two. + */ + const u32 buf_idx = raw_idx & (ring_buf_cnt - 1); + + new_sample_buf = + (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes]; + + kbasep_hwcnt_backend_csf_accumulate_sample( + &backend_csf->phys_layout, buf_dump_bytes, + backend_csf->accum_buf, old_sample_buf, new_sample_buf); + + old_sample_buf = new_sample_buf; + } + + /* Save the newest buffer as the old buffer for next time. */ + memcpy(backend_csf->old_sample_buf, new_sample_buf, buf_dump_bytes); + + /* Reset the prfcnt_en header on each sample before releasing them. */ + for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; + raw_idx++) { + const u32 buf_idx = raw_idx & (ring_buf_cnt - 1); + u32 *sample = (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes]; + + kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( + backend_csf, sample); + } + + /* Sync zeroed buffers to avoid coherency issues on future use. */ + backend_csf->info->csf_if->ring_buf_sync( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf, + extract_index_to_start, (insert_index_to_stop - 1), false); + + /* After consuming all samples between extract_idx and insert_idx, + * set the raw extract index to insert_idx so that the sample buffers + * can be released back to the ring buffer pool. + */ + backend_csf->info->csf_if->set_extract_index( + backend_csf->info->csf_if->ctx, insert_index_to_stop); +} + +static void kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + struct kbase_hwcnt_backend_csf *backend_csf, + enum kbase_hwcnt_backend_csf_enable_state new_state) +{ + lockdep_assert_held(&backend_csf->info->lock); + + if (backend_csf->enable_state != new_state) { + backend_csf->enable_state = new_state; + + wake_up(&backend_csf->enable_state_waitq); + } +} + +/** + * kbasep_hwcnt_backend_csf_dump_worker() - HWC dump worker. + * @work: Work structure. + * + * To accumulate all available samples in the ring buffer when a request has + * been done. + * + */ +static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf; + u32 insert_index_to_acc; + u32 extract_index; + u32 insert_index; + + WARN_ON(!work); + backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, + hwc_dump_work); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* Assert the backend is not destroyed. */ + WARN_ON(backend_csf != backend_csf->info->backend); + + /* The backend was disabled or had an error while the worker was being + * launched. + */ + if (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED && + backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { + WARN_ON(backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); + WARN_ON(!completion_done(&backend_csf->dump_completed)); + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return; + } + + WARN_ON(backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED); + + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING; + insert_index_to_acc = backend_csf->insert_index_to_accumulate; + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + /* Read the raw extract and insert indexes from the CSF interface. */ + backend_csf->info->csf_if->get_indexes(backend_csf->info->csf_if->ctx, + &extract_index, &insert_index); + + /* Accumulate up to the insert we grabbed at the prfcnt request + * interrupt. + */ + kbasep_hwcnt_backend_csf_accumulate_samples(backend_csf, extract_index, + insert_index_to_acc); + + /* Copy to the user buffer so if a threshold interrupt fires + * between now and get(), the accumulations are untouched. + */ + kbasep_hwcnt_backend_csf_update_user_sample(backend_csf); + + /* Dump done, set state back to COMPLETED for next request. */ + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* Assert the backend is not destroyed. */ + WARN_ON(backend_csf != backend_csf->info->backend); + + /* The backend was disabled or had an error while we were accumulating. + */ + if (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED && + backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { + WARN_ON(backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); + WARN_ON(!completion_done(&backend_csf->dump_completed)); + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return; + } + + WARN_ON(backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING); + + /* Our work here is done - set the wait object and unblock waiters. */ + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; + complete_all(&backend_csf->dump_completed); + spin_unlock_irqrestore(&backend_csf->info->lock, flags); +} + +/** + * kbasep_hwcnt_backend_csf_threshold_worker() - Threshold worker. + * + * @work: Work structure. + * + * Called when a HWC threshold interrupt raised to consume all available samples + * in the ring buffer. + */ +static void kbasep_hwcnt_backend_csf_threshold_worker(struct work_struct *work) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf; + u32 extract_index; + u32 insert_index; + + WARN_ON(!work); + + backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, + hwc_threshold_work); + + /* Read the raw extract and insert indexes from the CSF interface. */ + backend_csf->info->csf_if->get_indexes(backend_csf->info->csf_if->ctx, + &extract_index, &insert_index); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* Assert the backend is not destroyed. */ + WARN_ON(backend_csf != backend_csf->info->backend); + + /* The backend was disabled or had an error while the worker was being + * launched. + */ + if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return; + } + + /* Early out if we are not in the IDLE state or COMPLETED state, as this + * means a concurrent dump is in progress and we don't want to + * interfere. + */ + if ((backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) && + (backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED)) { + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return; + } + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + /* Accumulate everything we possibly can. We grabbed offsets before the + * spin lock, so we know it is not possible for a concurrent dump's + * insert_to_accumulate to exceed the insert we grabbed. + */ + kbasep_hwcnt_backend_csf_accumulate_samples(backend_csf, extract_index, + insert_index); + + /* No need to wake up anything since it is not a user dump request. */ +} + +static void kbase_hwcnt_backend_csf_submit_dump_worker( + struct kbase_hwcnt_backend_csf_info *csf_info) +{ + unsigned long flags; + u32 extract_index; + u32 insert_index; + + WARN_ON(!csf_info); + + csf_info->csf_if->get_indexes(csf_info->csf_if->ctx, &extract_index, + &insert_index); + + spin_lock_irqsave(&csf_info->lock, flags); + + /* Make sure the backend exists and is in the correct state. + * A lot of things could have happened to it in the period before we + * acquired the lock. + */ + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info) && + (csf_info->backend->enable_state == + KBASE_HWCNT_BACKEND_CSF_ENABLED || + csf_info->backend->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) && + csf_info->backend->dump_state == + KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT) { + csf_info->backend->insert_index_to_accumulate = insert_index; + csf_info->backend->dump_state = + KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED; + + /* Submit the accumulator task into the work queue. */ + while (true != queue_work(csf_info->backend->hwc_dump_workq, + &csf_info->backend->hwc_dump_work)) { + /* Spin until we have guaranteed the work has been + * submitted. + * Without this there is a potential race where a prior + * submission of the work may still technically be on + * the queue, even though all of its work is complete. + */ + } + } + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +static void kbasep_hwcnt_backend_csf_get_physical_enable( + struct kbase_hwcnt_backend_csf *backend_csf, + const struct kbase_hwcnt_enable_map *enable_map, + struct kbase_hwcnt_backend_csf_if_enable *enable) +{ + enum kbase_hwcnt_physical_set phys_counter_set; + struct kbase_hwcnt_physical_enable_map phys_enable_map; + + kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map); + + /* process the enable_map to guarantee the block header is enabled which + * is needed for delta calculation. + */ + kbasep_hwcnt_backend_csf_process_enable_map(&phys_enable_map); + + kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, + backend_csf->info->counter_set); + + /* Use processed enable_map to enable HWC in HW level. */ + enable->fe_bm = phys_enable_map.fe_bm; + enable->shader_bm = phys_enable_map.shader_bm; + enable->tiler_bm = phys_enable_map.tiler_bm; + enable->mmu_l2_bm = phys_enable_map.mmu_l2_bm; + enable->counter_set = phys_counter_set; + enable->clk_enable_map = enable_map->clk_enable_map; +} + +static int kbasep_hwcnt_backend_csf_dump_enable_impl( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map, + struct kbase_hwcnt_backend_csf_if_enable *out_enable) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_if_enable enable; + + WARN_ON(!out_enable); + + if (!backend_csf || !enable_map || + (enable_map->metadata != backend_csf->info->metadata)) + return -EINVAL; + + kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, + &enable); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* enable_state should be DISABLED before we transfer it to enabled */ + if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) { + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return -EIO; + } + + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + WARN_ON(!completion_done(&backend_csf->dump_completed)); + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED); + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + *out_enable = enable; + return 0; +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ +static int kbasep_hwcnt_backend_csf_dump_enable_nolock( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map) +{ + int errcode; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_if_enable enable; + + errcode = kbasep_hwcnt_backend_csf_dump_enable_impl(backend, enable_map, + &enable); + if (errcode) + return errcode; + + backend_csf->info->csf_if->dump_enable_nolock( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf, &enable); + + kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + + return 0; +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_enable_fn */ +static int kbasep_hwcnt_backend_csf_dump_enable( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map) +{ + int errcode; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_if_enable enable; + + errcode = kbasep_hwcnt_backend_csf_dump_enable_impl(backend, enable_map, + &enable); + if (errcode) + return errcode; + + backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, + backend_csf->ring_buf, &enable); + + kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + + return 0; +} + +static void kbasep_hwcnt_backend_csf_wait_enable_transition_complete( + struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) +{ + lockdep_assert_held(&backend_csf->info->lock); + + while ((backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) || + (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)) { + spin_unlock_irqrestore(&backend_csf->info->lock, *lock_flags); + + wait_event( + backend_csf->enable_state_waitq, + (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) && + (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)); + + spin_lock_irqsave(&backend_csf->info->lock, *lock_flags); + } +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_disable_fn */ +static void +kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *backend) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + bool do_disable = false; + + WARN_ON(!backend_csf); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + + /* Make sure we wait until any previous enable or disable have completed + * before doing anything. + */ + kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, + &flags); + + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || + backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { + /* If we are already disabled or in an unrecoverable error + * state, there is nothing for us to do. + */ + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return; + } + + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + /* Only disable if we were previously enabled - in all other + * cases the call to disable will have already been made. + */ + do_disable = true; + } + + WARN_ON(backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); + WARN_ON(!completion_done(&backend_csf->dump_completed)); + + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + /* Block until any async work has completed. We have transitioned out of + * the ENABLED state so we can guarantee no new work will concurrently + * be submitted. + */ + flush_workqueue(backend_csf->hwc_dump_workq); + + if (do_disable) + backend_csf->info->csf_if->dump_disable( + backend_csf->info->csf_if->ctx); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + + kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, + &flags); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: + WARN_ON(true); + break; + case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + break; + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); + break; + } + + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + /* After disable, zero the header of all buffers in the ring buffer back + * to 0 to prepare for the next enable. + */ + kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); + + /* Sync zeroed buffers to avoid coherency issues on future use. */ + backend_csf->info->csf_if->ring_buf_sync( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf, 0, + (backend_csf->info->ring_buf_cnt - 1), false); + + /* Reset accumulator, old_sample_buf and user_sample to all-0 to prepare + * for next enable. + */ + kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf); +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_request_fn */ +static int +kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *backend, + u64 *dump_time_ns) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + bool do_request = false; + + if (!backend_csf) + return -EINVAL; + + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* Make sure we are enabled or becoming enabled. */ + if ((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) && + (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED)) { + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + return -EIO; + } + + /* Make sure that this is either the first request since enable or the + * previous dump has completed, so we can avoid midway through a dump. + */ + if ((backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) && + (backend_csf->dump_state != + KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED)) { + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + /* HWC is disabled or another dump is ongoing, or we are on + * fault. + */ + return -EIO; + } + + /* If we are transitioning to enabled there is nothing to accumulate, + * and the user dump buffer is already zeroed. + * We can just short circuit to the DUMP_COMPLETED state. + */ + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + backend_csf->dump_state = + KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + *dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); + kbasep_hwcnt_backend_csf_cc_update(backend_csf); + return 0; + } + + /* Reset the completion so dump_wait() has something to wait on. */ + reinit_completion(&backend_csf->dump_completed); + + if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && + !backend_csf->info->fw_in_protected_mode) { + /* Only do the request if we are fully enabled and not in + * protected mode. + */ + backend_csf->dump_state = + KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED; + do_request = true; + } else { + /* Skip the request and waiting for ack and go straight to + * checking the insert and kicking off the worker to do the dump + */ + backend_csf->dump_state = + KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT; + } + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + /* CSF firmware might enter protected mode now, but still call request. + * That is fine, as we changed state while holding the lock, so the + * protected mode enter function will query the insert and launch the + * dumping worker. + * At some point we will get the dump request ACK saying a dump is done, + * but we can ignore it if we are not in the REQUESTED state and process + * it in next round dumping worker. + */ + + *dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); + kbasep_hwcnt_backend_csf_cc_update(backend_csf); + + if (do_request) { + backend_csf->info->csf_if->dump_request( + backend_csf->info->csf_if->ctx); + } else { + kbase_hwcnt_backend_csf_submit_dump_worker(backend_csf->info); + } + + return 0; +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_wait_fn */ +static int +kbasep_hwcnt_backend_csf_dump_wait(struct kbase_hwcnt_backend *backend) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + int errcode; + + if (!backend_csf) + return -EINVAL; + + wait_for_completion(&backend_csf->dump_completed); + + spin_lock_irqsave(&backend_csf->info->lock, flags); + /* Make sure the last dump actually succeeded. */ + errcode = (backend_csf->dump_state == + KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED) ? + 0 : + -EIO; + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + return errcode; +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_clear_fn */ +static int +kbasep_hwcnt_backend_csf_dump_clear(struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + int errcode; + u64 ts; + + if (!backend_csf) + return -EINVAL; + + /* Request a dump so we can clear all current counters. */ + errcode = kbasep_hwcnt_backend_csf_dump_request(backend, &ts); + if (!errcode) + /* Wait for the manual dump or auto dump to be done and + * accumulator to be updated. + */ + errcode = kbasep_hwcnt_backend_csf_dump_wait(backend); + + return errcode; +} + +/* CSF backend implementation of kbase_hwcnt_backend_dump_get_fn */ +static int kbasep_hwcnt_backend_csf_dump_get( + struct kbase_hwcnt_backend *backend, + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map, bool accumulate) +{ + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + int ret; + size_t clk; + + if (!backend_csf || !dst || !dst_enable_map || + (backend_csf->info->metadata != dst->metadata) || + (dst_enable_map->metadata != dst->metadata)) + return -EINVAL; + + kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { + if (!kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk)) + continue; + + /* Extract elapsed cycle count for each clock domain. */ + dst->clk_cnt_buf[clk] = backend_csf->cycle_count_elapsed[clk]; + } + + /* We just return the user buffer without checking the current state, + * as it is undefined to call this function without a prior succeeding + * one to dump_wait(). + */ + ret = kbase_hwcnt_csf_dump_get(dst, backend_csf->to_user_buf, + dst_enable_map, accumulate); + + return ret; +} + +/** + * kbasep_hwcnt_backend_csf_destroy() - Destroy CSF backend. + * @backend_csf: Pointer to CSF backend to destroy. + * + * Can be safely called on a backend in any state of partial construction. + * + */ +static void +kbasep_hwcnt_backend_csf_destroy(struct kbase_hwcnt_backend_csf *backend_csf) +{ + if (!backend_csf) + return; + + destroy_workqueue(backend_csf->hwc_dump_workq); + + if (backend_csf->info->csf_if->ring_buf_free) { + backend_csf->info->csf_if->ring_buf_free( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf); + } + + kfree(backend_csf->accum_buf); + backend_csf->accum_buf = NULL; + + kfree(backend_csf->old_sample_buf); + backend_csf->old_sample_buf = NULL; + + kfree(backend_csf->to_user_buf); + backend_csf->to_user_buf = NULL; + + kfree(backend_csf); +} + +/** + * kbasep_hwcnt_backend_csf_create() - Create a CSF backend instance. + * + * @csf_info: Non-NULL pointer to backend info. + * @out_backend: Non-NULL pointer to where backend is stored on success. + * Return: 0 on success, else error code. + */ +static int +kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info *csf_info, + struct kbase_hwcnt_backend_csf **out_backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = NULL; + int errcode = -ENOMEM; + + WARN_ON(!csf_info); + WARN_ON(!out_backend); + + backend_csf = kzalloc(sizeof(*backend_csf), GFP_KERNEL); + if (!backend_csf) + goto alloc_error; + + backend_csf->info = csf_info; + kbasep_hwcnt_backend_csf_init_layout(&csf_info->gpu_info, + &backend_csf->phys_layout); + + backend_csf->accum_buf = kzalloc(csf_info->dump_bytes, GFP_KERNEL); + if (!backend_csf->accum_buf) + goto err_alloc_acc_buf; + + backend_csf->old_sample_buf = kzalloc(csf_info->dump_bytes, GFP_KERNEL); + if (!backend_csf->old_sample_buf) + goto err_alloc_pre_sample_buf; + + backend_csf->to_user_buf = kzalloc(csf_info->dump_bytes, GFP_KERNEL); + if (!backend_csf->to_user_buf) + goto err_alloc_user_sample_buf; + + errcode = csf_info->csf_if->ring_buf_alloc( + csf_info->csf_if->ctx, csf_info->ring_buf_cnt, + &backend_csf->ring_buf_cpu_base, &backend_csf->ring_buf); + if (errcode) + goto err_ring_buf_alloc; + + /* Zero all performance enable header to prepare for first enable. */ + kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); + + /* Sync zeroed buffers to avoid coherency issues on use. */ + backend_csf->info->csf_if->ring_buf_sync( + backend_csf->info->csf_if->ctx, backend_csf->ring_buf, 0, + (backend_csf->info->ring_buf_cnt - 1), false); + + init_completion(&backend_csf->dump_completed); + + init_waitqueue_head(&backend_csf->enable_state_waitq); + + /* Allocate a single threaded work queue for dump worker and threshold + * worker. + */ + backend_csf->hwc_dump_workq = + alloc_workqueue("mali_hwc_dump_wq", WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!backend_csf->hwc_dump_workq) + goto err_alloc_workqueue; + + INIT_WORK(&backend_csf->hwc_dump_work, + kbasep_hwcnt_backend_csf_dump_worker); + INIT_WORK(&backend_csf->hwc_threshold_work, + kbasep_hwcnt_backend_csf_threshold_worker); + + backend_csf->enable_state = KBASE_HWCNT_BACKEND_CSF_DISABLED; + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + + *out_backend = backend_csf; + return 0; + + destroy_workqueue(backend_csf->hwc_dump_workq); +err_alloc_workqueue: + backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, + backend_csf->ring_buf); +err_ring_buf_alloc: + kfree(backend_csf->to_user_buf); + backend_csf->to_user_buf = NULL; +err_alloc_user_sample_buf: + kfree(backend_csf->old_sample_buf); + backend_csf->old_sample_buf = NULL; +err_alloc_pre_sample_buf: + kfree(backend_csf->accum_buf); + backend_csf->accum_buf = NULL; +err_alloc_acc_buf: + kfree(backend_csf); +alloc_error: + return errcode; +} + +/* CSF backend implementation of kbase_hwcnt_backend_init_fn */ +static int +kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info *info, + struct kbase_hwcnt_backend **out_backend) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = NULL; + struct kbase_hwcnt_backend_csf_info *csf_info = + (struct kbase_hwcnt_backend_csf_info *)info; + int errcode; + bool success = false; + + if (!info || !out_backend) + return -EINVAL; + + /* Create the backend. */ + errcode = kbasep_hwcnt_backend_csf_create(csf_info, &backend_csf); + if (errcode) + return errcode; + + /* If it was not created before, attach it to csf_info. + * Use spin lock to avoid concurrent initialization. + */ + spin_lock_irqsave(&csf_info->lock, flags); + if (csf_info->backend == NULL) { + csf_info->backend = backend_csf; + *out_backend = (struct kbase_hwcnt_backend *)backend_csf; + success = true; + if (csf_info->unrecoverable_error_happened) { + backend_csf->enable_state = + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR; + } + } + spin_unlock_irqrestore(&csf_info->lock, flags); + + /* Destroy the new created backend if the backend has already created + * before. In normal case, this won't happen if the client call init() + * function properly. + */ + if (!success) { + kbasep_hwcnt_backend_csf_destroy(backend_csf); + return -EBUSY; + } + + return 0; +} + +/* CSF backend implementation of kbase_hwcnt_backend_term_fn */ +static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf *backend_csf = + (struct kbase_hwcnt_backend_csf *)backend; + + if (!backend) + return; + + kbasep_hwcnt_backend_csf_dump_disable(backend); + + /* Set the backend in csf_info to NULL so we won't handle any external + * notification anymore since we are terminating. + */ + spin_lock_irqsave(&backend_csf->info->lock, flags); + backend_csf->info->backend = NULL; + spin_unlock_irqrestore(&backend_csf->info->lock, flags); + + kbasep_hwcnt_backend_csf_destroy(backend_csf); +} + +/** + * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. + * @info: Pointer to info to destroy. + * + * Can be safely called on a backend info in any state of partial construction. + * + */ +static void kbasep_hwcnt_backend_csf_info_destroy( + const struct kbase_hwcnt_backend_csf_info *info) +{ + if (!info) + return; + + /* The backend should be destroyed before the info object destroy. */ + WARN_ON(info->backend != NULL); + + /* The metadata should be destroyed before the info object destroy. */ + WARN_ON(info->metadata != NULL); + + kfree(info); +} + +/** + * kbasep_hwcnt_backend_csf_info_create() - Create a CSF backend info. + * + * @csf_if: Non-NULL pointer to a hwcnt backend CSF interface structure + * used to create backend interface. + * @ring_buf_cnt: The buffer count of the CSF hwcnt backend ring buffer. + * MUST be power of 2. + * @out_info: Non-NULL pointer to where info is stored on success. + * @return 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_csf_info_create( + struct kbase_hwcnt_backend_csf_if *csf_if, u32 ring_buf_cnt, + const struct kbase_hwcnt_backend_csf_info **out_info) +{ + struct kbase_hwcnt_backend_csf_info *info = NULL; + + WARN_ON(!csf_if); + WARN_ON(!out_info); + WARN_ON(!is_power_of_2(ring_buf_cnt)); + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + spin_lock_init(&info->lock); + +#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) + info->counter_set = KBASE_HWCNT_SET_SECONDARY; +#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) + info->counter_set = KBASE_HWCNT_SET_TERTIARY; +#else + /* Default to primary */ + info->counter_set = KBASE_HWCNT_SET_PRIMARY; +#endif + + info->backend = NULL; + info->csf_if = csf_if; + info->ring_buf_cnt = ring_buf_cnt; + info->fw_in_protected_mode = false; + info->unrecoverable_error_happened = false; + + *out_info = info; + + return 0; +} + +/* CSF backend implementation of kbase_hwcnt_backend_metadata_fn */ +static const struct kbase_hwcnt_metadata * +kbasep_hwcnt_backend_csf_metadata(const struct kbase_hwcnt_backend_info *info) +{ + if (!info) + return NULL; + + WARN_ON(!((const struct kbase_hwcnt_backend_csf_info *)info)->metadata); + + return ((const struct kbase_hwcnt_backend_csf_info *)info)->metadata; +} + +static void kbasep_hwcnt_backend_csf_handle_unrecoverable_error( + struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) +{ + bool do_disable = false; + + lockdep_assert_held(&backend_csf->info->lock); + + /* We are already in or transitioning to the unrecoverable error state. + * Early out. + */ + if ((backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) || + (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER)) + return; + + /* If we are disabled, we know we have no pending workers, so skip the + * waiting state. + */ + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); + return; + } + + /* Trigger a disable only if we are not already transitioning to + * disabled, we don't want to disable twice if an unrecoverable error + * happens while we are disabling. + */ + do_disable = (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER); + + /* Transition the dump to the IDLE state and unblock any waiters. The + * IDLE state signifies an error. + */ + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + + /* Trigger a disable only if we are not already transitioning to + * disabled, - we don't want to disable twice if an unrecoverable error + * happens while we are disabling. + */ + if (do_disable) { + spin_unlock_irqrestore(&backend_csf->info->lock, *lock_flags); + backend_csf->info->csf_if->dump_disable( + backend_csf->info->csf_if->ctx); + spin_lock_irqsave(&backend_csf->info->lock, *lock_flags); + } +} + +static void kbasep_hwcnt_backend_csf_handle_recoverable_error( + struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) +{ + lockdep_assert_held(&backend_csf->info->lock); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: + /* Already disabled or disabling, or in an unrecoverable error. + * Nothing to be done to handle the error. + */ + return; + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + /* A seemingly recoverable error that occurs while we are + * transitioning to enabled is probably unrecoverable. + */ + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(backend_csf, + lock_flags); + return; + case KBASE_HWCNT_BACKEND_CSF_ENABLED: + /* Start transitioning to the disabled state. We can't wait for + * it as this recoverable error might be triggered from an + * interrupt. The wait will be done in the eventual call to + * disable(). + */ + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + /* Transition the dump to the IDLE state and unblock any + * waiters. The IDLE state signifies an error. + */ + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + + /* Unlock spin lock before we call csf_if disable(). */ + spin_unlock_irqrestore(&backend_csf->info->lock, *lock_flags); + + backend_csf->info->csf_if->dump_disable( + backend_csf->info->csf_if->ctx); + + /* Lock spin lock again to match the spin lock pairs. */ + spin_lock_irqsave(&backend_csf->info->lock, *lock_flags); + return; + } +} + +void kbase_hwcnt_backend_csf_protm_entered( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + csf_info->fw_in_protected_mode = true; + + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + + backend_csf = csf_info->backend; + /* If we are not in REQUESTED state, we don't need to do the dumping. */ + if (backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT; + + spin_unlock_irqrestore(&csf_info->lock, flags); + kbase_hwcnt_backend_csf_submit_dump_worker(csf_info); +} + +void kbase_hwcnt_backend_csf_protm_exited( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + csf_info->fw_in_protected_mode = false; + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_unrecoverable_error( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + csf_info->unrecoverable_error_happened = true; + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(csf_info->backend, + &flags); + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_before_reset( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + csf_info->unrecoverable_error_happened = false; + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf = csf_info->backend; + + if ((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && + (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR)) { + /* Before a reset occurs, we must either have been disabled + * (else we lose data) or we should have encountered an + * unrecoverable error. Either way, we will have disabled the + * interface and waited for any workers that might have still + * been in flight. + * If not in these states, fire off one more disable to make + * sure everything is turned off before the power is pulled. + * We can't wait for this disable to complete, but it doesn't + * really matter, the power is being pulled. + */ + kbasep_hwcnt_backend_csf_handle_unrecoverable_error( + csf_info->backend, &flags); + } + + /* A reset is the only way to exit the unrecoverable error state */ + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + } + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_prfcnt_sample( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf = csf_info->backend; + + /* If the current state is not REQUESTED, this HWC sample will be + * skipped and processed in next dump_request. + */ + if (backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT; + + spin_unlock_irqrestore(&csf_info->lock, flags); + kbase_hwcnt_backend_csf_submit_dump_worker(csf_info); +} + +void kbase_hwcnt_backend_csf_on_prfcnt_threshold( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf = csf_info->backend; + + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { + /* Submit the threshold work into the work queue to consume the + * available samples. + */ + queue_work(backend_csf->hwc_dump_workq, + &backend_csf->hwc_threshold_work); + } + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_prfcnt_overflow( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + + /* Called when an overflow occurs. We treat this as a recoverable error, + * so we start transitioning to the disabled state. + * We could try and handle it while enabled, but in a real system we + * never expect an overflow to occur so there is no point implementing + * complex recovery code when we can just turn ourselves off instead for + * a while. + */ + kbasep_hwcnt_backend_csf_handle_recoverable_error(csf_info->backend, + &flags); + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_prfcnt_enable( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf = csf_info->backend; + + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_ENABLED); + } else if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_ENABLED) { + /* Unexpected, but we are already in the right state so just + * ignore it. + */ + } else { + /* Unexpected state change, assume everything is broken until + * we reset. + */ + kbasep_hwcnt_backend_csf_handle_unrecoverable_error( + csf_info->backend, &flags); + } + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +void kbase_hwcnt_backend_csf_on_prfcnt_disable( + struct kbase_hwcnt_backend_interface *iface) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + spin_lock_irqsave(&csf_info->lock, flags); + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + spin_unlock_irqrestore(&csf_info->lock, flags); + return; + } + backend_csf = csf_info->backend; + + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER); + } else if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_DISABLED) { + /* Unexpected, but we are already in the right state so just + * ignore it. + */ + } else { + /* Unexpected state change, assume everything is broken until + * we reset. + */ + kbasep_hwcnt_backend_csf_handle_unrecoverable_error( + csf_info->backend, &flags); + } + + spin_unlock_irqrestore(&csf_info->lock, flags); +} + +int kbase_hwcnt_backend_csf_metadata_init( + struct kbase_hwcnt_backend_interface *iface) +{ + int errcode; + struct kbase_hwcnt_backend_csf_info *csf_info; + + if (!iface) + return -EINVAL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + WARN_ON(!csf_info->csf_if->get_gpu_info); + csf_info->csf_if->get_gpu_info(csf_info->csf_if->ctx, + &csf_info->dump_bytes, + &csf_info->gpu_info.l2_count, + &csf_info->gpu_info.core_mask, + &csf_info->gpu_info.clk_cnt); + + /* The clock domain counts should not exceed the number of maximum + * number of clock regulators. + */ + if (csf_info->gpu_info.clk_cnt > BASE_MAX_NR_CLOCKS_REGULATORS) + return -EIO; + + errcode = kbase_hwcnt_csf_metadata_create(&csf_info->gpu_info, + csf_info->counter_set, + &csf_info->metadata); + if (errcode) + return errcode; + + /* + * Dump abstraction size should be exactly the same size and layout as + * the physical dump size, for backwards compatibility. + */ + WARN_ON(csf_info->dump_bytes != csf_info->metadata->dump_buf_bytes); + + return 0; +} + +void kbase_hwcnt_backend_csf_metadata_term( + struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + + if (!iface) + return; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + if (csf_info->metadata) { + kbase_hwcnt_csf_metadata_destroy(csf_info->metadata); + csf_info->metadata = NULL; + } +} + +int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, + u32 ring_buf_cnt, + struct kbase_hwcnt_backend_interface *iface) +{ + int errcode; + const struct kbase_hwcnt_backend_csf_info *info = NULL; + + if (!iface || !csf_if) + return -EINVAL; + + /* The buffer count must be power of 2 */ + if (!is_power_of_2(ring_buf_cnt)) + return -EINVAL; + + errcode = kbasep_hwcnt_backend_csf_info_create(csf_if, ring_buf_cnt, + &info); + if (errcode) + return errcode; + + iface->info = (struct kbase_hwcnt_backend_info *)info; + iface->metadata = kbasep_hwcnt_backend_csf_metadata; + iface->init = kbasep_hwcnt_backend_csf_init; + iface->term = kbasep_hwcnt_backend_csf_term; + iface->timestamp_ns = kbasep_hwcnt_backend_csf_timestamp_ns; + iface->dump_enable = kbasep_hwcnt_backend_csf_dump_enable; + iface->dump_enable_nolock = kbasep_hwcnt_backend_csf_dump_enable_nolock; + iface->dump_disable = kbasep_hwcnt_backend_csf_dump_disable; + iface->dump_clear = kbasep_hwcnt_backend_csf_dump_clear; + iface->dump_request = kbasep_hwcnt_backend_csf_dump_request; + iface->dump_wait = kbasep_hwcnt_backend_csf_dump_wait; + iface->dump_get = kbasep_hwcnt_backend_csf_dump_get; + + return 0; +} + +void kbase_hwcnt_backend_csf_destroy( + struct kbase_hwcnt_backend_interface *iface) +{ + if (!iface) + return; + + kbasep_hwcnt_backend_csf_info_destroy( + (const struct kbase_hwcnt_backend_csf_info *)iface->info); + memset(iface, 0, sizeof(*iface)); +} diff --git a/mali_kbase/mali_kbase_hwcnt_backend_csf.h b/mali_kbase/mali_kbase_hwcnt_backend_csf.h new file mode 100644 index 0000000..c2b3644 --- /dev/null +++ b/mali_kbase/mali_kbase_hwcnt_backend_csf.h @@ -0,0 +1,184 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/** + * Concrete implementation of mali_kbase_hwcnt_backend interface for CSF + * backend. + */ + +#ifndef _KBASE_HWCNT_BACKEND_CSF_H_ +#define _KBASE_HWCNT_BACKEND_CSF_H_ + +#include "mali_kbase_hwcnt_backend.h" +#include "mali_kbase_hwcnt_backend_csf_if.h" + +/** + * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend + * interface. + * @csf_if: Non-NULL pointer to a hwcnt backend CSF interface structure + * used to create backend interface. + * @ring_buf_cnt: The buffer count of CSF hwcnt backend, used when allocate ring + * buffer, MUST be power of 2. + * @iface: Non-NULL pointer to backend interface structure that is filled + * in on creation success. + * + * Calls to iface->dump_enable_nolock() require kbdev->hwaccess_lock held. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, + u32 ring_buf_cnt, + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_metadata_init() - Initialize the metadata for a CSF + * hardware counter backend. + * @iface: Non-NULL pointer to backend interface structure + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_backend_csf_metadata_init( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_metadata_term() - Terminate the metadata for a CSF + * hardware counter backend. + * @iface: Non-NULL pointer to backend interface structure. + */ +void kbase_hwcnt_backend_csf_metadata_term( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_destroy() - Destroy a CSF hardware counter backend + * interface. + * @iface: Pointer to interface to destroy. + * + * Can be safely called on an all-zeroed interface, or on an already destroyed + * interface. + */ +void kbase_hwcnt_backend_csf_destroy( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_protm_entered() - CSf HWC backend function to receive + * notification that protected mode + * has been entered. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_protm_entered( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_protm_exited() - CSf HWC backend function to receive + * notification that protected mode has + * been exited. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_protm_exited( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_unrecoverable_error() - CSf HWC backend function + * to be called when an + * unrecoverable error + * occurs, such as the + * firmware has died or bus + * error, this puts us into + * the unrecoverable error + * state, which we can only + * get out of by a reset. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_unrecoverable_error( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_before_reset() - CSf HWC backend function to be + * called immediately before a + * reset. Takes us out of the + * unrecoverable error state, if we + * were in it. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_before_reset( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_prfcnt_sample() - CSF performance counter sample + * complete interrupt handler. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_prfcnt_sample( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_prfcnt_threshold() - CSF performance counter + * buffer reach threshold + * interrupt handler. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_prfcnt_threshold( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_prfcnt_overflow() - CSF performance counter buffer + * overflow interrupt handler. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_prfcnt_overflow( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_prfcnt_enable() - CSF performance counter enabled + * interrupt handler. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_prfcnt_enable( + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_prfcnt_disable() - CSF performance counter + * disabled interrupt handler. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_prfcnt_disable( + struct kbase_hwcnt_backend_interface *iface); + +#endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */ diff --git a/mali_kbase/mali_kbase_hwcnt_backend_csf_if.h b/mali_kbase/mali_kbase_hwcnt_backend_csf_if.h new file mode 100644 index 0000000..bfc0e3c --- /dev/null +++ b/mali_kbase/mali_kbase_hwcnt_backend_csf_if.h @@ -0,0 +1,280 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/* + * Virtual interface for CSF hardware counter backend. + */ + +#ifndef _KBASE_HWCNT_BACKEND_CSF_IF_H_ +#define _KBASE_HWCNT_BACKEND_CSF_IF_H_ + +#include <linux/types.h> + +/** + * struct kbase_hwcnt_backend_csf_if_ctx - Opaque pointer to a CSF interface + * context. + */ +struct kbase_hwcnt_backend_csf_if_ctx; + +/** + * struct kbase_hwcnt_backend_csf_if_ring_buf - Opaque pointer to a CSF + * interface ring buffer. + */ +struct kbase_hwcnt_backend_csf_if_ring_buf; + +/** + * struct kbase_hwcnt_backend_csf_if_enable - enable hardware counter collection + * structure. + * @fe_bm: Front End counters selection bitmask. + * @shader_bm: Shader counters selection bitmask. + * @tiler_bm: Tiler counters selection bitmask. + * @mmu_l2_bm: MMU_L2 counters selection bitmask. + * @counter_set: The performance counter set to enable. + * @clk_enable_map: An array of u64 bitfields, each bit of which enables cycle + * counter for a given clock domain. + */ +struct kbase_hwcnt_backend_csf_if_enable { + u32 fe_bm; + u32 shader_bm; + u32 tiler_bm; + u32 mmu_l2_bm; + u8 counter_set; + u64 clk_enable_map; +}; + +/** + * typedef kbase_hwcnt_backend_csf_if_get_gpu_info_fn - Get GPU information + * @ctx: Non-NULL pointer to a CSF context. + * @dump_size: Non-NULL pointer to where the dump size of performance counter + * sample is stored. + * @l2_count: Non-NULL pointer to where the MMU L2 cache count is stored. + * @core_mask: Non-NULL pointer to where shader core mask is stored. + * + * @clk_cnt: Non-NULL pointer to where clock domain count in the system is + * stored. + */ +typedef void (*kbase_hwcnt_backend_csf_if_get_gpu_info_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, size_t *dump_size, + size_t *l2_count, u64 *core_mask, u8 *clk_cnt); + +/** + * typedef kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn - Allocate a ring buffer + * for CSF interface. + * @ctx: Non-NULL pointer to a CSF context. + * @buf_count: The buffer count in the ring buffer to be allocated, + * MUST be power of 2. + * @cpu_dump_base: Non-NULL pointer to where ring buffer CPU base address is + * stored when success. + * @ring_buf: Non-NULL pointer to where ring buffer is stored when success. + * + * A ring buffer is needed by the CSF interface to do manual HWC sample and + * automatic HWC samples, the buffer count in the ring buffer MUST be power + * of 2 to meet the hardware requirement. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 buf_count, + void **cpu_dump_base, + struct kbase_hwcnt_backend_csf_if_ring_buf **ring_buf); + +/** + * typedef kbase_hwcnt_backend_csf_if_ring_buf_sync_fn - Sync HWC dump buffers + * memory. + * @ctx: Non-NULL pointer to a CSF context. + * @ring_buf: Non-NULL pointer to the ring buffer. + * @buf_index_first: The first buffer index in the ring buffer to be synced, + * inclusive. + * @buf_index_last: The last buffer index in the ring buffer to be synced, + * exclusive. + * @for_cpu: The direction of sync to be applied. + * It is set to true when CPU cache needs to be invalidated + * before reading the ring buffer contents. And set to false + * when CPU cache needs to be flushed after writing to the + * ring buffer. + * + * After HWC sample request is done in GPU side, we need to sync the dump memory + * to CPU side to access the HWC data. + */ +typedef void (*kbase_hwcnt_backend_csf_if_ring_buf_sync_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + u32 buf_index_first, u32 buf_index_last, bool for_cpu); + +/** + * typedef kbase_hwcnt_backend_csf_if_ring_buf_free_fn - Free a ring buffer for + * the CSF interface. + * + * @ctx: Non-NULL pointer to a CSF interface context. + * @ring_buf: Non-NULL pointer to the ring buffer which to be freed. + */ +typedef void (*kbase_hwcnt_backend_csf_if_ring_buf_free_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf); + +/** + * typedef kbase_hwcnt_backend_csf_if_timestamp_ns_fn - Get the current + * timestamp of the CSF + * interface. + * @ctx: Non-NULL pointer to a CSF interface context. + * + * Return: CSF interface timestamp in nanoseconds. + */ +typedef u64 (*kbase_hwcnt_backend_csf_if_timestamp_ns_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx); + +/** + * typedef kbase_hwcnt_backend_csf_if_dump_enable_fn - Setup and enable hardware + * counter in CSF interface. + * @ctx: Non-NULL pointer to a CSF interface context. + * @ring_buf: Non-NULL pointer to the ring buffer which used to setup the HWC. + * @enable: Non-NULL pointer to the enable map of HWC. + */ +typedef void (*kbase_hwcnt_backend_csf_if_dump_enable_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + struct kbase_hwcnt_backend_csf_if_enable *enable); + +/** + * typedef kbase_hwcnt_backend_csf_if_dump_enable_nolock_fn - Setup and enable + * hardware counter + * in CSF interface. + * @ctx: Non-NULL pointer to a CSF interface context. + * @ring_buf: Non-NULL pointer to the ring buffer which used to setup the HWC. + * @enable: Non-NULL pointer to the enable map of HWC. + * + * Exactly the same as kbase_hwcnt_backend_csf_if_dump_enable_fn(), except must + * be called in an atomic context with the spinlock documented by the specific + * backend interface held. + */ +typedef void (*kbase_hwcnt_backend_csf_if_dump_enable_nolock_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + struct kbase_hwcnt_backend_csf_if_enable *enable); + +/** + * typedef kbase_hwcnt_backend_csf_if_dump_disable_fn - Disable hardware counter + * in CSF interface. + * @ctx: Non-NULL pointer to a CSF interface context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_dump_disable_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx); + +/** + * typedef kbase_hwcnt_backend_csf_if_dump_request_fn - Request a HWC dump. + * @ctx: Non-NULL pointer to the interface context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_dump_request_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx); + +/** + * typedef kbase_hwcnt_backend_csf_if_get_indexes_fn - Get current extract and + * insert indexes of the + * ring buffer. + * @ctx: Non-NULL pointer to a CSF interface context. + * @extract_index: Non-NULL pointer where current extract index to be saved. + * @insert_index: Non-NULL pointer where current insert index to be saved. + */ +typedef void (*kbase_hwcnt_backend_csf_if_get_indexes_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 *extract_index, + u32 *insert_index); + +/** + * typedef kbase_hwcnt_backend_csf_if_set_extract_index_fn - Update the extract + * index of the ring + * buffer. + * @ctx: Non-NULL pointer to a CSF interface context. + * @extract_index: New extract index to be set. + */ +typedef void (*kbase_hwcnt_backend_csf_if_set_extract_index_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 extract_index); + +/** + * typedef kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn - Get the current + * GPU cycle count. + * @ctx: Non-NULL pointer to a CSF interface context. + * @cycle_counts: Non-NULL pointer to an array where cycle counts to be saved, + * the array size should be at least as big as the number of + * clock domains returned by get_gpu_info interface. + * @clk_enable_map: An array of bitfields, each bit specifies an enabled clock + * domain. + */ +typedef void (*kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn)( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u64 *cycle_counts, + u64 clk_enable_map); + +/** + * struct kbase_hwcnt_backend_csf_if - Hardware counter backend CSF virtual + * interface. + * @ctx: CSF interface context. + * @get_gpu_info: Function ptr to get HWC related information. + * @ring_buf_alloc: Function ptr to allocate ring buffer for CSF HWC. + * @ring_buf_sync: Function ptr to sync ring buffer to CPU. + * @ring_buf_free: Function ptr to free ring buffer for CSF HWC. + * @timestamp_ns: Function ptr to get the current CSF interface + * timestamp. + * @dump_enable: Function ptr to enable dumping. + * @dump_enable_nolock: Function ptr to enable dumping while the + * backend-specific spinlock is already held. + * @dump_disable: Function ptr to disable dumping. + * @dump_request: Function ptr to request a dump. + * @get_indexes: Function ptr to get extract and insert indexes of the + * ring buffer. + * @set_extract_index: Function ptr to set extract index of ring buffer. + * @get_gpu_cycle_count: Function ptr to get the GPU cycle count. + */ +struct kbase_hwcnt_backend_csf_if { + struct kbase_hwcnt_backend_csf_if_ctx *ctx; + kbase_hwcnt_backend_csf_if_get_gpu_info_fn get_gpu_info; + kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn ring_buf_alloc; + kbase_hwcnt_backend_csf_if_ring_buf_sync_fn ring_buf_sync; + kbase_hwcnt_backend_csf_if_ring_buf_free_fn ring_buf_free; + kbase_hwcnt_backend_csf_if_timestamp_ns_fn timestamp_ns; + kbase_hwcnt_backend_csf_if_dump_enable_fn dump_enable; + kbase_hwcnt_backend_csf_if_dump_enable_nolock_fn dump_enable_nolock; + kbase_hwcnt_backend_csf_if_dump_disable_fn dump_disable; + kbase_hwcnt_backend_csf_if_dump_request_fn dump_request; + kbase_hwcnt_backend_csf_if_get_indexes_fn get_indexes; + kbase_hwcnt_backend_csf_if_set_extract_index_fn set_extract_index; + kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn get_gpu_cycle_count; +}; + +#endif /* #define _KBASE_HWCNT_BACKEND_CSF_IF_H_ */ diff --git a/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.c b/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.c new file mode 100644 index 0000000..21e9853 --- /dev/null +++ b/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -0,0 +1,771 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * CSF GPU HWC backend firmware interface APIs. + */ + +#include <mali_kbase.h> +#include <gpu/mali_kbase_gpu_regmap.h> +#include <device/mali_kbase_device.h> +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_hwcnt_types.h" +#include "csf/mali_gpu_csf_registers.h" + +#include "csf/mali_kbase_csf_firmware.h" +#include "mali_kbase_hwcnt_backend_csf_if_fw.h" +#include "mali_kbase_hwaccess_time.h" +#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" + +#include <linux/log2.h> +#include "mali_kbase_ccswe.h" +#ifdef CONFIG_MALI_NO_MALI +#include <backend/gpu/mali_kbase_model_dummy.h> +#endif + +/** The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/* Ring buffer virtual address start at 4GB */ +#define KBASE_HWC_CSF_RING_BUFFER_VA_START (1ull << 32) + +/** + * struct kbase_hwcnt_backend_csf_if_fw_ring_buf - ring buffer for CSF interface + * used to save the manual and + * auto HWC samples from + * firmware. + * @gpu_dump_base: Starting GPU base address of the ring buffer. + * @cpu_dump_base: Starting CPU address for the mapping. + * @buf_count: Buffer count in the ring buffer, MUST be power of 2. + * @as_nr: Address space number for the memory mapping. + * @phys: Physical memory allocation used by the mapping. + * @num_pages: Size of the mapping, in memory pages. + */ +struct kbase_hwcnt_backend_csf_if_fw_ring_buf { + u64 gpu_dump_base; + void *cpu_dump_base; + size_t buf_count; + u32 as_nr; + struct tagged_addr *phys; + size_t num_pages; +}; + +/** + * struct kbase_hwcnt_backend_csf_if_fw_ctx - Firmware context for the CSF + * interface, used to communicate + * with firmware. + * @kbdev: KBase device. + * @buf_bytes: The size in bytes for each buffer in the ring buffer. + * @clk_cnt: The number of clock domains in the system. + * The maximum is 64. + * @rate_listener: Clock rate listener callback state. + * @ccswe_shader_cores: Shader cores cycle count software estimator. + */ +struct kbase_hwcnt_backend_csf_if_fw_ctx { + struct kbase_device *kbdev; + size_t buf_bytes; + u8 clk_cnt; + u64 clk_enable_map; + struct kbase_clk_rate_listener rate_listener; + struct kbase_ccswe ccswe_shader_cores; +}; + +/** + * kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback + * + * @rate_listener: Callback state + * @clk_index: Clock index + * @clk_rate_hz: Clock frequency(hz) + */ +static void kbasep_hwcnt_backend_csf_if_fw_on_freq_change( + struct kbase_clk_rate_listener *rate_listener, u32 clk_index, + u32 clk_rate_hz) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + container_of(rate_listener, + struct kbase_hwcnt_backend_csf_if_fw_ctx, + rate_listener); + u64 timestamp_ns; + + if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) + return; + + timestamp_ns = ktime_get_raw_ns(); + kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores, timestamp_ns, + clk_rate_hz); +} + +/** + * kbasep_hwcnt_backend_csf_if_fw_cc_enable() - Enable cycle count tracking + * + * @fw_ctx: Non-NULL pointer to CSF firmware interface context. + * @enable_map: Non-NULL pointer to enable map specifying enabled counters. + */ +static void kbasep_hwcnt_backend_csf_if_fw_cc_enable( + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx, u64 clk_enable_map) +{ + struct kbase_device *kbdev = fw_ctx->kbdev; + + if (kbase_hwcnt_clk_enable_map_enabled( + clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { + /* software estimation for non-top clock domains */ + struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; + const struct kbase_clk_data *clk_data = + rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; + u32 cur_freq; + unsigned long flags; + u64 timestamp_ns; + + timestamp_ns = ktime_get_raw_ns(); + + spin_lock_irqsave(&rtm->lock, flags); + + cur_freq = (u32)clk_data->clock_val; + kbase_ccswe_reset(&fw_ctx->ccswe_shader_cores); + kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores, + timestamp_ns, cur_freq); + + kbase_clk_rate_trace_manager_subscribe_no_lock( + rtm, &fw_ctx->rate_listener); + + spin_unlock_irqrestore(&rtm->lock, flags); + } + + fw_ctx->clk_enable_map = clk_enable_map; +} + +/** + * kbasep_hwcnt_backend_csf_if_fw_cc_disable() - Disable cycle count tracking + * + * @fw_ctx: Non-NULL pointer to CSF firmware interface context. + */ +static void kbasep_hwcnt_backend_csf_if_fw_cc_disable( + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx) +{ + struct kbase_device *kbdev = fw_ctx->kbdev; + struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; + u64 clk_enable_map = fw_ctx->clk_enable_map; + + if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, + KBASE_CLOCK_DOMAIN_SHADER_CORES)) + kbase_clk_rate_trace_manager_unsubscribe( + rtm, &fw_ctx->rate_listener); +} + +static void kbasep_hwcnt_backend_csf_if_fw_get_gpu_info( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, size_t *dump_size, + size_t *l2_count, u64 *core_mask, u8 *clk_cnt) +{ +#ifdef CONFIG_MALI_NO_MALI + *l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; + *core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; + *dump_size = KBASE_DUMMY_MODEL_MAX_NUM_PERF_BLOCKS * + KBASE_DUMMY_MODEL_BLOCK_SIZE; + *clk_cnt = 1; +#else + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx; + struct kbase_device *kbdev; + u32 prfcnt_size; + u32 prfcnt_hw_size = 0; + u32 prfcnt_fw_size = 0; + + WARN_ON(!ctx); + WARN_ON(!dump_size); + WARN_ON(!l2_count); + WARN_ON(!core_mask); + WARN_ON(!clk_cnt); + + fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + kbdev = fw_ctx->kbdev; + prfcnt_size = kbdev->csf.global_iface.prfcnt_size; + prfcnt_hw_size = (prfcnt_size & 0xFF) << 8; + prfcnt_fw_size = (prfcnt_size >> 16) << 8; + fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size; + *dump_size = fw_ctx->buf_bytes; + + *l2_count = kbdev->gpu_props.props.l2_props.num_l2_slices; + *core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + + *clk_cnt = fw_ctx->clk_cnt; +#endif +} + +static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 buf_count, + void **cpu_dump_base, + struct kbase_hwcnt_backend_csf_if_ring_buf **out_ring_buf) +{ + struct kbase_device *kbdev; + struct tagged_addr *phys; + struct page **page_list; + void *cpu_addr; + int ret; + int i; + size_t num_pages; + u64 flags; + struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf; + + pgprot_t cpu_map_prot = PAGE_KERNEL; + u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START; + + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + WARN_ON(!cpu_dump_base); + WARN_ON(!out_ring_buf); + + kbdev = fw_ctx->kbdev; + + /* The buffer count must be power of 2 */ + if (!is_power_of_2(buf_count)) + return -EINVAL; + + /* alignment failure */ + if (gpu_va_base & (2048 - 1)) + return -EINVAL; + + fw_ring_buf = kzalloc(sizeof(*fw_ring_buf), GFP_KERNEL); + if (!fw_ring_buf) + return -ENOMEM; + + num_pages = PFN_UP(fw_ctx->buf_bytes * buf_count); + phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); + if (!phys) + goto phys_alloc_error; + + page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); + if (!page_list) + goto page_list_alloc_error; + + /* Get physical page for the buffer */ + ret = kbase_mem_pool_alloc_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, + phys, false); + if (ret != num_pages) + goto phys_mem_pool_alloc_error; + + /* Get the CPU virtual address */ + for (i = 0; i < num_pages; i++) + page_list[i] = as_page(phys[i]); + + cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); + if (!cpu_addr) + goto vmap_error; + + flags = KBASE_REG_GPU_WR | KBASE_REG_GPU_NX | + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + + /* Update MMU table */ + ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, + gpu_va_base >> PAGE_SHIFT, phys, num_pages, + flags, MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW); + if (ret) + goto mmu_insert_failed; + + kfree(page_list); + + fw_ring_buf->gpu_dump_base = gpu_va_base; + fw_ring_buf->cpu_dump_base = cpu_addr; + fw_ring_buf->phys = phys; + fw_ring_buf->num_pages = num_pages; + fw_ring_buf->buf_count = buf_count; + fw_ring_buf->as_nr = MCU_AS_NR; + + *cpu_dump_base = fw_ring_buf->cpu_dump_base; + *out_ring_buf = + (struct kbase_hwcnt_backend_csf_if_ring_buf *)fw_ring_buf; + + return 0; + +mmu_insert_failed: + vunmap(cpu_addr); +vmap_error: + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, + phys, false, false); +phys_mem_pool_alloc_error: + kfree(page_list); +page_list_alloc_error: + kfree(phys); +phys_alloc_error: + kfree(fw_ring_buf); + return -ENOMEM; +} + +static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + u32 buf_index_first, u32 buf_index_last, bool for_cpu) +{ + struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = + (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + size_t i; + size_t pg_first; + size_t pg_last; + u64 start_address; + u64 stop_address; + u32 ring_buf_index_first; + u32 ring_buf_index_last; + + WARN_ON(!ctx); + WARN_ON(!ring_buf); + + /* Get the buffer indexes in the ring buffer. */ + ring_buf_index_first = buf_index_first & (fw_ring_buf->buf_count - 1); + ring_buf_index_last = buf_index_last & (fw_ring_buf->buf_count - 1); + + /* The start address is the offset of the first buffer. */ + start_address = fw_ctx->buf_bytes * ring_buf_index_first; + pg_first = start_address >> PAGE_SHIFT; + + /* The stop address is the last byte in the final buffer. */ + stop_address = (fw_ctx->buf_bytes * (ring_buf_index_last + 1)) - 1; + pg_last = stop_address >> PAGE_SHIFT; + + /* Check whether the buffer range wraps. */ + if (start_address > stop_address) { + /* sync the first part to the end of ring buffer. */ + for (i = pg_first; i < fw_ring_buf->num_pages; i++) { + struct page *pg = as_page(fw_ring_buf->phys[i]); + if (for_cpu) { + kbase_sync_single_for_cpu(fw_ctx->kbdev, + kbase_dma_addr(pg), + PAGE_SIZE, + DMA_BIDIRECTIONAL); + } else { + kbase_sync_single_for_device(fw_ctx->kbdev, + kbase_dma_addr(pg), + PAGE_SIZE, + DMA_BIDIRECTIONAL); + } + } + + /* second part starts from page 0. */ + pg_first = 0; + } + + for (i = pg_first; i <= pg_last; i++) { + struct page *pg = as_page(fw_ring_buf->phys[i]); + if (for_cpu) { + kbase_sync_single_for_cpu(fw_ctx->kbdev, + kbase_dma_addr(pg), PAGE_SIZE, + DMA_BIDIRECTIONAL); + } else { + kbase_sync_single_for_device(fw_ctx->kbdev, + kbase_dma_addr(pg), + PAGE_SIZE, + DMA_BIDIRECTIONAL); + } + } +} + +static u64 kbasep_hwcnt_backend_csf_if_fw_timestamp_ns( + struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + CSTD_UNUSED(ctx); + return ktime_get_raw_ns(); +} + +static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_free( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf) +{ + struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = + (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + if (!fw_ring_buf) + return; + + if (fw_ring_buf->phys) { + u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START; + + WARN_ON(kbase_mmu_teardown_pages( + fw_ctx->kbdev, &fw_ctx->kbdev->csf.mcu_mmu, + gpu_va_base >> PAGE_SHIFT, fw_ring_buf->num_pages, + MCU_AS_NR)); + + vunmap(fw_ring_buf->cpu_dump_base); + + kbase_mem_pool_free_pages( + &fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + fw_ring_buf->num_pages, fw_ring_buf->phys, false, + false); + + kfree(fw_ring_buf->phys); + + kfree(fw_ring_buf); + } +} + +static void kbasep_hwcnt_backend_csf_if_fw_dump_enable_nolock( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + struct kbase_hwcnt_backend_csf_if_enable *enable) +{ + unsigned long flags; + u32 prfcnt_config; + struct kbase_device *kbdev; + struct kbase_csf_global_iface *global_iface; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = + (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; + + WARN_ON(!ctx); + WARN_ON(!ring_buf); + WARN_ON(!enable); + + kbdev = fw_ctx->kbdev; + global_iface = &kbdev->csf.global_iface; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Configure */ + prfcnt_config = fw_ring_buf->buf_count; + prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + + /* Configure the ring buffer base address */ + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_JASID, + fw_ring_buf->as_nr); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_LO, + fw_ring_buf->gpu_dump_base & U32_MAX); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_HI, + fw_ring_buf->gpu_dump_base >> 32); + + /* Set extract position to 0 */ + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_EXTRACT, 0); + + /* Configure the enable bitmap */ + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSF_EN, + enable->fe_bm); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_SHADER_EN, + enable->shader_bm); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_MMU_L2_EN, + enable->mmu_l2_bm); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_TILER_EN, + enable->tiler_bm); + + /* Configure the HWC set and buffer size */ + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CONFIG, + prfcnt_config); + + kbdev->csf.hwcnt.enable_pending = true; + + /* Unmask the interrupts */ + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK, + GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK); + + /* Enable the HWC */ + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, + (1 << GLB_REQ_PRFCNT_ENABLE_SHIFT), + GLB_REQ_PRFCNT_ENABLE_MASK); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + + prfcnt_config = kbase_csf_firmware_global_input_read(global_iface, + GLB_PRFCNT_CONFIG); + + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + kbasep_hwcnt_backend_csf_if_fw_cc_enable(fw_ctx, + enable->clk_enable_map); +} + +static void kbasep_hwcnt_backend_csf_if_fw_dump_enable( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, + struct kbase_hwcnt_backend_csf_if_enable *enable) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + WARN_ON(!ring_buf); + WARN_ON(!enable); + + kbdev = fw_ctx->kbdev; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbasep_hwcnt_backend_csf_if_fw_dump_enable_nolock(ctx, ring_buf, + enable); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static void kbasep_hwcnt_backend_csf_if_fw_dump_disable( + struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbase_csf_global_iface *global_iface; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + + kbdev = fw_ctx->kbdev; + global_iface = &kbdev->csf.global_iface; + + /* Disable the HWC */ + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbdev->csf.hwcnt.enable_pending = true; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, 0, + GLB_REQ_PRFCNT_ENABLE_MASK); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + + /* mask the interrupts */ + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, 0, + GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, 0, + GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_ACK_IRQ_MASK, 0, + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK); + + /* In case we have a previous request in flight when the disable + * happens. + */ + kbdev->csf.hwcnt.request_pending = false; + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + kbasep_hwcnt_backend_csf_if_fw_cc_disable(fw_ctx); +} + +static void kbasep_hwcnt_backend_csf_if_fw_dump_request( + struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + unsigned long flags; + u32 glb_req; + struct kbase_device *kbdev; + struct kbase_csf_global_iface *global_iface; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + + kbdev = fw_ctx->kbdev; + global_iface = &kbdev->csf.global_iface; + + /* Trigger dumping */ + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbdev->csf.hwcnt.request_pending = true; + glb_req = kbase_csf_firmware_global_input_read(global_iface, GLB_REQ); + glb_req ^= GLB_REQ_PRFCNT_SAMPLE_MASK; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, + GLB_REQ_PRFCNT_SAMPLE_MASK); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); +} + +static void kbasep_hwcnt_backend_csf_if_fw_get_indexes( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 *extract_index, + u32 *insert_index) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + WARN_ON(!extract_index); + WARN_ON(!insert_index); + + kbase_csf_scheduler_spin_lock(fw_ctx->kbdev, &flags); + *extract_index = kbase_csf_firmware_global_input_read( + &fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_EXTRACT); + *insert_index = kbase_csf_firmware_global_output( + &fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_INSERT); + kbase_csf_scheduler_spin_unlock(fw_ctx->kbdev, flags); +} + +static void kbasep_hwcnt_backend_csf_if_fw_set_extract_index( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 extract_idx) +{ + unsigned long flags; + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + WARN_ON(!ctx); + + /* Set the raw extract index to release the buffer back to the ring + * buffer. + */ + kbase_csf_scheduler_spin_lock(fw_ctx->kbdev, &flags); + kbase_csf_firmware_global_input(&fw_ctx->kbdev->csf.global_iface, + GLB_PRFCNT_EXTRACT, extract_idx); + kbase_csf_scheduler_spin_unlock(fw_ctx->kbdev, flags); +} + +static void kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count( + struct kbase_hwcnt_backend_csf_if_ctx *ctx, u64 *cycle_counts, + u64 clk_enable_map) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + u8 clk; + u64 timestamp_ns = ktime_get_raw_ns(); + + WARN_ON(!ctx); + WARN_ON(!cycle_counts); + + for (clk = 0; clk < fw_ctx->clk_cnt; clk++) { + if (!(clk_enable_map & (1ull << clk))) + continue; + + if (clk == KBASE_CLOCK_DOMAIN_TOP) { + /* Read cycle count for top clock domain. */ + kbase_backend_get_gpu_time_norequest( + fw_ctx->kbdev, &cycle_counts[clk], NULL, NULL); + } else { + /* Estimate cycle count for non-top clock domain. */ + cycle_counts[clk] = kbase_ccswe_cycle_at( + &fw_ctx->ccswe_shader_cores, timestamp_ns); + } + } +} + +/** + * @brief Destroy a CSF FW interface context. + * + * @param[in,out] fw_ctx Pointer to context to destroy. + */ +static void kbasep_hwcnt_backend_csf_if_fw_ctx_destroy( + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx) +{ + if (!fw_ctx) + return; + + kfree(fw_ctx); +} + +/** + * kbasep_hwcnt_backend_csf_if_fw_ctx_create() - Create a CSF Firmware context. + * + * @kbdev: Non_NULL pointer to kbase device. + * @out_ctx: Non-NULL pointer to where info is stored on success. + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_csf_if_fw_ctx_create( + struct kbase_device *kbdev, + struct kbase_hwcnt_backend_csf_if_fw_ctx **out_ctx) +{ + u8 clk; + int errcode = -ENOMEM; + struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL; + + WARN_ON(!kbdev); + WARN_ON(!out_ctx); + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + goto error; + + ctx->kbdev = kbdev; + + /* Determine the number of available clock domains. */ + for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { + if (kbdev->pm.clk_rtm.clks[clk] == NULL) + break; + } + ctx->clk_cnt = clk; + + ctx->clk_enable_map = 0; + kbase_ccswe_init(&ctx->ccswe_shader_cores); + ctx->rate_listener.notify = + kbasep_hwcnt_backend_csf_if_fw_on_freq_change; + + *out_ctx = ctx; + + return 0; +error: + kbasep_hwcnt_backend_csf_if_fw_ctx_destroy(ctx); + return errcode; +} + +void kbase_hwcnt_backend_csf_if_fw_destroy( + struct kbase_hwcnt_backend_csf_if *if_fw) +{ + if (!if_fw) + return; + + kbasep_hwcnt_backend_csf_if_fw_ctx_destroy( + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)if_fw->ctx); + memset(if_fw, 0, sizeof(*if_fw)); +} + +int kbase_hwcnt_backend_csf_if_fw_create( + struct kbase_device *kbdev, struct kbase_hwcnt_backend_csf_if *if_fw) +{ + int errcode; + struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL; + + if (!kbdev || !if_fw) + return -EINVAL; + + errcode = kbasep_hwcnt_backend_csf_if_fw_ctx_create(kbdev, &ctx); + if (errcode) + return errcode; + + if_fw->ctx = (struct kbase_hwcnt_backend_csf_if_ctx *)ctx; + if_fw->get_gpu_info = kbasep_hwcnt_backend_csf_if_fw_get_gpu_info; + if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc; + if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync; + if_fw->ring_buf_free = kbasep_hwcnt_backend_csf_if_fw_ring_buf_free; + if_fw->timestamp_ns = kbasep_hwcnt_backend_csf_if_fw_timestamp_ns; + if_fw->dump_enable = kbasep_hwcnt_backend_csf_if_fw_dump_enable; + if_fw->dump_enable_nolock = + kbasep_hwcnt_backend_csf_if_fw_dump_enable_nolock; + if_fw->dump_disable = kbasep_hwcnt_backend_csf_if_fw_dump_disable; + if_fw->dump_request = kbasep_hwcnt_backend_csf_if_fw_dump_request; + if_fw->get_gpu_cycle_count = + kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count; + if_fw->get_indexes = kbasep_hwcnt_backend_csf_if_fw_get_indexes; + if_fw->set_extract_index = + kbasep_hwcnt_backend_csf_if_fw_set_extract_index; + + return 0; +} diff --git a/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.h b/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.h new file mode 100644 index 0000000..b826a50 --- /dev/null +++ b/mali_kbase/mali_kbase_hwcnt_backend_csf_if_fw.h @@ -0,0 +1,70 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/* + * Concrete implementation of kbase_hwcnt_backend_csf_if interface for CSF FW + */ + +#ifndef _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ +#define _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ + +#include "mali_kbase_hwcnt_backend_csf_if.h" + +/** + * kbase_hwcnt_backend_csf_if_fw_create() - Create a firmware CSF interface + * of hardware counter backend. + * @kbdev: Non-NULL pointer to Kbase device. + * @if_fw: Non-NULL pointer to backend interface structure that is filled in on + * creation success. + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_backend_csf_if_fw_create( + struct kbase_device *kbdev, struct kbase_hwcnt_backend_csf_if *if_fw); + +/** + * kbase_hwcnt_backend_csf_if_fw_destroy() - Destroy a firmware CSF interface of + * hardware counter backend. + * @if_fw: Pointer to a CSF interface to destroy. + */ +void kbase_hwcnt_backend_csf_if_fw_destroy( + struct kbase_hwcnt_backend_csf_if *if_fw); + +#endif /* _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ */ diff --git a/mali_kbase/mali_kbase_hwcnt_backend_jm.c b/mali_kbase/mali_kbase_hwcnt_backend_jm.c index 7d1334e..a1daaf2 100644 --- a/mali_kbase/mali_kbase_hwcnt_backend_jm.c +++ b/mali_kbase/mali_kbase_hwcnt_backend_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -34,11 +35,7 @@ #endif #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -#if MALI_USE_CSF -#include "mali_kbase_ctx_sched.h" -#else #include "backend/gpu/mali_kbase_pm_internal.h" -#endif /** * struct kbase_hwcnt_backend_jm_info - Information used to create an instance @@ -94,6 +91,47 @@ struct kbase_hwcnt_backend_jm { }; /** + * kbase_hwcnt_gpu_info_init() - Initialise an info structure used to create the + * hwcnt metadata. + * @kbdev: Non-NULL pointer to kbase device. + * @info: Non-NULL pointer to data structure to be filled in. + * + * The initialised info struct will only be valid for use while kbdev is valid. + */ +int kbase_hwcnt_gpu_info_init(struct kbase_device *kbdev, + struct kbase_hwcnt_gpu_info *info) +{ + size_t clk; + + if (!kbdev || !info) + return -EINVAL; + +#ifdef CONFIG_MALI_NO_MALI + info->l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; + info->core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; +#else /* CONFIG_MALI_NO_MALI */ + { + const struct base_gpu_props *props = &kbdev->gpu_props.props; + const size_t l2_count = props->l2_props.num_l2_slices; + const size_t core_mask = + props->coherency_info.group[0].core_mask; + + info->l2_count = l2_count; + info->core_mask = core_mask; + } +#endif /* CONFIG_MALI_NO_MALI */ + + /* Determine the number of available clock domains. */ + for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { + if (kbdev->pm.clk_rtm.clks[clk] == NULL) + break; + } + info->clk_cnt = clk; + + return 0; +} + +/** * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback * * @rate_listener: Callback state @@ -135,10 +173,8 @@ static void kbasep_hwcnt_backend_jm_cc_enable( if (kbase_hwcnt_clk_enable_map_enabled( clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { -#if !MALI_USE_CSF /* turn on the cycle counter */ kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); -#endif /* Read cycle count for top clock domain. */ kbase_backend_get_gpu_time_norequest( kbdev, &cycle_count, NULL, NULL); @@ -191,13 +227,12 @@ static void kbasep_hwcnt_backend_jm_cc_disable( struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; u64 clk_enable_map = backend_jm->clk_enable_map; -#if !MALI_USE_CSF if (kbase_hwcnt_clk_enable_map_enabled( clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { /* turn off the cycle counter */ kbase_pm_release_gpu_cycle_counter(kbdev); } -#endif + if (kbase_hwcnt_clk_enable_map_enabled( clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { @@ -427,9 +462,9 @@ static int kbasep_hwcnt_backend_jm_dump_get( dst->clk_cnt_buf[clk] = backend_jm->cycle_count_elapsed[clk]; } - return kbase_hwcnt_gpu_dump_get( - dst, backend_jm->cpu_dump_va, dst_enable_map, - backend_jm->pm_core_mask, accumulate); + return kbase_hwcnt_jm_dump_get(dst, backend_jm->cpu_dump_va, + dst_enable_map, backend_jm->pm_core_mask, + accumulate); } /** @@ -497,9 +532,6 @@ static void kbasep_hwcnt_backend_jm_destroy( return; if (backend->kctx) { -#if MALI_USE_CSF - unsigned long flags; -#endif struct kbase_context *kctx = backend->kctx; struct kbase_device *kbdev = kctx->kbdev; @@ -510,13 +542,7 @@ static void kbasep_hwcnt_backend_jm_destroy( kbasep_hwcnt_backend_jm_dump_free( kctx, backend->gpu_dump_va); -#if MALI_USE_CSF - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_ctx_sched_release_ctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else kbasep_js_release_privileged_ctx(kbdev, kctx); -#endif kbase_destroy_context(kctx); } @@ -534,9 +560,6 @@ static int kbasep_hwcnt_backend_jm_create( const struct kbase_hwcnt_backend_jm_info *info, struct kbase_hwcnt_backend_jm **out_backend) { -#if MALI_USE_CSF - unsigned long flags; -#endif int errcode; struct kbase_device *kbdev; struct kbase_hwcnt_backend_jm *backend = NULL; @@ -557,17 +580,7 @@ static int kbasep_hwcnt_backend_jm_create( if (!backend->kctx) goto alloc_error; -#if MALI_USE_CSF - kbase_pm_context_active(kbdev); - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_ctx_sched_retain_ctx(backend->kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - kbase_pm_context_idle(kbdev); -#else kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); -#endif errcode = kbasep_hwcnt_backend_jm_dump_alloc( info, backend->kctx, &backend->gpu_dump_va); @@ -597,6 +610,16 @@ error: return errcode; } +/* JM backend implementation of kbase_hwcnt_backend_metadata_fn */ +static const struct kbase_hwcnt_metadata * +kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info *info) +{ + if (!info) + return NULL; + + return ((const struct kbase_hwcnt_backend_jm_info *)info)->metadata; +} + /* JM backend implementation of kbase_hwcnt_backend_init_fn */ static int kbasep_hwcnt_backend_jm_init( const struct kbase_hwcnt_backend_info *info, @@ -641,7 +664,7 @@ static void kbasep_hwcnt_backend_jm_info_destroy( if (!info) return; - kbase_hwcnt_gpu_metadata_destroy(info->metadata); + kbase_hwcnt_jm_metadata_destroy(info->metadata); kfree(info); } @@ -682,10 +705,10 @@ static int kbasep_hwcnt_backend_jm_info_create( info->counter_set = KBASE_HWCNT_SET_PRIMARY; #endif - errcode = kbase_hwcnt_gpu_metadata_create(&hwcnt_gpu_info, - info->counter_set, - &info->metadata, - &info->dump_bytes); + errcode = kbase_hwcnt_jm_metadata_create(&hwcnt_gpu_info, + info->counter_set, + &info->metadata, + &info->dump_bytes); if (errcode) goto error; @@ -712,8 +735,8 @@ int kbase_hwcnt_backend_jm_create( if (errcode) return errcode; - iface->metadata = info->metadata; iface->info = (struct kbase_hwcnt_backend_info *)info; + iface->metadata = kbasep_hwcnt_backend_jm_metadata; iface->init = kbasep_hwcnt_backend_jm_init; iface->term = kbasep_hwcnt_backend_jm_term; iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; diff --git a/mali_kbase/mali_kbase_hwcnt_backend_jm.h b/mali_kbase/mali_kbase_hwcnt_backend_jm.h index f15faeb..b0b24e6 100644 --- a/mali_kbase/mali_kbase_hwcnt_backend_jm.h +++ b/mali_kbase/mali_kbase_hwcnt_backend_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_hwcnt_context.h b/mali_kbase/mali_kbase_hwcnt_context.h index bc50ad1..99d9d1c 100644 --- a/mali_kbase/mali_kbase_hwcnt_context.h +++ b/mali_kbase/mali_kbase_hwcnt_context.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** @@ -28,6 +47,7 @@ #define _KBASE_HWCNT_CONTEXT_H_ #include <linux/types.h> +#include <linux/workqueue.h> struct kbase_hwcnt_backend_interface; struct kbase_hwcnt_context; @@ -66,7 +86,7 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( /** * kbase_hwcnt_context_disable() - Increment the disable count of the context. - * @hctx: Pointer to the hardware counter context. + * @hctx: Non-NULL pointer to the hardware counter context. * * If a call to this function increments the disable count from 0 to 1, and * an accumulator has been acquired, then a counter dump will be performed @@ -84,7 +104,7 @@ void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx); * kbase_hwcnt_context_disable_atomic() - Increment the disable count of the * context if possible in an atomic * context. - * @hctx: Pointer to the hardware counter context. + * @hctx: Non-NULL pointer to the hardware counter context. * * This function will only succeed if hardware counters are effectively already * disabled, i.e. there is no accumulator, the disable count is already @@ -99,7 +119,7 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); /** * kbase_hwcnt_context_enable() - Decrement the disable count of the context. - * @hctx: Pointer to the hardware counter context. + * @hctx: Non-NULL pointer to the hardware counter context. * * If a call to this function decrements the disable count from 1 to 0, and * an accumulator has been acquired, then counters will be re-enabled via the @@ -116,4 +136,36 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); */ void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx); +/** + * kbase_hwcnt_context_queue_work() - Queue hardware counter related async + * work on a workqueue specialized for + * hardware counters. + * @hctx: Non-NULL pointer to the hardware counter context. + * @work: Non-NULL pointer to work to queue. + * + * Return: false if work was already on a queue, true otherwise. + * + * Performance counter related work is high priority, short running, and + * generally CPU locality is unimportant. There is no standard workqueue that + * can service this flavor of work. + * + * Rather than have each user of counters define their own workqueue, we have + * a centralized one in here that anybody using this hardware counter API + * should use. + * + * Before the context is destroyed, all work submitted must have been completed. + * Given that the work enqueued via this function is likely to be hardware + * counter related and will therefore use the context object, this is likely + * to be behavior that will occur naturally. + * + * Historical note: prior to this centralized workqueue, the system_highpri_wq + * was used. This was generally fine, except when a particularly long running, + * higher priority thread ended up scheduled on the enqueuing CPU core. Given + * that hardware counters requires tight integration with power management, + * this meant progress through the power management states could be stalled + * for however long that higher priority thread took. + */ +bool kbase_hwcnt_context_queue_work(struct kbase_hwcnt_context *hctx, + struct work_struct *work); + #endif /* _KBASE_HWCNT_CONTEXT_H_ */ diff --git a/mali_kbase/mali_kbase_hwcnt_gpu.c b/mali_kbase/mali_kbase_hwcnt_gpu.c index 1f4953f..9760643 100644 --- a/mali_kbase/mali_kbase_hwcnt_gpu.c +++ b/mali_kbase/mali_kbase_hwcnt_gpu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -22,10 +23,9 @@ #include "mali_kbase_hwcnt_gpu.h" #include "mali_kbase_hwcnt_types.h" -#include "mali_kbase.h" -#ifdef CONFIG_MALI_NO_MALI -#include "backend/gpu/mali_kbase_model_dummy.h" -#endif + +#include <linux/bug.h> +#include <linux/err.h> #define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 #define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 @@ -118,17 +118,18 @@ static void kbasep_get_memsys_block_type(u64 *dst, } /** - * kbasep_hwcnt_backend_gpu_metadata_v5_create() - Create hardware counter - * metadata for a v5 GPU. - * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. + * kbasep_hwcnt_backend_gpu_metadata_create() - Create hardware counter metadata + * for the GPU. + * @gpu_info: Non-NULL pointer to hwcnt info for current GPU. + * @is_csf: true for CSF GPU, otherwise false. * @counter_set: The performance counter set to use. * @metadata: Non-NULL pointer to where created metadata is stored * on success. * * Return: 0 on success, else error code. */ -static int kbasep_hwcnt_backend_gpu_metadata_v5_create( - const struct kbase_hwcnt_gpu_v5_info *v5_info, +static int kbasep_hwcnt_backend_gpu_metadata_create( + const struct kbase_hwcnt_gpu_info *gpu_info, const bool is_csf, enum kbase_hwcnt_set counter_set, const struct kbase_hwcnt_metadata **metadata) { @@ -139,13 +140,13 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( size_t non_sc_block_count; size_t sc_block_count; - WARN_ON(!v5_info); + WARN_ON(!gpu_info); WARN_ON(!metadata); /* Calculate number of block instances that aren't shader cores */ - non_sc_block_count = 2 + v5_info->l2_count; + non_sc_block_count = 2 + gpu_info->l2_count; /* Calculate number of block instances that are shader cores */ - sc_block_count = fls64(v5_info->core_mask); + sc_block_count = fls64(gpu_info->core_mask); /* * A system can have up to 64 shader cores, but the 64-bit @@ -158,7 +159,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( return -EINVAL; /* One Front End block */ - kbasep_get_fe_block_type(&blks[0].type, counter_set, v5_info->is_csf); + kbasep_get_fe_block_type(&blks[0].type, counter_set, is_csf); blks[0].inst_cnt = 1; blks[0].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; blks[0].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; @@ -171,7 +172,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( /* l2_count memsys blks */ kbasep_get_memsys_block_type(&blks[2].type, counter_set); - blks[2].inst_cnt = v5_info->l2_count; + blks[2].inst_cnt = gpu_info->l2_count; blks[2].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; blks[2].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; @@ -191,7 +192,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( * requirements, and embed the core mask into the availability mask so * we can determine later which shader cores physically exist. */ - kbasep_get_sc_block_type(&blks[3].type, counter_set, v5_info->is_csf); + kbasep_get_sc_block_type(&blks[3].type, counter_set, is_csf); blks[3].inst_cnt = sc_block_count; blks[3].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; blks[3].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; @@ -204,77 +205,34 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( desc.grp_cnt = 1; desc.grps = &group; - desc.clk_cnt = v5_info->clk_cnt; + desc.clk_cnt = gpu_info->clk_cnt; /* The JM, Tiler, and L2s are always available, and are before cores */ desc.avail_mask = (1ull << non_sc_block_count) - 1; /* Embed the core mask directly in the availability mask */ - desc.avail_mask |= (v5_info->core_mask << non_sc_block_count); + desc.avail_mask |= (gpu_info->core_mask << non_sc_block_count); return kbase_hwcnt_metadata_create(&desc, metadata); } /** - * kbasep_hwcnt_backend_gpu_v5_dump_bytes() - Get the raw dump buffer size for a - * V5 GPU. - * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. + * kbasep_hwcnt_backend_jm_dump_bytes() - Get the raw dump buffer size for the + * GPU. + * @v5_info: Non-NULL pointer to hwcnt info for the GPU. * - * Return: Size of buffer the V5 GPU needs to perform a counter dump. + * Return: Size of buffer the GPU needs to perform a counter dump. */ -static size_t kbasep_hwcnt_backend_gpu_v5_dump_bytes( - const struct kbase_hwcnt_gpu_v5_info *v5_info) +static size_t +kbasep_hwcnt_backend_jm_dump_bytes(const struct kbase_hwcnt_gpu_info *gpu_info) { - WARN_ON(!v5_info); - return (2 + v5_info->l2_count + fls64(v5_info->core_mask)) * - KBASE_HWCNT_V5_VALUES_PER_BLOCK * - KBASE_HWCNT_VALUE_BYTES; -} - -int kbase_hwcnt_gpu_info_init( - struct kbase_device *kbdev, - struct kbase_hwcnt_gpu_info *info) -{ - size_t clk; - - if (!kbdev || !info) - return -EINVAL; - -#ifdef CONFIG_MALI_NO_MALI - /* NO_MALI uses V5 layout, regardless of the underlying platform. */ - info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; - info->v5.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; - info->v5.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; -#else /* CONFIG_MALI_NO_MALI */ - { - const struct base_gpu_props *props = &kbdev->gpu_props.props; - const size_t l2_count = props->l2_props.num_l2_slices; - const size_t core_mask = - props->coherency_info.group[0].core_mask; - - info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; - info->v5.l2_count = l2_count; - info->v5.core_mask = core_mask; - } -#endif /* CONFIG_MALI_NO_MALI */ - -#ifdef MALI_USE_CSF - info->v5.is_csf = true; -#else - info->v5.is_csf = false; -#endif - - /* Determine the number of available clock domains. */ - for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { - if (kbdev->pm.clk_rtm.clks[clk] == NULL) - break; - } - info->v5.clk_cnt = clk; + WARN_ON(!gpu_info); - return 0; + return (2 + gpu_info->l2_count + fls64(gpu_info->core_mask)) * + KBASE_HWCNT_V5_VALUES_PER_BLOCK * KBASE_HWCNT_VALUE_BYTES; } -int kbase_hwcnt_gpu_metadata_create( - const struct kbase_hwcnt_gpu_info *info, +int kbase_hwcnt_jm_metadata_create( + const struct kbase_hwcnt_gpu_info *gpu_info, enum kbase_hwcnt_set counter_set, const struct kbase_hwcnt_metadata **out_metadata, size_t *out_dump_bytes) @@ -283,16 +241,12 @@ int kbase_hwcnt_gpu_metadata_create( const struct kbase_hwcnt_metadata *metadata; size_t dump_bytes; - if (!info || !out_metadata || !out_dump_bytes) + if (!gpu_info || !out_metadata || !out_dump_bytes) return -EINVAL; - if (info->type == KBASE_HWCNT_GPU_GROUP_TYPE_V5) { - dump_bytes = kbasep_hwcnt_backend_gpu_v5_dump_bytes(&info->v5); - errcode = kbasep_hwcnt_backend_gpu_metadata_v5_create( - &info->v5, counter_set, &metadata); - } else { - return -EINVAL; - } + dump_bytes = kbasep_hwcnt_backend_jm_dump_bytes(gpu_info); + errcode = kbasep_hwcnt_backend_gpu_metadata_create( + gpu_info, false, counter_set, &metadata); if (errcode) return errcode; @@ -307,9 +261,38 @@ int kbase_hwcnt_gpu_metadata_create( return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_create); -void kbase_hwcnt_gpu_metadata_destroy( +void kbase_hwcnt_jm_metadata_destroy( + const struct kbase_hwcnt_metadata *metadata) +{ + if (!metadata) + return; + + kbase_hwcnt_metadata_destroy(metadata); +} + +int kbase_hwcnt_csf_metadata_create( + const struct kbase_hwcnt_gpu_info *gpu_info, + enum kbase_hwcnt_set counter_set, + const struct kbase_hwcnt_metadata **out_metadata) +{ + int errcode; + const struct kbase_hwcnt_metadata *metadata; + + if (!gpu_info || !out_metadata) + return -EINVAL; + + errcode = kbasep_hwcnt_backend_gpu_metadata_create( + gpu_info, true, counter_set, &metadata); + if (errcode) + return errcode; + + *out_metadata = metadata; + + return 0; +} + +void kbase_hwcnt_csf_metadata_destroy( const struct kbase_hwcnt_metadata *metadata) { if (!metadata) @@ -317,7 +300,6 @@ void kbase_hwcnt_gpu_metadata_destroy( kbase_hwcnt_metadata_destroy(metadata); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_destroy); static bool is_block_type_shader( const u64 grp_type, @@ -338,12 +320,9 @@ static bool is_block_type_shader( return is_shader; } -int kbase_hwcnt_gpu_dump_get( - struct kbase_hwcnt_dump_buffer *dst, - void *src, - const struct kbase_hwcnt_enable_map *dst_enable_map, - u64 pm_core_mask, - bool accumulate) +int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, + const struct kbase_hwcnt_enable_map *dst_enable_map, + u64 pm_core_mask, bool accumulate) { const struct kbase_hwcnt_metadata *metadata; const u32 *dump_src; @@ -402,7 +381,52 @@ int kbase_hwcnt_gpu_dump_get( return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_dump_get); + +int kbase_hwcnt_csf_dump_get( + struct kbase_hwcnt_dump_buffer *dst, void *src, + const struct kbase_hwcnt_enable_map *dst_enable_map, + bool accumulate) +{ + const struct kbase_hwcnt_metadata *metadata; + const u32 *dump_src; + size_t src_offset, grp, blk, blk_inst; + + if (!dst || !src || !dst_enable_map || + (dst_enable_map->metadata != dst->metadata)) + return -EINVAL; + + metadata = dst->metadata; + dump_src = (const u32 *)src; + src_offset = 0; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + const size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count( + metadata, grp, blk); + const size_t ctr_cnt = + kbase_hwcnt_metadata_block_counters_count(metadata, grp, + blk); + + /* Early out if no values in the dest block are enabled */ + if (kbase_hwcnt_enable_map_block_enabled(dst_enable_map, grp, + blk, blk_inst)) { + u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + const u32 *src_blk = dump_src + src_offset; + + if (accumulate) { + kbase_hwcnt_dump_buffer_block_accumulate( + dst_blk, src_blk, hdr_cnt, ctr_cnt); + } else { + kbase_hwcnt_dump_buffer_block_copy( + dst_blk, src_blk, (hdr_cnt + ctr_cnt)); + } + } + + src_offset += (hdr_cnt + ctr_cnt); + } + + return 0; +} /** * kbasep_hwcnt_backend_gpu_block_map_to_physical() - Convert from a block @@ -558,7 +582,6 @@ void kbase_hwcnt_gpu_enable_map_to_physical( dst->mmu_l2_bm = kbasep_hwcnt_backend_gpu_block_map_to_physical(mmu_l2_bm, 0); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_to_physical); void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, enum kbase_hwcnt_set src) @@ -577,7 +600,6 @@ void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, WARN_ON(true); } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_set_to_physical); void kbase_hwcnt_gpu_enable_map_from_physical( struct kbase_hwcnt_enable_map *dst, @@ -649,7 +671,6 @@ void kbase_hwcnt_gpu_enable_map_from_physical( } } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_from_physical); void kbase_hwcnt_gpu_patch_dump_headers( struct kbase_hwcnt_dump_buffer *buf, @@ -683,4 +704,3 @@ void kbase_hwcnt_gpu_patch_dump_headers( } } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_patch_dump_headers); diff --git a/mali_kbase/mali_kbase_hwcnt_gpu.h b/mali_kbase/mali_kbase_hwcnt_gpu.h index c9039b2..244577d 100644 --- a/mali_kbase/mali_kbase_hwcnt_gpu.h +++ b/mali_kbase/mali_kbase_hwcnt_gpu.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_HWCNT_GPU_H_ @@ -30,6 +49,14 @@ struct kbase_hwcnt_metadata; struct kbase_hwcnt_enable_map; struct kbase_hwcnt_dump_buffer; +#define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 +#define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 +#define KBASE_HWCNT_V5_COUNTERS_PER_BLOCK 60 +#define KBASE_HWCNT_V5_VALUES_PER_BLOCK \ + (KBASE_HWCNT_V5_HEADERS_PER_BLOCK + KBASE_HWCNT_V5_COUNTERS_PER_BLOCK) +/** Index of the PRFCNT_EN header into a V5 counter block */ +#define KBASE_HWCNT_V5_PRFCNT_EN_HEADER 2 + /** * enum kbase_hwcnt_gpu_group_type - GPU hardware counter group types, used to * identify metadata groups. @@ -109,92 +136,110 @@ enum kbase_hwcnt_physical_set { }; /** - * struct kbase_hwcnt_gpu_v5_info - Information about hwcnt blocks on v5 GPUs. + * struct kbase_hwcnt_gpu_info - Information about hwcnt blocks on the GPUs. * @l2_count: L2 cache count. * @core_mask: Shader core mask. May be sparse. * @clk_cnt: Number of clock domains available. - * @is_csf: Whether CSF is used. */ -struct kbase_hwcnt_gpu_v5_info { +struct kbase_hwcnt_gpu_info { size_t l2_count; u64 core_mask; u8 clk_cnt; - bool is_csf; }; /** - * struct kbase_hwcnt_gpu_info - Tagged union with information about the current - * GPU's hwcnt blocks. - * @type: GPU type. - * @v5: Info filled in if a v5 GPU. + * kbase_hwcnt_jm_metadata_create() - Create hardware counter metadata for the + * JM GPUs. + * @info: Non-NULL pointer to info struct. + * @counter_set: The performance counter set used. + * @out_metadata: Non-NULL pointer to where created metadata is stored on + * success. + * @out_dump_bytes: Non-NULL pointer to where the size of the GPU counter dump + * buffer is stored on success. + * + * Return: 0 on success, else error code. */ -struct kbase_hwcnt_gpu_info { - enum kbase_hwcnt_gpu_group_type type; - struct kbase_hwcnt_gpu_v5_info v5; -}; +int kbase_hwcnt_jm_metadata_create( + const struct kbase_hwcnt_gpu_info *info, + enum kbase_hwcnt_set counter_set, + const struct kbase_hwcnt_metadata **out_metadata, + size_t *out_dump_bytes); /** - * kbase_hwcnt_gpu_info_init() - Initialise an info structure used to create the - * hwcnt metadata. - * @kbdev: Non-NULL pointer to kbase device. - * @info: Non-NULL pointer to data structure to be filled in. + * kbase_hwcnt_jm_metadata_destroy() - Destroy JM GPU hardware counter metadata. * - * The initialised info struct will only be valid for use while kbdev is valid. + * @metadata: Pointer to metadata to destroy. */ -int kbase_hwcnt_gpu_info_init( - struct kbase_device *kbdev, - struct kbase_hwcnt_gpu_info *info); +void kbase_hwcnt_jm_metadata_destroy( + const struct kbase_hwcnt_metadata *metadata); /** - * kbase_hwcnt_gpu_metadata_create() - Create hardware counter metadata for the - * current GPU. - * @info: Non-NULL pointer to info struct initialised by - * kbase_hwcnt_gpu_info_init. + * kbase_hwcnt_csf_metadata_create() - Create hardware counter metadata for the + * CSF GPUs. + * @info: Non-NULL pointer to info struct. * @counter_set: The performance counter set used. * @out_metadata: Non-NULL pointer to where created metadata is stored on * success. - * @out_dump_bytes: Non-NULL pointer to where the size of the GPU counter dump - * buffer is stored on success. * * Return: 0 on success, else error code. */ -int kbase_hwcnt_gpu_metadata_create( +int kbase_hwcnt_csf_metadata_create( const struct kbase_hwcnt_gpu_info *info, enum kbase_hwcnt_set counter_set, - const struct kbase_hwcnt_metadata **out_metadata, - size_t *out_dump_bytes); + const struct kbase_hwcnt_metadata **out_metadata); /** - * kbase_hwcnt_gpu_metadata_destroy() - Destroy GPU hardware counter metadata. + * kbase_hwcnt_csf_metadata_destroy() - Destroy CSF GPU hardware counter + * metadata. * @metadata: Pointer to metadata to destroy. */ -void kbase_hwcnt_gpu_metadata_destroy( +void kbase_hwcnt_csf_metadata_destroy( const struct kbase_hwcnt_metadata *metadata); /** - * kbase_hwcnt_gpu_dump_get() - Copy or accumulate enabled counters from the raw + * kbase_hwcnt_jm_dump_get() - Copy or accumulate enabled counters from the raw + * dump buffer in src into the dump buffer + * abstraction in dst. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src raw dump buffer, of same length + * as returned in out_dump_bytes parameter of + * kbase_hwcnt_jm_metadata_create. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * @pm_core_mask: PM state synchronized shaders core mask with the dump. + * @accumulate: True if counters in src should be accumulated into dst, + * rather than copied. + * + * The dst and dst_enable_map MUST have been created from the same metadata as + * returned from the call to kbase_hwcnt_jm_metadata_create as was used to get + * the length of src. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, + const struct kbase_hwcnt_enable_map *dst_enable_map, + const u64 pm_core_mask, bool accumulate); + +/** + * kbase_hwcnt_csf_dump_get() - Copy or accumulate enabled counters from the raw * dump buffer in src into the dump buffer * abstraction in dst. * @dst: Non-NULL pointer to dst dump buffer. * @src: Non-NULL pointer to src raw dump buffer, of same length * as returned in out_dump_bytes parameter of - * kbase_hwcnt_gpu_metadata_create. + * kbase_hwcnt_csf_metadata_create. * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. - * @pm_core_mask: PM state synchronized shaders core mask with the dump. * @accumulate: True if counters in src should be accumulated into dst, * rather than copied. * * The dst and dst_enable_map MUST have been created from the same metadata as - * returned from the call to kbase_hwcnt_gpu_metadata_create as was used to get + * returned from the call to kbase_hwcnt_csf_metadata_create as was used to get * the length of src. * * Return: 0 on success, else error code. */ -int kbase_hwcnt_gpu_dump_get( - struct kbase_hwcnt_dump_buffer *dst, - void *src, +int kbase_hwcnt_csf_dump_get( + struct kbase_hwcnt_dump_buffer *dst, void *src, const struct kbase_hwcnt_enable_map *dst_enable_map, - const u64 pm_core_mask, bool accumulate); /** @@ -204,7 +249,7 @@ int kbase_hwcnt_gpu_dump_get( * @src: Non-NULL pointer to src enable map abstraction. * * The src must have been created from a metadata returned from a call to - * kbase_hwcnt_gpu_metadata_create. + * kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. * * This is a lossy conversion, as the enable map abstraction has one bit per * individual counter block value, but the physical enable map uses 1 bit for @@ -231,7 +276,7 @@ void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, * @src: Non-NULL pointer to src physical enable map. * * The dst must have been created from a metadata returned from a call to - * kbase_hwcnt_gpu_metadata_create. + * kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. * * This is a lossy conversion, as the physical enable map can technically * support counter blocks with 128 counters each, but no hardware actually uses @@ -250,7 +295,7 @@ void kbase_hwcnt_gpu_enable_map_from_physical( * @enable_map: Non-NULL pointer to enable map. * * The buf and enable_map must have been created from a metadata returned from - * a call to kbase_hwcnt_gpu_metadata_create. + * a call to kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. * * This function should be used before handing off a dump buffer over the * kernel-user boundary, to ensure the header is accurate for the enable map diff --git a/mali_kbase/mali_kbase_hwcnt_legacy.c b/mali_kbase/mali_kbase_hwcnt_legacy.c index 794ef39..de9b669 100644 --- a/mali_kbase/mali_kbase_hwcnt_legacy.c +++ b/mali_kbase/mali_kbase_hwcnt_legacy.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_hwcnt_legacy.h b/mali_kbase/mali_kbase_hwcnt_legacy.h index 7a610ae..fd0d01e 100644 --- a/mali_kbase/mali_kbase_hwcnt_legacy.h +++ b/mali_kbase/mali_kbase_hwcnt_legacy.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_hwcnt_reader.h b/mali_kbase/mali_kbase_hwcnt_reader.h index 8cd3835..081244c 100644 --- a/mali_kbase/mali_kbase_hwcnt_reader.h +++ b/mali_kbase/mali_kbase_hwcnt_reader.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_HWCNT_READER_H_ diff --git a/mali_kbase/mali_kbase_hwcnt_types.c b/mali_kbase/mali_kbase_hwcnt_types.c index 2b9fe02..f4c0def 100644 --- a/mali_kbase/mali_kbase_hwcnt_types.c +++ b/mali_kbase/mali_kbase_hwcnt_types.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. @@ -21,7 +22,8 @@ */ #include "mali_kbase_hwcnt_types.h" -#include "mali_kbase.h" + +#include <linux/slab.h> /* Minimum alignment of each block of hardware counters */ #define KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT \ @@ -175,13 +177,11 @@ int kbase_hwcnt_metadata_create( *out_metadata = metadata; return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_create); void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) { kfree(metadata); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_destroy); int kbase_hwcnt_enable_map_alloc( const struct kbase_hwcnt_metadata *metadata, @@ -205,7 +205,6 @@ int kbase_hwcnt_enable_map_alloc( enable_map->hwcnt_enable_map = enable_map_buf; return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_alloc); void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map) { @@ -216,7 +215,6 @@ void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map) enable_map->hwcnt_enable_map = NULL; enable_map->metadata = NULL; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_free); int kbase_hwcnt_dump_buffer_alloc( const struct kbase_hwcnt_metadata *metadata, @@ -243,7 +241,6 @@ int kbase_hwcnt_dump_buffer_alloc( return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_alloc); void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) { @@ -253,7 +250,6 @@ void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) kfree(dump_buf->dump_buf); memset(dump_buf, 0, sizeof(*dump_buf)); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_free); int kbase_hwcnt_dump_buffer_array_alloc( const struct kbase_hwcnt_metadata *metadata, @@ -309,7 +305,6 @@ int kbase_hwcnt_dump_buffer_array_alloc( return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_alloc); void kbase_hwcnt_dump_buffer_array_free( struct kbase_hwcnt_dump_buffer_array *dump_bufs) @@ -321,7 +316,6 @@ void kbase_hwcnt_dump_buffer_array_free( free_pages(dump_bufs->page_addr, dump_bufs->page_order); memset(dump_bufs, 0, sizeof(*dump_bufs)); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_free); void kbase_hwcnt_dump_buffer_zero( struct kbase_hwcnt_dump_buffer *dst, @@ -356,7 +350,6 @@ void kbase_hwcnt_dump_buffer_zero( memset(dst->clk_cnt_buf, 0, sizeof(*dst->clk_cnt_buf) * metadata->clk_cnt); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero); void kbase_hwcnt_dump_buffer_zero_strict( struct kbase_hwcnt_dump_buffer *dst) @@ -369,7 +362,6 @@ void kbase_hwcnt_dump_buffer_zero_strict( memset(dst->clk_cnt_buf, 0, sizeof(*dst->clk_cnt_buf) * dst->metadata->clk_cnt); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_strict); void kbase_hwcnt_dump_buffer_zero_non_enabled( struct kbase_hwcnt_dump_buffer *dst, @@ -409,7 +401,6 @@ void kbase_hwcnt_dump_buffer_zero_non_enabled( } } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_non_enabled); void kbase_hwcnt_dump_buffer_copy( struct kbase_hwcnt_dump_buffer *dst, @@ -455,7 +446,6 @@ void kbase_hwcnt_dump_buffer_copy( dst->clk_cnt_buf[clk] = src->clk_cnt_buf[clk]; } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy); void kbase_hwcnt_dump_buffer_copy_strict( struct kbase_hwcnt_dump_buffer *dst, @@ -502,7 +492,6 @@ void kbase_hwcnt_dump_buffer_copy_strict( dst->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0; } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy_strict); void kbase_hwcnt_dump_buffer_accumulate( struct kbase_hwcnt_dump_buffer *dst, @@ -552,7 +541,6 @@ void kbase_hwcnt_dump_buffer_accumulate( dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate); void kbase_hwcnt_dump_buffer_accumulate_strict( struct kbase_hwcnt_dump_buffer *dst, @@ -601,4 +589,3 @@ void kbase_hwcnt_dump_buffer_accumulate_strict( dst->clk_cnt_buf[clk] = 0; } } -KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate_strict); diff --git a/mali_kbase/mali_kbase_hwcnt_types.h b/mali_kbase/mali_kbase_hwcnt_types.h index 3394b12..47f3c0f 100644 --- a/mali_kbase/mali_kbase_hwcnt_types.h +++ b/mali_kbase/mali_kbase_hwcnt_types.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** @@ -85,7 +104,6 @@ #include <linux/kernel.h> #include <linux/string.h> #include <linux/types.h> -#include "mali_malisw.h" /* Number of bytes in each bitfield */ #define KBASE_HWCNT_BITFIELD_BYTES (sizeof(u64)) diff --git a/mali_kbase/mali_kbase_hwcnt_virtualizer.c b/mali_kbase/mali_kbase_hwcnt_virtualizer.c index 917e47c..637ebf5 100644 --- a/mali_kbase/mali_kbase_hwcnt_virtualizer.c +++ b/mali_kbase/mali_kbase_hwcnt_virtualizer.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,9 +25,6 @@ #include "mali_kbase_hwcnt_accumulator.h" #include "mali_kbase_hwcnt_context.h" #include "mali_kbase_hwcnt_types.h" -#include "mali_malisw.h" -#include "mali_kbase_debug.h" -#include "mali_kbase_linux.h" #include <linux/mutex.h> #include <linux/slab.h> @@ -87,7 +85,6 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( return hvirt->metadata; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_metadata); /** * kbasep_hwcnt_virtualizer_client_free - Free a virtualizer client's memory. @@ -496,7 +493,6 @@ int kbase_hwcnt_virtualizer_client_set_counters( return errcode; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_set_counters); /** * kbasep_hwcnt_virtualizer_client_dump - Perform a dump of the client's @@ -686,7 +682,6 @@ int kbase_hwcnt_virtualizer_client_dump( return errcode; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_dump); int kbase_hwcnt_virtualizer_client_create( struct kbase_hwcnt_virtualizer *hvirt, @@ -719,7 +714,6 @@ int kbase_hwcnt_virtualizer_client_create( *out_hvcli = hvcli; return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_create); void kbase_hwcnt_virtualizer_client_destroy( struct kbase_hwcnt_virtualizer_client *hvcli) @@ -735,7 +729,6 @@ void kbase_hwcnt_virtualizer_client_destroy( kbasep_hwcnt_virtualizer_client_free(hvcli); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_destroy); int kbase_hwcnt_virtualizer_init( struct kbase_hwcnt_context *hctx, @@ -766,7 +759,6 @@ int kbase_hwcnt_virtualizer_init( *out_hvirt = virt; return 0; } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_init); void kbase_hwcnt_virtualizer_term( struct kbase_hwcnt_virtualizer *hvirt) @@ -787,4 +779,12 @@ void kbase_hwcnt_virtualizer_term( kfree(hvirt); } -KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_term); + +bool kbase_hwcnt_virtualizer_queue_work(struct kbase_hwcnt_virtualizer *hvirt, + struct work_struct *work) +{ + if (WARN_ON(!hvirt) || WARN_ON(!work)) + return false; + + return kbase_hwcnt_context_queue_work(hvirt->hctx, work); +} diff --git a/mali_kbase/mali_kbase_hwcnt_virtualizer.h b/mali_kbase/mali_kbase_hwcnt_virtualizer.h index 8f628c3..d143de2 100644 --- a/mali_kbase/mali_kbase_hwcnt_virtualizer.h +++ b/mali_kbase/mali_kbase_hwcnt_virtualizer.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** @@ -31,6 +50,7 @@ #define _KBASE_HWCNT_VIRTUALIZER_H_ #include <linux/types.h> +#include <linux/workqueue.h> struct kbase_hwcnt_context; struct kbase_hwcnt_virtualizer; @@ -142,4 +162,19 @@ int kbase_hwcnt_virtualizer_client_dump( u64 *ts_end_ns, struct kbase_hwcnt_dump_buffer *dump_buf); +/** + * kbase_hwcnt_virtualizer_queue_work() - Queue hardware counter related async + * work on a workqueue specialized for + * hardware counters. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @work: Non-NULL pointer to work to queue. + * + * Return: false if work was already on a queue, true otherwise. + * + * This is a convenience function that directly calls the underlying + * kbase_hwcnt_context's kbase_hwcnt_context_queue_work. + */ +bool kbase_hwcnt_virtualizer_queue_work(struct kbase_hwcnt_virtualizer *hvirt, + struct work_struct *work); + #endif /* _KBASE_HWCNT_VIRTUALIZER_H_ */ diff --git a/mali_kbase/mali_kbase_ioctl.h b/mali_kbase/mali_kbase_ioctl.h index 1180b3a..b4ee788 100644 --- a/mali_kbase/mali_kbase_ioctl.h +++ b/mali_kbase/mali_kbase_ioctl.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_IOCTL_H_ @@ -672,6 +691,18 @@ union kbase_ioctl_get_cpu_gpu_timeinfo { #define KBASE_IOCTL_GET_CPU_GPU_TIMEINFO \ _IOWR(KBASE_IOCTL_TYPE, 50, union kbase_ioctl_get_cpu_gpu_timeinfo) +/** + * struct kbase_ioctl_context_priority_check - Check the max possible priority + * @priority: Input priority & output priority + */ + +struct kbase_ioctl_context_priority_check { + __u8 priority; +}; + +#define KBASE_IOCTL_CONTEXT_PRIORITY_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 54, struct kbase_ioctl_context_priority_check) + /*************** * test ioctls * ***************/ @@ -826,11 +857,11 @@ struct kbase_ioctl_tlstream_stats { #define KBASE_GPUPROP_TEXTURE_FEATURES_3 80 #define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_3 81 -#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 +#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 #define KBASE_GPUPROP_RAW_THREAD_TLS_ALLOC 83 #define KBASE_GPUPROP_TLS_ALLOC 84 - +#define KBASE_GPUPROP_RAW_GPU_FEATURES 85 #ifdef __cpluscplus } #endif diff --git a/mali_kbase/mali_kbase_jd.c b/mali_kbase/mali_kbase_jd.c index a8cdf82..9407146 100644 --- a/mali_kbase/mali_kbase_jd.c +++ b/mali_kbase/mali_kbase_jd.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - #include <linux/dma-buf.h> #ifdef CONFIG_COMPAT #include <linux/compat.h> @@ -30,6 +29,7 @@ #include <linux/random.h> #include <linux/version.h> #include <linux/ratelimit.h> +#include <linux/priority_control_manager.h> #include <mali_kbase_jm.h> #include <mali_kbase_kinstr_jm.h> @@ -44,13 +44,9 @@ #define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) -/* random32 was renamed to prandom_u32 in 3.8 */ -#define prandom_u32 random32 -#endif - /* Return whether katom will run on the GPU or not. Currently only soft jobs and - * dependency-only atoms do not run on the GPU */ + * dependency-only atoms do not run on the GPU + */ #define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ BASE_JD_REQ_DEP))) @@ -243,7 +239,8 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st /* copy user buffer to the end of our real buffer. * Make sure the struct sizes haven't changed in a way - * we don't support */ + * we don't support + */ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); input_extres = (struct base_external_resource *) (((unsigned char *)katom->extres) + @@ -259,13 +256,14 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync) { - info.resv_objs = kmalloc_array(katom->nr_extres, -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) - sizeof(struct reservation_object *), + info.resv_objs = + kmalloc_array(katom->nr_extres, +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) + sizeof(struct reservation_object *), #else - sizeof(struct dma_resv *), + sizeof(struct dma_resv *), #endif - GFP_KERNEL); + GFP_KERNEL); if (!info.resv_objs) { err_ret_val = -ENOMEM; goto early_err_out; @@ -319,7 +317,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync && reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) struct reservation_object *resv; #else struct dma_resv *resv; @@ -336,7 +334,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st * at least not before the first write) as we overwrite elements * as we loop and could be overwriting ourself, so no writes * until the last read for an element. - * */ + */ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ katom->extres[res_no].alloc = alloc; } @@ -489,7 +487,8 @@ static inline void jd_resolve_dep(struct list_head *out_list, static bool is_dep_valid(struct kbase_jd_atom *katom) { /* If there's no dependency then this is 'valid' from the perspective of - * early dependency submission */ + * early dependency submission + */ if (!katom) return true; @@ -498,7 +497,8 @@ static bool is_dep_valid(struct kbase_jd_atom *katom) return false; /* If dependency has completed and has failed or will fail then it is - * not valid */ + * not valid + */ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && (katom->event_code != BASE_JD_EVENT_DONE || katom->will_fail_event_code)) @@ -735,7 +735,8 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ /* This is needed in case an atom is failed due to being invalid, this - * can happen *before* the jobs that the atom depends on have completed */ + * can happen *before* the jobs that the atom depends on have completed + */ for (i = 0; i < 2; i++) { if (kbase_jd_katom_dep_atom(&katom->dep[i])) { list_del(&katom->dep_item[i]); @@ -793,7 +794,8 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, !node->will_fail_event_code) { /* Node successfully submitted, try submitting * dependencies as they may now be representable - * in JS */ + * in JS + */ jd_try_submitting_deps(&runnable_jobs, node); } } @@ -809,10 +811,14 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, /* Decrement and check the TOTAL number of jobs. This includes * those not tracked by the scheduler: 'not ready to run' and - * 'dependency-only' jobs. */ + * 'dependency-only' jobs. + */ if (--kctx->jctx.job_nr == 0) - wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter - * that we've got no more jobs (so we can be safely terminated) */ + /* All events are safely queued now, and we can signal + * any waiter that we've got no more jobs (so we can be + * safely terminated) + */ + wake_up(&kctx->jctx.zero_jobs_wait); } return need_to_try_schedule_context; @@ -907,10 +913,11 @@ static bool jd_submit_atom(struct kbase_context *const kctx, dev_dbg(kbdev->dev, "User did JD submit atom %p\n", (void *)katom); /* Update the TOTAL number of jobs. This includes those not tracked by - * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + * the scheduler: 'not ready to run' and 'dependency-only' jobs. + */ jctx->job_nr++; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE katom->start_timestamp.tv64 = 0; #else katom->start_timestamp = 0; @@ -983,7 +990,8 @@ static bool jd_submit_atom(struct kbase_context *const kctx, /* Wrong dependency setup. Atom will be sent * back to user space. Do not record any - * dependencies. */ + * dependencies. + */ jd_trace_atom_submit(kctx, katom, NULL); return jd_done_nolock(katom, NULL); @@ -1069,7 +1077,10 @@ static bool jd_submit_atom(struct kbase_context *const kctx, sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; - katom->sched_priority = sched_prio; + + /* Cap the priority to jctx.max_priority */ + katom->sched_priority = (sched_prio < kctx->jctx.max_priority) ? + kctx->jctx.max_priority : sched_prio; /* Create a new atom. */ jd_trace_atom_submit(kctx, katom, &katom->sched_priority); @@ -1535,7 +1546,9 @@ void kbase_jd_done_worker(struct work_struct *data) mutex_unlock(&jctx->lock); /* Job is now no longer running, so can now safely release the context - * reference, and handle any actions that were logged against the atom's retained state */ + * reference, and handle any actions that were logged against the + * atom's retained state + */ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); @@ -1543,7 +1556,8 @@ void kbase_jd_done_worker(struct work_struct *data) if (!atomic_dec_return(&kctx->work_count)) { /* If worker now idle then post all events that jd_done_nolock() - * has queued */ + * has queued + */ mutex_lock(&jctx->lock); while (!list_empty(&kctx->completed_jobs)) { struct kbase_jd_atom *atom = list_entry( @@ -1618,7 +1632,8 @@ static void jd_cancel_worker(struct work_struct *data) need_to_try_schedule_context = jd_done_nolock(katom, NULL); /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to * schedule the context. There's also no need for the jsctx_mutex to have been taken - * around this too. */ + * around this too. + */ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); /* katom may have been freed now, do not use! */ @@ -1687,10 +1702,10 @@ void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { struct kbase_context *kctx; - KBASE_DEBUG_ASSERT(NULL != kbdev); - KBASE_DEBUG_ASSERT(NULL != katom); + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); kctx = katom->kctx; - KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(kctx != NULL); dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom); KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); @@ -1761,12 +1776,15 @@ int kbase_jd_init(struct kbase_context *kctx) { int i; int mali_err = 0; + struct priority_control_manager_device *pcm_device = NULL; KBASE_DEBUG_ASSERT(kctx); + pcm_device = kctx->kbdev->pcm_dev; + kctx->jctx.max_priority = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", WQ_HIGHPRI | WQ_UNBOUND, 1); - if (NULL == kctx->jctx.job_done_wq) { + if (kctx->jctx.job_done_wq == NULL) { mali_err = -ENOMEM; goto out1; } @@ -1802,6 +1820,11 @@ int kbase_jd_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->completed_jobs); atomic_set(&kctx->work_count, 0); + /* Check if there are platform rules for maximum priority */ + if (pcm_device) + kctx->jctx.max_priority = pcm_device->ops.pcm_scheduler_priority_check( + pcm_device, current, KBASE_JS_ATOM_SCHED_PRIO_REALTIME); + return 0; out1: diff --git a/mali_kbase/mali_kbase_jd_debugfs.c b/mali_kbase/mali_kbase_jd_debugfs.c index 5dd3f00..7b2dade 100644 --- a/mali_kbase/mali_kbase_jd_debugfs.c +++ b/mali_kbase/mali_kbase_jd_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -66,42 +67,40 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, struct kbase_fence_cb *cb; if (atom->dma_fence.fence) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence = atom->dma_fence.fence; #else struct dma_fence *fence = atom->dma_fence.fence; #endif seq_printf(sfile, -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) - "Sd(%u#%u: %s) ", +#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) + "Sd(%u#%u: %s) ", #else - "Sd(%llu#%u: %s) ", + "Sd(%llu#%u: %s) ", #endif - fence->context, - fence->seqno, - dma_fence_is_signaled(fence) ? - "signaled" : "active"); + fence->context, fence->seqno, + dma_fence_is_signaled(fence) ? "signaled" : + "active"); } list_for_each_entry(cb, &atom->dma_fence.callbacks, node) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence = cb->fence; #else struct dma_fence *fence = cb->fence; #endif seq_printf(sfile, -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) - "Wd(%u#%u: %s) ", +#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) + "Wd(%u#%u: %s) ", #else - "Wd(%llu#%u: %s) ", + "Wd(%llu#%u: %s) ", #endif - fence->context, - fence->seqno, - dma_fence_is_signaled(fence) ? - "signaled" : "active"); + fence->context, fence->seqno, + dma_fence_is_signaled(fence) ? "signaled" : + "active"); } } #endif /* CONFIG_MALI_DMA_FENCE */ @@ -180,7 +179,8 @@ static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) /* start_timestamp is cleared as soon as the atom leaves UNUSED state * and set before a job is submitted to the h/w, a non-zero value means - * it is valid */ + * it is valid + */ if (ktime_to_ns(atom->start_timestamp)) start_timestamp = ktime_to_ns( ktime_sub(ktime_get(), atom->start_timestamp)); diff --git a/mali_kbase/mali_kbase_jd_debugfs.h b/mali_kbase/mali_kbase_jd_debugfs.h index 697bdef..c3cee2e 100644 --- a/mali_kbase/mali_kbase_jd_debugfs.h +++ b/mali_kbase/mali_kbase_jd_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_jm.c b/mali_kbase/mali_kbase_jm.c index 16ae320..d5e64a1 100644 --- a/mali_kbase/mali_kbase_jm.c +++ b/mali_kbase/mali_kbase_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. @@ -20,7 +21,6 @@ * */ - /* * HW access job manager common APIs */ diff --git a/mali_kbase/mali_kbase_jm.h b/mali_kbase/mali_kbase_jm.h index 132db41..697a3ff 100644 --- a/mali_kbase/mali_kbase_jm.h +++ b/mali_kbase/mali_kbase_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2013-2014, 2016, 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2013-2014, 2016, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/mali_kbase_js.c b/mali_kbase/mali_kbase_js.c index 0b0c5bf..0bde5e2 100644 --- a/mali_kbase/mali_kbase_js.c +++ b/mali_kbase/mali_kbase_js.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - /* * Job Scheduler Implementation */ @@ -37,6 +36,7 @@ #include "mali_kbase_jm.h" #include "mali_kbase_hwaccess_jm.h" +#include <linux/priority_control_manager.h> /* * Private types @@ -45,26 +45,30 @@ /* Bitpattern indicating the result of releasing a context */ enum { /* The context was descheduled - caller should try scheduling in a new - * one to keep the runpool full */ + * one to keep the runpool full + */ KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), /* Ctx attributes were changed - caller should try scheduling all - * contexts */ + * contexts + */ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) }; typedef u32 kbasep_js_release_result; const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { - KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ - KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ - KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ + KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ + KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ + KBASE_JS_ATOM_SCHED_PRIO_LOW, /* BASE_JD_PRIO_LOW */ + KBASE_JS_ATOM_SCHED_PRIO_REALTIME /* BASE_JD_PRIO_REALTIME */ }; const base_jd_prio kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { - BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ - BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ - BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ + BASE_JD_PRIO_REALTIME, /* KBASE_JS_ATOM_SCHED_PRIO_REALTIME */ + BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ + BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ + BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ }; @@ -184,7 +188,7 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) return false; @@ -277,7 +281,7 @@ jsctx_queue_foreach(struct kbase_context *kctx, int js, { int prio; - for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) jsctx_queue_foreach_prio(kctx, js, prio, callback); } @@ -319,7 +323,7 @@ jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) * @js: Job slot id to check. * * Check the ring buffers for all priorities, starting from - * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a + * KBASE_JS_ATOM_SCHED_PRIO_REALTIME, for the specified @js and @prio and return a * pointer to the next atom, unless all the priority's ring buffers are empty. * * Caller must hold the hwaccess_lock. @@ -333,7 +337,7 @@ jsctx_rb_peek(struct kbase_context *kctx, int js) lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { struct kbase_jd_atom *katom; @@ -448,7 +452,8 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) #ifdef CONFIG_MALI_DEBUG /* Soft-stop will be disabled on a single context by default unless - * softstop_always is set */ + * softstop_always is set + */ jsdd->softstop_always = false; #endif /* CONFIG_MALI_DEBUG */ jsdd->nr_all_contexts_running = 0; @@ -531,7 +536,8 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) kbdev->gpu_props.props.raw_props.js_features[i]); /* On error, we could continue on: providing none of the below resources - * rely on the ones above */ + * rely on the ones above + */ mutex_init(&jsdd->runpool_mutex); mutex_init(&jsdd->queue_mutex); @@ -595,11 +601,13 @@ int kbasep_js_kctx_init(struct kbase_context *const kctx) sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); /* Initially, the context is disabled from submission until the create - * flags are set */ + * flags are set + */ kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); /* On error, we could continue on: providing none of the below resources - * rely on the ones above */ + * rely on the ones above + */ mutex_init(&js_kctx_info->ctx.jsctx_mutex); init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); @@ -1020,7 +1028,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, } /* Dependencies with different priorities can't - be represented in the ringbuffer */ + * be represented in the ringbuffer + */ if (prio != dep_prio) { dev_dbg(kbdev->dev, "Different atom priorities\n"); @@ -1030,7 +1039,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, if (js == dep_js) { /* Only one same-slot dependency can be - * represented in the ringbuffer */ + * represented in the ringbuffer + */ if (has_dep) { dev_dbg(kbdev->dev, "Too many same-slot deps\n"); @@ -1038,7 +1048,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, break; } /* Each dependee atom can only have one - * same-slot dependency */ + * same-slot dependency + */ if (dep_atom->post_dep) { dev_dbg(kbdev->dev, "Too many same-slot successors\n"); @@ -1048,7 +1059,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, has_dep = true; } else { /* Only one cross-slot dependency can be - * represented in the ringbuffer */ + * represented in the ringbuffer + */ if (has_x_dep) { dev_dbg(kbdev->dev, "Too many cross-slot deps\n"); @@ -1056,7 +1068,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, break; } /* Each dependee atom can only have one - * cross-slot dependency */ + * cross-slot dependency + */ if (dep_atom->x_post_dep) { dev_dbg(kbdev->dev, "Too many cross-slot successors\n"); @@ -1064,7 +1077,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, break; } /* The dependee atom can not already be in the - * HW access ringbuffer */ + * HW access ringbuffer + */ if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { dev_dbg(kbdev->dev, @@ -1074,7 +1088,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, break; } /* The dependee atom can not already have - * completed */ + * completed + */ if (dep_atom->status != KBASE_JD_ATOM_STATE_IN_JS) { dev_dbg(kbdev->dev, @@ -1092,7 +1107,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, } /* If dependencies can be represented by ringbuffer then clear them from - * atom structure */ + * atom structure + */ if (ret) { for (i = 0; i < 2; i++) { struct kbase_jd_atom *dep_atom = katom->dep[i].atom; @@ -1181,7 +1197,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) /* Determine the new priority for context, as per the priority * of currently in-use atoms. */ - for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { if (kctx->atoms_count[prio]) { new_priority = prio; @@ -1192,6 +1208,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) kbase_js_set_ctx_priority(kctx, new_priority); } +KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); /** * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler @@ -1349,7 +1366,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx, (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); /* Setting atom status back to queued as it still has unresolved - * dependencies */ + * dependencies + */ atom->status = KBASE_JD_ATOM_STATE_QUEUED; dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)atom); @@ -1389,7 +1407,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx, kbdev, kctx, atom->slot_nr); } /* If this context is active and the atom is the first on its slot, - * kick the job manager to attempt to fast-start the atom */ + * kick the job manager to attempt to fast-start the atom + */ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx[atom->slot_nr]) kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); @@ -1404,17 +1423,20 @@ bool kbasep_js_add_job(struct kbase_context *kctx, if (kbase_ctx_flag(kctx, KCTX_DYING)) { /* A job got added while/after kbase_job_zap_context() * was called on a non-scheduled context. Kill that job - * by killing the context. */ + * by killing the context. + */ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); } else if (js_kctx_info->ctx.nr_jobs == 1) { /* Handle Refcount going from 0 to 1: schedule the - * context on the Queue */ + * context on the Queue + */ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); - /* Queue was updated - caller must try to - * schedule the head context */ + /* Queue was updated - caller must try to schedule the + * head context + */ WARN_ON(!enqueue_required); } } @@ -1478,7 +1500,8 @@ bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, * * This is because it returns false for soft-stopped atoms, but we * want to override that, because we're cancelling an atom regardless of - * whether it was soft-stopped or not */ + * whether it was soft-stopped or not + */ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, &katom_retained_state); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -1525,7 +1548,8 @@ static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { /* A change in runpool ctx attributes might mean we can - * run more jobs than before */ + * run more jobs than before + */ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, @@ -1624,7 +1648,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( /* Make a set of checks to see if the context should be scheduled out. * Note that there'll always be at least 1 reference to the context - * which was previously acquired by kbasep_js_schedule_ctx(). */ + * which was previously acquired by kbasep_js_schedule_ctx(). + */ if (new_ref_count == 1 && (!kbasep_js_is_submit_allowed(js_devdata, kctx) || #ifdef CONFIG_MALI_ARBITER_SUPPORT @@ -1635,7 +1660,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( int slot; /* Last reference, and we've been told to remove this context - * from the Run Pool */ + * from the Run Pool + */ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, kbasep_js_is_submit_allowed(js_devdata, kctx)); @@ -1662,7 +1688,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); /* Releasing the context and katom retained state can allow - * more jobs to run */ + * more jobs to run + */ release_result |= kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, @@ -1702,7 +1729,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); /* Signal any waiter that the context is not scheduled, so is * safe for termination - once the jsctx_mutex is also dropped, - * and jobs have finished. */ + * and jobs have finished. + */ wake_up(&js_kctx_info->ctx.is_scheduled_wait); /* Queue an action to occur after we've dropped the lock */ @@ -1744,7 +1772,8 @@ void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, if (kbase_ctx_flag(kctx, KCTX_DYING)) { /* Dying: don't requeue, but kill all jobs on the context. This - * happens asynchronously */ + * happens asynchronously + */ dev_dbg(kbdev->dev, "JS: ** Killing Context %p on RunPool Remove **", kctx); kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); @@ -1798,7 +1827,8 @@ void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, } /* Variant of kbasep_js_runpool_release_ctx() that doesn't call into - * kbase_js_sched_all() */ + * kbase_js_sched_all() + */ static void kbasep_js_runpool_release_ctx_no_schedule( struct kbase_device *kbdev, struct kbase_context *kctx) { @@ -1867,7 +1897,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, kbdev, kctx); if (as_nr != KBASEP_AS_NR_INVALID) { /* Attempt to retain the context again, this should - * succeed */ + * succeed + */ mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); as_nr = kbase_ctx_sched_retain_ctx(kctx); @@ -1926,7 +1957,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); /* Cause any future waiter-on-termination to wait until the context is - * descheduled */ + * descheduled + */ wake_up(&js_kctx_info->ctx.is_scheduled_wait); /* Re-check for suspending: a suspend could've occurred, and all the @@ -1939,7 +1971,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, * was taken (i.e. this condition doesn't execute), then the * kbasep_js_suspend() code will cleanup this context instead (by virtue * of it being called strictly after the suspend flag is set, and will - * wait for this lock to drop) */ + * wait for this lock to drop) + */ #ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { #else @@ -1967,13 +2000,15 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, mutex_unlock(&js_devdata->runpool_mutex); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); /* Note: after this point, the context could potentially get scheduled - * out immediately */ + * out immediately + */ if (kctx_suspended) { /* Finishing forcing out the context due to a suspend. Use a * variant of kbasep_js_runpool_release_ctx() that doesn't * schedule a new context, to prevent a risk of recursion back - * into this function */ + * into this function + */ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); return false; } @@ -2059,7 +2094,8 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, kbase_js_sync_timers(kbdev); /* Fast-starting requires the jsctx_mutex to be dropped, - * because it works on multiple ctxs */ + * because it works on multiple ctxs + */ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); @@ -2071,7 +2107,8 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); } else { /* Already scheduled in - We need to retain it to keep the - * corresponding address space */ + * corresponding address space + */ WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); @@ -2116,7 +2153,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) js_devdata->runpool_irq.submit_allowed = 0; /* Retain each of the contexts, so we can cause it to leave even if it - * had no refcount to begin with */ + * had no refcount to begin with + */ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { struct kbase_context *kctx = kbdev->as_to_kctx[i]; @@ -2137,7 +2175,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* De-ref the previous retain to ensure each context gets pulled out - * sometime later. */ + * sometime later. + */ for (i = 0; i < BASE_MAX_NR_AS; ++i, retained = retained >> 1) { @@ -2148,7 +2187,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) } /* Caller must wait for all Power Manager active references to be - * dropped */ + * dropped + */ } void kbasep_js_resume(struct kbase_device *kbdev) @@ -2162,7 +2202,7 @@ void kbasep_js_resume(struct kbase_device *kbdev) mutex_lock(&js_devdata->queue_mutex); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { struct kbase_context *kctx, *n; unsigned long flags; @@ -2283,7 +2323,8 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, lockdep_assert_held(&kctx->jctx.lock); /* If slot will transition from unpullable to pullable then add to - * pullable list */ + * pullable list + */ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { enqueue_required = true; } else { @@ -2472,7 +2513,8 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) /* Due to ordering restrictions when unpulling atoms on failure, we do * not allow multiple runs of fail-dep atoms from the same context to be - * present on the same slot */ + * present on the same slot + */ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { struct kbase_jd_atom *prev_atom = kbase_backend_inspect_tail(kbdev, js); @@ -2771,13 +2813,15 @@ static void js_return_worker(struct work_struct *data) timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); /* If this slot has been blocked due to soft-stopped atoms, and all - * atoms have now been processed, then unblock the slot */ + * atoms have now been processed, then unblock the slot + */ if (!kctx->atoms_pulled_slot_pri[js][prio] && kctx->blocked_js[js][prio]) { kctx->blocked_js[js][prio] = false; /* Only mark the slot as pullable if the context is not idle - - * that case is handled below */ + * that case is handled below + */ if (atomic_read(&kctx->atoms_pulled) && kbase_js_ctx_pullable(kctx, js, true)) timer_sync |= kbase_js_ctx_list_add_pullable_nolock( @@ -3091,7 +3135,8 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, } /* If this slot has been blocked due to soft-stopped atoms, and - * all atoms have now been processed, then unblock the slot */ + * all atoms have now been processed, then unblock the slot + */ if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] && kctx->blocked_js[atom_slot][prio]) { dev_dbg(kbdev->dev, @@ -3412,7 +3457,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) dev_dbg(kbdev->dev, "Suspend pending (s:%d)\n", js); /* Suspend pending - return context to - * queue and stop scheduling */ + * queue and stop scheduling + */ mutex_lock( &kctx->jctx.sched_info.ctx.jsctx_mutex); if (kbase_js_ctx_list_add_pullable_head( @@ -3483,7 +3529,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) /* Failed to pull jobs - push to head of list. * Unless this context is already 'active', in * which case it's effectively already scheduled - * so push it to the back of the list. */ + * so push it to the back of the list. + */ if (pullable && kctx == last_active[js] && kbase_ctx_flag(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << @@ -3508,7 +3555,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) * slot, then we need to remove the active * marker to prevent it from submitting atoms in * the IRQ handler, which would prevent this - * context from making progress. */ + * context from making progress. + */ if (last_active[js] && kctx != last_active[js] && kbase_js_ctx_pullable( last_active[js], js, true)) @@ -3580,7 +3628,8 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* First, atomically do the following: * - mark the context as dying - * - try to evict it from the queue */ + * - try to evict it from the queue + */ mutex_lock(&kctx->jctx.lock); mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); @@ -3655,8 +3704,9 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* Only cancel jobs when we evicted from the * queue. No Power Manager active reference was held. * - * Having is_dying set ensures that this kills, and - * doesn't requeue */ + * Having is_dying set ensures that this kills, and doesn't + * requeue + */ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); @@ -3667,7 +3717,8 @@ void kbase_js_zap_context(struct kbase_context *kctx) bool was_retained; /* Case c: didn't evict, but it is scheduled - it's in the Run - * Pool */ + * Pool + */ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); @@ -3678,18 +3729,21 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* Retain and (later) release the context whilst it is is now * disallowed from submitting jobs - ensures that someone - * somewhere will be removing the context later on */ + * somewhere will be removing the context later on + */ was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); /* Since it's scheduled and we have the jsctx_mutex, it must be - * retained successfully */ + * retained successfully + */ KBASE_DEBUG_ASSERT(was_retained); dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); /* Cancel any remaining running jobs for this kctx - if any. * Submit is disallowed which takes effect immediately, so no - * more new jobs will appear after we do this. */ + * more new jobs will appear after we do this. + */ kbase_backend_jm_kill_running_jobs_from_kctx(kctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -3711,7 +3765,8 @@ void kbase_js_zap_context(struct kbase_context *kctx) * to be destroyed, and the context to be de-scheduled (if it was on the * runpool). * - * kbase_jd_zap_context() will do this. */ + * kbase_jd_zap_context() will do this. + */ } static inline int trace_get_refcnt(struct kbase_device *kbdev, @@ -3758,3 +3813,18 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } + +base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority) +{ + struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; + int req_priority, out_priority; + base_jd_prio out_jd_priority = priority; + + if (pcm_device) { + req_priority = kbasep_js_atom_prio_to_sched_prio(priority); + out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, req_priority); + out_jd_priority = kbasep_js_sched_prio_to_atom_prio(out_priority); + } + return out_jd_priority; +} + diff --git a/mali_kbase/mali_kbase_js.h b/mali_kbase/mali_kbase_js.h index 541acd4..3e9719a 100644 --- a/mali_kbase/mali_kbase_js.h +++ b/mali_kbase/mali_kbase_js.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_js.h * Job Scheduler APIs. diff --git a/mali_kbase/mali_kbase_js_ctx_attr.c b/mali_kbase/mali_kbase_js_ctx_attr.c index 141d04a..67c4952 100644 --- a/mali_kbase/mali_kbase_js_ctx_attr.c +++ b/mali_kbase/mali_kbase_js_ctx_attr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. @@ -20,7 +21,6 @@ * */ - #include <mali_kbase.h> #include <mali_kbase_config.h> @@ -211,7 +211,8 @@ void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kb /* We don't need to know about state changed, because retaining a * context occurs on scheduling it, and that itself will also try - * to run new atoms */ + * to run new atoms + */ CSTD_UNUSED(runpool_state_changed); } } @@ -251,9 +252,9 @@ void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); } - /* We don't need to know about state changed, because retaining an - * atom occurs on adding it, and that itself will also try to run - * new atoms */ + /* We don't need to know about state changed, because retaining an atom + * occurs on adding it, and that itself will also try to run new atoms + */ CSTD_UNUSED(runpool_state_changed); } diff --git a/mali_kbase/mali_kbase_js_ctx_attr.h b/mali_kbase/mali_kbase_js_ctx_attr.h index 25fd397..fa8731a 100644 --- a/mali_kbase/mali_kbase_js_ctx_attr.h +++ b/mali_kbase/mali_kbase_js_ctx_attr.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2015, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_js_ctx_attr.h * Job Scheduler Context Attribute APIs diff --git a/mali_kbase/mali_kbase_kinstr_jm.c b/mali_kbase/mali_kbase_kinstr_jm.c index 1e91a7c..98b82f2 100644 --- a/mali_kbase/mali_kbase_kinstr_jm.c +++ b/mali_kbase/mali_kbase_kinstr_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -70,13 +71,7 @@ typedef unsigned int __poll_t; /* Allows us to perform ASM goto for the tracing * https://www.kernel.org/doc/Documentation/static-keys.txt */ -#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); -#else -struct static_key basep_kinstr_jm_reader_static_key = STATIC_KEY_INIT_FALSE; -#define static_branch_inc(key) static_key_slow_inc(key) -#define static_branch_dec(key) static_key_slow_dec(key) -#endif /* KERNEL_VERSION(4 ,3, 0) <= LINUX_VERSION_CODE */ #define KBASE_KINSTR_JM_VERSION 1 diff --git a/mali_kbase/mali_kbase_kinstr_jm.h b/mali_kbase/mali_kbase_kinstr_jm.h index 555edfe..a276361 100644 --- a/mali_kbase/mali_kbase_kinstr_jm.h +++ b/mali_kbase/mali_kbase_kinstr_jm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019,2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -127,14 +146,7 @@ void kbasep_kinstr_jm_atom_state( * shouldn't be changed externally, but if you do, make sure you use * a static_key_inc()/static_key_dec() pair. */ -#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE extern struct static_key_false basep_kinstr_jm_reader_static_key; -#else -/* Pre-4.3 kernels have a different API for static keys, but work - * mostly the same with less type safety. */ -extern struct static_key basep_kinstr_jm_reader_static_key; -#define static_branch_unlikely(key) static_key_false(key) -#endif /* KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE */ /** * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state diff --git a/mali_kbase/mali_kbase_kinstr_jm_reader.h b/mali_kbase/mali_kbase_kinstr_jm_reader.h index e267e6b..1d2c141 100644 --- a/mali_kbase/mali_kbase_kinstr_jm_reader.h +++ b/mali_kbase/mali_kbase_kinstr_jm_reader.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/mali_kbase_linux.h b/mali_kbase/mali_kbase_linux.h index 003ac9e..eb4ce77 100644 --- a/mali_kbase/mali_kbase_linux.h +++ b/mali_kbase/mali_kbase_linux.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2014, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_linux.h * Base kernel APIs, Linux implementation. diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c index 7ec6094..c054205 100644 --- a/mali_kbase/mali_kbase_mem.c +++ b/mali_kbase/mali_kbase_mem.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -99,7 +100,8 @@ static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) } /* This function finds out which RB tree the given pfn from the GPU VA belongs - * to based on the memory zone the pfn refers to */ + * to based on the memory zone the pfn refers to + */ static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, u64 gpu_pfn) { @@ -229,7 +231,7 @@ struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; struct rb_root *rbtree = NULL; - KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(kctx != NULL); lockdep_assert_held(&kctx->reg_lock); @@ -289,7 +291,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( struct rb_root *rbtree = NULL; /* Note that this search is a linear search, as we do not have a target - address in mind, so does not benefit from the rbtree search */ + * address in mind, so does not benefit from the rbtree search + */ rbtree = reg_reqs->rbtree; for (rbnode = rb_first(rbtree); rbnode; rbnode = rb_next(rbnode)) { @@ -304,7 +307,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( * (start_pfn + align_mask) & ~(align_mask) * * Otherwise, it aligns to n*align + offset, for the - * lowest value n that makes this still >start_pfn */ + * lowest value n that makes this still >start_pfn + */ start_pfn += align_mask; start_pfn -= (start_pfn - align_offset) & (align_mask); @@ -368,8 +372,9 @@ int kbase_remove_va_region(struct kbase_va_region *reg) if (rbprev) { prev = rb_entry(rbprev, struct kbase_va_region, rblink); if (prev->flags & KBASE_REG_FREE) { - /* We're compatible with the previous VMA, - * merge with it */ + /* We're compatible with the previous VMA, merge with + * it + */ WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != (reg->flags & KBASE_REG_ZONE_MASK)); prev->nr_pages += reg->nr_pages; @@ -512,8 +517,8 @@ int kbase_add_va_region(struct kbase_context *kctx, int gpu_pc_bits = kbdev->gpu_props.props.core_props.log2_program_counter_size; - KBASE_DEBUG_ASSERT(NULL != kctx); - KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(reg != NULL); lockdep_assert_held(&kctx->reg_lock); @@ -1293,8 +1298,8 @@ int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 else attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); - KBASE_DEBUG_ASSERT(NULL != kctx); - KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(reg != NULL); err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); if (err) @@ -1320,7 +1325,9 @@ int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 if (err) goto bad_insert; - kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); + /* Note: mapping count is tracked at alias + * creation time + */ } else { err = kbase_mmu_insert_single_page(kctx, reg->start_pfn + i * stride, @@ -1379,13 +1386,6 @@ bad_insert: reg->start_pfn, reg->nr_pages, kctx->as_nr); - if (alloc->type == KBASE_MEM_TYPE_ALIAS) { - KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); - while (i--) - if (alloc->imported.alias.aliased[i].alloc) - kbase_mem_phy_alloc_gpu_unmapped(alloc->imported.alias.aliased[i].alloc); - } - kbase_remove_va_region(reg); return err; @@ -1399,7 +1399,6 @@ static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) { int err = 0; - size_t i; if (reg->start_pfn == 0) return 0; @@ -1424,10 +1423,9 @@ int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) /* Update tracking, and other cleanup, depending on memory type. */ switch (reg->gpu_alloc->type) { case KBASE_MEM_TYPE_ALIAS: - KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); - for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) - if (reg->gpu_alloc->imported.alias.aliased[i].alloc) - kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + /* We mark the source allocs as unmapped from the GPU when + * putting reg's allocs + */ break; case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { struct kbase_alloc_import_user_buf *user_buf = @@ -1736,8 +1734,8 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re { int err; - KBASE_DEBUG_ASSERT(NULL != kctx); - KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(reg != NULL); dev_dbg(kctx->kbdev->dev, "%s %p in kctx %p\n", __func__, (void *)reg, (void *)kctx); lockdep_assert_held(&kctx->reg_lock); @@ -1803,7 +1801,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) return -EINVAL; } - if (0 == gpu_addr) { + if (gpu_addr == 0) { dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); return -EINVAL; } @@ -1856,7 +1854,7 @@ KBASE_EXPORT_TEST_API(kbase_mem_free); int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region *reg, unsigned long flags) { - KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(reg != NULL); KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); @@ -1988,7 +1986,8 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, &kctx->kbdev->memdev.used_pages); /* Increase mm counters before we allocate pages so that this - * allocation is visible to the OOM killer */ + * allocation is visible to the OOM killer + */ kbase_process_page_usage_inc(kctx, nr_pages_requested); tp = alloc->pages + alloc->nents; @@ -2392,7 +2391,7 @@ int kbase_free_phy_pages_helper( } /* early out if nothing to do */ - if (0 == nr_pages_to_free) + if (nr_pages_to_free == 0) return 0; start_free = alloc->pages + alloc->nents - nr_pages_to_free; @@ -2640,8 +2639,10 @@ void kbase_mem_kref_free(struct kref *kref) aliased = alloc->imported.alias.aliased; if (aliased) { for (i = 0; i < alloc->imported.alias.nents; i++) - if (aliased[i].alloc) + if (aliased[i].alloc) { + kbase_mem_phy_alloc_gpu_unmapped(aliased[i].alloc); kbase_mem_phy_alloc_put(aliased[i].alloc); + } vfree(aliased); } break; @@ -2692,7 +2693,7 @@ KBASE_EXPORT_TEST_API(kbase_mem_kref_free); int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) { - KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(reg != NULL); KBASE_DEBUG_ASSERT(vsize > 0); /* validate user provided arguments */ @@ -2705,7 +2706,7 @@ int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) goto out_term; - KBASE_DEBUG_ASSERT(0 != vsize); + KBASE_DEBUG_ASSERT(vsize != 0); if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) goto out_term; @@ -2777,7 +2778,8 @@ bool kbase_check_alloc_flags(unsigned long flags) #endif /* !MALI_USE_CSF */ /* GPU should have at least read or write access otherwise there is no - reason for allocating. */ + * reason for allocating. + */ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) return false; @@ -2785,14 +2787,15 @@ bool kbase_check_alloc_flags(unsigned long flags) if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) return false; - /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported - * memory */ + /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported memory + */ if ((flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) == BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) return false; /* Should not combine BASE_MEM_COHERENT_LOCAL with - * BASE_MEM_COHERENT_SYSTEM */ + * BASE_MEM_COHERENT_SYSTEM + */ if ((flags & (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) == (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) return false; @@ -2825,7 +2828,8 @@ bool kbase_check_import_flags(unsigned long flags) #endif /* !MALI_USE_CSF */ /* GPU should have at least read or write access otherwise there is no - reason for importing. */ + * reason for importing. + */ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) return false; @@ -2849,7 +2853,7 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, #define KBASE_MSG_PRE "GPU allocation attempted with " - if (0 == va_pages) { + if (va_pages == 0) { dev_warn(dev, KBASE_MSG_PRE "0 va_pages!"); return -EINVAL; } @@ -2861,7 +2865,8 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, } /* Note: commit_pages is checked against va_pages during - * kbase_alloc_phy_pages() */ + * kbase_alloc_phy_pages() + */ /* Limit GPU executable allocs to GPU PC size */ if ((flags & BASE_MEM_PROT_GPU_EX) && (va_pages > gpu_pc_pages_max)) { @@ -2916,7 +2921,8 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, return -EINVAL; } /* For use with is_power_of_2, which takes unsigned long, so - * must ensure e.g. on 32-bit kernel it'll fit in that type */ + * must ensure e.g. on 32-bit kernel it'll fit in that type + */ small_extension = (unsigned long)large_extension; if (!is_power_of_2(small_extension)) { @@ -4309,7 +4315,7 @@ int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, if (WARN_ON(reg->gpu_alloc->imported.user_buf.mm != current->mm)) return -EINVAL; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE pinned_pages = get_user_pages(NULL, mm, address, alloc->imported.user_buf.nr_pages, @@ -4321,19 +4327,19 @@ KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); #endif -#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +#elif KERNEL_VERSION(4, 9, 0) > LINUX_VERSION_CODE pinned_pages = get_user_pages_remote(NULL, mm, address, alloc->imported.user_buf.nr_pages, reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +#elif KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE pinned_pages = get_user_pages_remote(NULL, mm, address, alloc->imported.user_buf.nr_pages, reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, pages, NULL); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) +#elif KERNEL_VERSION(5, 9, 0) > LINUX_VERSION_CODE pinned_pages = get_user_pages_remote(NULL, mm, address, alloc->imported.user_buf.nr_pages, @@ -4525,7 +4531,8 @@ struct kbase_mem_phy_alloc *kbase_map_external_resource( goto exit; reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; - if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { + if (reg->gpu_alloc->imported.user_buf + .current_mapping_usage_count == 1) { err = kbase_jd_user_buf_map(kctx, reg); if (err) { reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; @@ -4560,7 +4567,7 @@ void kbase_unmap_external_resource(struct kbase_context *kctx, case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { alloc->imported.user_buf.current_mapping_usage_count--; - if (0 == alloc->imported.user_buf.current_mapping_usage_count) { + if (alloc->imported.user_buf.current_mapping_usage_count == 0) { bool writeable = true; if (!kbase_is_region_invalid_or_free(reg) && diff --git a/mali_kbase/mali_kbase_mem.h b/mali_kbase/mali_kbase_mem.h index 7a5cc66..973fb7a 100644 --- a/mali_kbase/mali_kbase_mem.h +++ b/mali_kbase/mali_kbase_mem.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_mem.h * Base kernel memory APIs @@ -48,10 +65,13 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, /* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ -/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. -The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and -page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table -updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ +/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by + * 8 pages. The MMU reads in 8 page table entries from memory at a time, if we + * have more than one page fault within the same 8 pages and page tables are + * updated accordingly, the MMU does not re-read the page table entries from + * memory for the subsequent page table updates and generates duplicate page + * faults as the page table information used by the MMU is not valid. + */ #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ @@ -81,7 +101,8 @@ enum kbase_memory_type { }; /* internal structure, mirroring base_mem_aliasing_info, - * but with alloc instead of a gpu va (handle) */ + * but with alloc instead of a gpu va (handle) + */ struct kbase_aliased { struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ u64 offset; /* in pages */ @@ -105,7 +126,9 @@ struct kbase_aliased { * updated as part of the change. * * @kref: number of users of this alloc - * @gpu_mappings: count number of times mapped on the GPU + * @gpu_mappings: count number of times mapped on the GPU. Indicates the number + * of references there are to the physical pages from different + * GPU VA regions. * @nents: 0..N * @pages: N elements, only 0..nents are valid * @mappings: List of CPU mappings of this physical memory allocation. @@ -211,7 +234,7 @@ static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc * KBASE_DEBUG_ASSERT(alloc); /* we only track mappings of NATIVE buffers */ if (alloc->type == KBASE_MEM_TYPE_NATIVE) - if (0 > atomic_dec_return(&alloc->gpu_mappings)) { + if (atomic_dec_return(&alloc->gpu_mappings) < 0) { pr_err("Mismatched %s:\n", __func__); dump_stack(); } @@ -342,7 +365,8 @@ struct kbase_va_region { #if !MALI_USE_CSF /* The top of the initial commit is aligned to extension pages. - * Extent must be a power of 2 */ + * Extent must be a power of 2 + */ #define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) #else /* Bit 23 is reserved. @@ -1400,7 +1424,8 @@ static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the * private field stays the same. So we have to be clever and * use the fact that we only store DMA addresses of whole pages, - * so the low bits should be zero */ + * so the low bits should be zero + */ KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); set_page_private(p, dma_addr >> PAGE_SHIFT); } else { diff --git a/mali_kbase/mali_kbase_mem_linux.c b/mali_kbase/mali_kbase_mem_linux.c index f6d386f..553b8fd 100644 --- a/mali_kbase/mali_kbase_mem_linux.c +++ b/mali_kbase/mali_kbase_mem_linux.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /** * @file mali_kbase_mem_linux.c * Base kernel memory APIs, Linux implementation. @@ -35,10 +34,9 @@ #include <linux/fs.h> #include <linux/version.h> #include <linux/dma-mapping.h> -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ - (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) +#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) #include <linux/dma-attrs.h> -#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ +#endif /* LINUX_VERSION_CODE < 4.8.0 */ #include <linux/dma-buf.h> #include <linux/shrinker.h> #include <linux/cache.h> @@ -51,6 +49,7 @@ #include <mmu/mali_kbase_mmu.h> #include <mali_kbase_caps.h> #include <mali_kbase_trace_gpu_mem.h> +#include <mali_kbase_reset_gpu.h> #if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) @@ -416,7 +415,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, if (*flags & BASE_MEM_GROW_ON_GPF) { /* kbase_check_alloc_sizes() already checks extension is valid for - * assigning to reg->extension */ + * assigning to reg->extension + */ reg->extension = extension; #if !MALI_USE_CSF } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { @@ -726,35 +726,18 @@ out_unlock: return freed; } -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, - struct shrink_control *sc) -{ - if (sc->nr_to_scan == 0) - return kbase_mem_evictable_reclaim_count_objects(s, sc); - - return kbase_mem_evictable_reclaim_scan_objects(s, sc); -} -#endif - int kbase_mem_evictable_init(struct kbase_context *kctx) { INIT_LIST_HEAD(&kctx->evict_list); mutex_init(&kctx->jit_evict_lock); - /* Register shrinker */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) - kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; -#else kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; -#endif kctx->reclaim.seeks = DEFAULT_SEEKS; /* Kernel versions prior to 3.1 : - * struct shrinker does not define batch */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + * struct shrinker does not define batch + */ kctx->reclaim.batch = 0; -#endif register_shrinker(&kctx->reclaim); return 0; } @@ -932,7 +915,12 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; if (prev_needed != new_needed) { - /* Aliased allocations can't be made ephemeral */ + /* Aliased allocations can't be shrunk as the code doesn't + * support looking up: + * - all physical pages assigned to different GPU VAs + * - CPU mappings for the physical pages at different vm_pgoff + * (==GPU VA) locations. + */ if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) goto out_unlock; @@ -1633,21 +1621,21 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( down_read(kbase_mem_get_process_mmap_lock()); -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE faulted_pages = get_user_pages(current, current->mm, address, *va_pages, #if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE - reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + reg->flags & KBASE_REG_CPU_WR ? FOLL_WRITE : 0, pages, NULL); #else - reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); + reg->flags & KBASE_REG_CPU_WR, 0, pages, NULL); #endif -#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +#elif KERNEL_VERSION(4, 9, 0) > LINUX_VERSION_CODE faulted_pages = get_user_pages(address, *va_pages, - reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); + reg->flags & KBASE_REG_CPU_WR, 0, pages, NULL); #else faulted_pages = get_user_pages(address, *va_pages, - reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + reg->flags & KBASE_REG_CPU_WR ? FOLL_WRITE : 0, pages, NULL); #endif @@ -1759,7 +1747,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, #ifdef CONFIG_64BIT if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { /* 64-bit tasks must MMAP anyway, but not expose this address to - * clients */ + * clients + */ *flags |= BASE_MEM_NEED_MMAP; reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, *num_pages, @@ -1805,7 +1794,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, goto bad_handle; /* must be > 0 */ if (ai[i].length > stride) goto bad_handle; /* can't be larger than the - stride */ + * stride + */ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; } else { struct kbase_va_region *aliasing_reg; @@ -1820,6 +1810,15 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, goto bad_handle; /* Not found/already free */ if (aliasing_reg->flags & KBASE_REG_DONT_NEED) goto bad_handle; /* Ephemeral region */ + if (aliasing_reg->flags & KBASE_REG_NO_USER_FREE) + goto bad_handle; /* JIT regions can't be + * aliased. NO_USER_FREE flag + * covers the entire lifetime + * of JIT regions. The other + * types of regions covered + * by this flag also shall + * not be aliased. + */ if (!(aliasing_reg->flags & KBASE_REG_GPU_CACHED)) goto bad_handle; /* GPU uncached memory */ if (!aliasing_reg->gpu_alloc) @@ -1827,16 +1826,18 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) goto bad_handle; /* Not a native alloc */ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) - goto bad_handle; - /* Non-coherent memory cannot alias - coherent memory, and vice versa.*/ + goto bad_handle; /* Non-coherent memory cannot + * alias coherent memory, and + * vice versa. + */ /* check size against stride */ if (!ai[i].length) goto bad_handle; /* must be > 0 */ if (ai[i].length > stride) goto bad_handle; /* can't be larger than the - stride */ + * stride + */ alloc = aliasing_reg->gpu_alloc; @@ -1849,6 +1850,18 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; + + /* Ensure the underlying alloc is marked as being + * mapped at >1 different GPU VA immediately, even + * though mapping might not happen until later. + * + * Otherwise, we would (incorrectly) allow shrinking of + * the source region (aliasing_reg) and so freeing the + * physical pages (without freeing the entire alloc) + * whilst we still hold an implicit reference on those + * physical pages. + */ + kbase_mem_phy_alloc_gpu_mapped(alloc); } } @@ -1892,6 +1905,10 @@ no_cookie: #endif no_mmap: bad_handle: + /* Marking the source allocs as not being mapped on the GPU and putting + * them is handled by putting reg's allocs, so no rollback of those + * actions is done here. + */ kbase_gpu_vm_unlock(kctx); no_aliased_array: invalid_flags: @@ -2145,7 +2162,15 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) if (new_pages > reg->nr_pages) goto out_unlock; - /* can't be mapped more than once on the GPU */ + /* Can't shrink when physical pages are mapped to different GPU + * VAs. The code doesn't support looking up: + * - all physical pages assigned to different GPU VAs + * - CPU mappings for the physical pages at different vm_pgoff + * (==GPU VA) locations. + * + * Note that for Native allocs mapped at multiple GPU VAs, growth of + * such allocs is not a supported use-case. + */ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) goto out_unlock; /* can't grow regions which are ephemeral */ @@ -2447,11 +2472,7 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, * See MIDBASE-1057 */ -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; -#else - vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -#endif vma->vm_ops = &kbase_vm_ops; vma->vm_private_data = map; @@ -2632,7 +2653,8 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, /* incorrect mmap size */ /* leave the cookie for a potential later * mapping, or to be reclaimed later when the - * context is freed */ + * context is freed + */ err = -ENOMEM; goto out; } @@ -2698,7 +2720,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, if (!(vma->vm_flags & VM_WRITE)) vma->vm_flags &= ~VM_MAYWRITE; - if (0 == nr_pages) { + if (nr_pages == 0) { err = -EINVAL; goto out; } @@ -2719,7 +2741,8 @@ int kbase_context_mmap(struct kbase_context *const kctx, /* if not the MTP, verify that the MTP has been mapped */ rcu_read_lock(); /* catches both when the special page isn't present or - * when we've forked */ + * when we've forked + */ if (rcu_dereference(kctx->process_mm) != current->mm) { err = -EINVAL; rcu_read_unlock(); @@ -2736,7 +2759,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): /* MMU dump */ err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); - if (0 != err) + if (err != 0) goto out_unlock; /* free the region on munmap */ free_on_close = 1; @@ -2759,7 +2782,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, &aligned_offset); - if (0 != err) + if (err != 0) goto out_unlock; /* free the region on munmap */ free_on_close = 1; @@ -2832,7 +2855,8 @@ int kbase_context_mmap(struct kbase_context *const kctx, if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { /* MMU dump - userspace should now have a reference on - * the pages, so we can now free the kernel mapping */ + * the pages, so we can now free the kernel mapping + */ vfree(kaddr); /* CPU mapping of GPU allocations have GPU VA as the vm_pgoff * and that is used to shrink the mapping when the commit size @@ -2940,7 +2964,8 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, /* Note: enforcing a RO prot_request onto prot is not done, since: * - CPU-arch-specific integration required - * - kbase_vmap() requires no access checks to be made/enforced */ + * - kbase_vmap() requires no access checks to be made/enforced + */ cpu_addr = vmap(pages, page_count, VM_MAP, prot); @@ -3017,7 +3042,8 @@ void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, * be made. * * As mentioned in kbase_vmap_prot() this means that a kernel-side - * CPU-RO mapping is not enforced to allow this to work */ + * CPU-RO mapping is not enforced to allow this to work + */ return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); } KBASE_EXPORT_TEST_API(kbase_vmap); @@ -3049,7 +3075,7 @@ KBASE_EXPORT_TEST_API(kbase_vunmap); static void kbasep_add_mm_counter(struct mm_struct *mm, int member, long value) { -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)) +#if (KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE) /* To avoid the build breakage due to an unexported kernel symbol * 'mm_trace_rss_stat' from later kernels, i.e. from V4.19.0 onwards, * we inline here the equivalent of 'add_mm_counter()' from linux @@ -3133,11 +3159,7 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ /* no real access */ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; -#else - vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -#endif vma->vm_ops = &kbase_vm_special_ops; vma->vm_private_data = kctx; @@ -3172,16 +3194,32 @@ static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) { struct kbase_queue *queue = vma->vm_private_data; struct kbase_context *kctx; + struct kbase_device *kbdev; + int err; + bool reset_prevented = false; if (WARN_ON(!queue)) return; kctx = queue->kctx; + kbdev = kctx->kbdev; + + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) + dev_warn( + kbdev->dev, + "Unsuccessful GPU reset detected when unbinding queue (csi_index=%d), attempting to unbind regardless", + queue->csi_index); + else + reset_prevented = true; mutex_lock(&kctx->csf.lock); kbase_csf_queue_unbind(queue); mutex_unlock(&kctx->csf.lock); + if (reset_prevented) + kbase_reset_gpu_allow(kbdev); + /* Now as the vma is closed, drop the reference on mali device file */ fput(kctx->filp); } @@ -3316,11 +3354,7 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, if (err) goto map_failed; -#if (KERNEL_VERSION(3, 7, 0) <= LINUX_VERSION_CODE) vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; -#else - vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -#endif /* TODO use VM_MIXEDMAP, since it is more appropriate as both types of * memory with and without "struct page" backing are being inserted here. * Hw Doorbell pages comes from the device register area so kernel does @@ -3343,7 +3377,13 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, return 0; map_failed: - kbase_csf_queue_unbind(queue); + /* The queue cannot have got to KBASE_CSF_QUEUE_BOUND state if we + * reached here, so safe to use a variant of unbind that only works on + * stopped queues + * + * This is so we don't enter the CSF scheduler from this path. + */ + kbase_csf_queue_unbind_stopped(queue); return err; } diff --git a/mali_kbase/mali_kbase_mem_linux.h b/mali_kbase/mali_kbase_mem_linux.h index c80d885..c96498f 100644 --- a/mali_kbase/mali_kbase_mem_linux.h +++ b/mali_kbase/mali_kbase_mem_linux.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010, 2012-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010, 2012-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_mem_linux.h * Base kernel memory APIs, Linux implementation. @@ -468,11 +485,11 @@ static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, */ static inline struct rw_semaphore *kbase_mem_get_process_mmap_lock(void) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) +#if KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE return ¤t->mm->mmap_sem; -#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */ +#else /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */ return ¤t->mm->mmap_lock; -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */ +#endif /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */ } #endif /* _KBASE_MEM_LINUX_H_ */ diff --git a/mali_kbase/mali_kbase_mem_lowlevel.h b/mali_kbase/mali_kbase_mem_lowlevel.h index 7011603..96a0b66 100644 --- a/mali_kbase/mali_kbase_mem_lowlevel.h +++ b/mali_kbase/mali_kbase_mem_lowlevel.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2014,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2014, 2016-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _KBASE_MEM_LOWLEVEL_H #define _KBASE_MEM_LOWLEVEL_H diff --git a/mali_kbase/mali_kbase_mem_pool.c b/mali_kbase/mali_kbase_mem_pool.c index 0723e32..c075727 100644 --- a/mali_kbase/mali_kbase_mem_pool.c +++ b/mali_kbase/mali_kbase_mem_pool.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -154,20 +155,12 @@ static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) { struct page *p; - gfp_t gfp; + gfp_t gfp = GFP_HIGHUSER | __GFP_ZERO; struct kbase_device *const kbdev = pool->kbdev; struct device *const dev = kbdev->dev; dma_addr_t dma_addr; int i; -#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ - LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) - /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ - gfp = GFP_USER | __GFP_ZERO; -#else - gfp = GFP_HIGHUSER | __GFP_ZERO; -#endif - /* don't warn on higher order failures */ if (pool->order) gfp |= __GFP_NOWARN; @@ -364,17 +357,6 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, return freed; } -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, - struct shrink_control *sc) -{ - if (sc->nr_to_scan == 0) - return kbase_mem_pool_reclaim_count_objects(s, sc); - - return kbase_mem_pool_reclaim_scan_objects(s, sc); -} -#endif - int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config, unsigned int order, @@ -398,19 +380,13 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, spin_lock_init(&pool->pool_lock); INIT_LIST_HEAD(&pool->page_list); - /* Register shrinker */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) - pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; -#else pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; -#endif pool->reclaim.seeks = DEFAULT_SEEKS; /* Kernel versions prior to 3.1 : - * struct shrinker does not define batch */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + * struct shrinker does not define batch + */ pool->reclaim.batch = 0; -#endif register_shrinker(&pool->reclaim); pool_dbg(pool, "initialized\n"); diff --git a/mali_kbase/mali_kbase_mem_pool_debugfs.c b/mali_kbase/mali_kbase_mem_pool_debugfs.c index 5879fdf..3ed0d1a 100644 --- a/mali_kbase/mali_kbase_mem_pool_debugfs.c +++ b/mali_kbase/mali_kbase_mem_pool_debugfs.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mem_pool_debugfs.h b/mali_kbase/mali_kbase_mem_pool_debugfs.h index 2932945..32dd843 100644 --- a/mali_kbase/mali_kbase_mem_pool_debugfs.h +++ b/mali_kbase/mali_kbase_mem_pool_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_MEM_POOL_DEBUGFS_H_ diff --git a/mali_kbase/mali_kbase_mem_pool_group.c b/mali_kbase/mali_kbase_mem_pool_group.c index aa25548..5d6269e 100644 --- a/mali_kbase/mali_kbase_mem_pool_group.c +++ b/mali_kbase/mali_kbase_mem_pool_group.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_mem_pool_group.h b/mali_kbase/mali_kbase_mem_pool_group.h index 0484f59..4b50155 100644 --- a/mali_kbase/mali_kbase_mem_pool_group.h +++ b/mali_kbase/mali_kbase_mem_pool_group.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_MEM_POOL_GROUP_H_ diff --git a/mali_kbase/mali_kbase_mem_profile_debugfs.c b/mali_kbase/mali_kbase_mem_profile_debugfs.c index 85723f8..72b4633 100644 --- a/mali_kbase/mali_kbase_mem_profile_debugfs.c +++ b/mali_kbase/mali_kbase_mem_profile_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2012-2017, 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_mem_profile_debugfs.h b/mali_kbase/mali_kbase_mem_profile_debugfs.h index 1462247..04e593f 100644 --- a/mali_kbase/mali_kbase_mem_profile_debugfs.h +++ b/mali_kbase/mali_kbase_mem_profile_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_mem_profile_debugfs.h * Header file for mem profiles entries in debugfs diff --git a/mali_kbase/mali_kbase_mem_profile_debugfs_buf_size.h b/mali_kbase/mali_kbase_mem_profile_debugfs_buf_size.h index d55cc85..9f530ca 100644 --- a/mali_kbase/mali_kbase_mem_profile_debugfs_buf_size.h +++ b/mali_kbase/mali_kbase_mem_profile_debugfs_buf_size.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_mipe_gen_header.h b/mali_kbase/mali_kbase_mipe_gen_header.h index 72acadf..669f843 100644 --- a/mali_kbase/mali_kbase_mipe_gen_header.h +++ b/mali_kbase/mali_kbase_mipe_gen_header.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. @@ -59,6 +78,17 @@ #endif /** + * A compiler attribute for packing structures + * + * e.g. __packed + * + * Default value is __attribute__((__packed__)) + */ +#if !defined(MIPE_HEADER_PACKED_ATTRIBUTE) +#define MIPE_HEADER_PACKED_ATTRIBUTE __attribute__((__packed__)) +#endif + +/** * MIPE stream id. * * See enum tl_stream_id. @@ -151,7 +181,7 @@ const struct char _arg_types[sizeof(arg_types)]; \ u32 _size_arg_names; \ char _arg_names[sizeof(arg_names)]; \ - } __attribute__ ((__packed__)) __ ## name; + } MIPE_HEADER_PACKED_ATTRIBUTE __ ## name; #define ENUM_DESC(arg_name, value) \ struct { \ @@ -161,13 +191,13 @@ const struct u32 _value; \ u32 _value_str_len; \ char _value_str[sizeof(#value)]; \ - } __attribute__ ((__packed__)) __ ## arg_name ## _ ## value; + } MIPE_HEADER_PACKED_ATTRIBUTE __ ## arg_name ## _ ## value; MIPE_HEADER_TRACEPOINT_LIST MIPE_HEADER_ENUM_LIST #undef TRACEPOINT_DESC #undef ENUM_DESC -} __attribute__((packed)) MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { +} MIPE_HEADER_PACKED_ATTRIBUTE MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { ._mipe_w0 = MIPE_PACKET_HEADER_W0( TL_PACKET_FAMILY_TL, MIPE_HEADER_PKT_CLASS, diff --git a/mali_kbase/mali_kbase_mipe_proto.h b/mali_kbase/mali_kbase_mipe_proto.h index 54667cf..5202bf6 100644 --- a/mali_kbase/mali_kbase_mipe_proto.h +++ b/mali_kbase/mali_kbase_mipe_proto.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. diff --git a/mali_kbase/mali_kbase_native_mgm.c b/mali_kbase/mali_kbase_native_mgm.c index 38ae46e..a3417a1 100644 --- a/mali_kbase/mali_kbase_native_mgm.c +++ b/mali_kbase/mali_kbase_native_mgm.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_native_mgm.h b/mali_kbase/mali_kbase_native_mgm.h index 431b1f4..4d43ea4 100644 --- a/mali_kbase/mali_kbase_native_mgm.h +++ b/mali_kbase/mali_kbase_native_mgm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_NATIVE_MGM_H_ diff --git a/mali_kbase/mali_kbase_platform_fake.c b/mali_kbase/mali_kbase_platform_fake.c index fbb090e..c3f85b5 100644 --- a/mali_kbase/mali_kbase_platform_fake.c +++ b/mali_kbase/mali_kbase_platform_fake.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2014, 2016-2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_pm.c b/mali_kbase/mali_kbase_pm.c index 630ab15..0edf4b6 100644 --- a/mali_kbase/mali_kbase_pm.c +++ b/mali_kbase/mali_kbase_pm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /** * @file mali_kbase_pm.c * Base kernel power management APIs @@ -191,7 +190,8 @@ void kbase_pm_driver_suspend(struct kbase_device *kbdev) #if !MALI_USE_CSF /* Suspend job scheduler and associated components, so that it releases all - * the PM active count references */ + * the PM active count references + */ kbasep_js_suspend(kbdev); #else kbase_csf_scheduler_pm_suspend(kbdev); diff --git a/mali_kbase/mali_kbase_pm.h b/mali_kbase/mali_kbase_pm.h index 1356518..8cc6625 100644 --- a/mali_kbase/mali_kbase_pm.h +++ b/mali_kbase/mali_kbase_pm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_kbase_pm.h * Power management API definitions diff --git a/mali_kbase/mali_kbase_regs_history_debugfs.c b/mali_kbase/mali_kbase_regs_history_debugfs.c index 640db95..1cc7b29 100644 --- a/mali_kbase/mali_kbase_regs_history_debugfs.c +++ b/mali_kbase/mali_kbase_regs_history_debugfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014, 2016, 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_regs_history_debugfs.h b/mali_kbase/mali_kbase_regs_history_debugfs.h index b043bd6..0d5ce9b 100644 --- a/mali_kbase/mali_kbase_regs_history_debugfs.h +++ b/mali_kbase/mali_kbase_regs_history_debugfs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2016, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mali_kbase_reset_gpu.h b/mali_kbase/mali_kbase_reset_gpu.h index 61bbb0b..a1b75bf 100644 --- a/mali_kbase/mali_kbase_reset_gpu.h +++ b/mali_kbase/mali_kbase_reset_gpu.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,12 +18,151 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_RESET_GPU_H_ #define _KBASE_RESET_GPU_H_ /** + * kbase_reset_gpu_prevent_and_wait - Prevent GPU resets from starting whilst + * the current thread is accessing the GPU, + * and wait for any in-flight reset to + * finish. + * @kbdev: Device pointer + * + * This should be used when a potential access to the HW is going to be made + * from a non-atomic context. + * + * It will wait for any in-flight reset to finish before returning. Hence, + * correct lock ordering must be observed with respect to the calling thread + * and the reset worker thread. + * + * This does not synchronize general access to the HW, and so multiple threads + * can prevent GPU reset concurrently, whilst not being serialized. This is + * advantageous as the threads can make this call at points where they do not + * know for sure yet whether they will indeed access the GPU (for example, to + * respect lock ordering), without unnecessarily blocking others. + * + * Threads must still use other synchronization to ensure they access the HW + * consistently, at a point where they are certain it needs to be accessed. + * + * On success, ensure that when access to the GPU by the caller thread has + * finished, that it calls kbase_reset_gpu_allow() again to allow resets to + * happen. + * + * This may return a failure in cases such as a previous failure to reset the + * GPU within a reasonable time. If that happens, the GPU might be + * non-operational and the caller should not attempt any further access. + * + * Note: + * For atomic context, instead check kbase_reset_gpu_is_active(). + * + * Return: 0 on success, or negative error code on failure. + */ +int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_try_prevent - Attempt to prevent GPU resets from starting + * whilst the current thread is accessing the + * GPU, unless a reset is already in progress. + * @kbdev: Device pointer + * + * Similar to kbase_reset_gpu_prevent_and_wait(), but it does not wait for an + * existing reset to complete. This can be used on codepaths that the Reset + * worker waits on, where use of kbase_reset_gpu_prevent_and_wait() would + * otherwise deadlock. + * + * Instead, a reset that is currently happening will cause this function to + * return an error code indicating that, and further resets will not have been + * prevented. + * + * In such cases, the caller must check for -EAGAIN, and take similar actions + * as for handling reset in atomic context. That is, they must cancel any + * actions that depended on reset being prevented, possibly deferring them + * until after the reset. + * + * Otherwise a successful return means that the caller can continue its actions + * safely in the knowledge that reset is prevented, and the reset worker will + * correctly wait instead of deadlocking against this thread. + * + * On success, ensure that when access to the GPU by the caller thread has + * finished, that it calls kbase_reset_gpu_allow() again to allow resets to + * happen. + * + * Refer to kbase_reset_gpu_prevent_and_wait() for more information. + * + * Return: 0 on success. -EAGAIN if a reset is currently happening. Other + * negative error codes on failure. + */ +int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_allow - Allow GPU resets to happen again after having been + * previously prevented. + * @kbdev: Device pointer + * + * This should be used when a potential access to the HW has finished from a + * non-atomic context. + * + * It must be used from the same thread that originally made a previously call + * to kbase_reset_gpu_prevent_and_wait(). It must not be deferred to another + * thread. + */ +void kbase_reset_gpu_allow(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_assert_prevented - Make debugging checks that GPU reset is + * currently prevented by the current + * thread. + * @kbdev: Device pointer + * + * Make debugging checks that the current thread has made a call to + * kbase_reset_gpu_prevent_and_wait(), but has yet to make a subsequent call to + * kbase_reset_gpu_allow(). + * + * CONFIG_LOCKDEP is required to prove that reset is indeed + * prevented. Otherwise only limited debugging checks can be made. + */ +void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_assert_failed_or_prevented - Make debugging checks that + * either GPU reset previously + * failed, or is currently + * prevented. + * + * @kbdev: Device pointer + * + * As with kbase_reset_gpu_assert_prevented(), but also allow for paths where + * reset was not prevented due to a failure, yet we still need to execute the + * cleanup code following. + * + * Cleanup code following this call must handle any inconsistent state modified + * by the failed GPU reset, and must timeout any blocking operations instead of + * waiting forever. + */ +void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev); + +/** * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. * @kbdev: Device pointer * @@ -95,8 +234,13 @@ int kbase_reset_gpu_silent(struct kbase_device *kbdev); * kbase_reset_gpu_is_active - Reports if the GPU is being reset * @kbdev: Device pointer * - * Return: True if the GPU is in the process of being reset (or if the reset of - * GPU failed, not applicable to Job Manager GPUs). + * Any changes made to the HW when this returns true may be lost, overwritten + * or corrupted. + * + * Note that unless appropriate locks are held when using this function, the + * state could change immediately afterwards. + * + * Return: True if the GPU is in the process of being reset. */ bool kbase_reset_gpu_is_active(struct kbase_device *kbdev); diff --git a/mali_kbase/mali_kbase_smc.c b/mali_kbase/mali_kbase_smc.c index b5c7b12..e1b884d 100644 --- a/mali_kbase/mali_kbase_smc.c +++ b/mali_kbase/mali_kbase_smc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2015, 2018, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_kbase_smc.h b/mali_kbase/mali_kbase_smc.h index 221eb21..5c384e4 100644 --- a/mali_kbase/mali_kbase_smc.h +++ b/mali_kbase/mali_kbase_smc.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _KBASE_SMC_H_ #define _KBASE_SMC_H_ diff --git a/mali_kbase/mali_kbase_softjobs.c b/mali_kbase/mali_kbase_softjobs.c index 0dc8c03..77e5b91 100644 --- a/mali_kbase/mali_kbase_softjobs.c +++ b/mali_kbase/mali_kbase_softjobs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - #include <mali_kbase.h> #include <linux/dma-buf.h> @@ -148,7 +147,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) /* Take the PM active reference as late as possible - otherwise, it could * delay suspend until we process the atom (which may be at the end of a - * long chain of dependencies */ + * long chain of dependencies + */ pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); if (pm_active_err) { struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; @@ -183,7 +183,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) /* GPU_WR access is checked on the range for returning the result to * userspace for the following reasons: * - security, this is currently how imported user bufs are checked. - * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ + * - userspace ddk guaranteed to assume region was mapped as GPU_WR + */ user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); if (!user_result) return 0; @@ -715,7 +716,8 @@ out_unlock: out_cleanup: /* Frees allocated memory for kbase_debug_copy_job struct, including - * members, and sets jc to 0 */ + * members, and sets jc to 0 + */ kbase_debug_copy_finish(katom); kfree(user_buffers); @@ -723,7 +725,7 @@ out_cleanup: } #endif /* !MALI_USE_CSF */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, unsigned long page_num, struct page **page) { @@ -804,16 +806,16 @@ int kbase_mem_copy_from_extres(struct kbase_context *kctx, dma_to_copy = min(dma_buf->size, (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); ret = dma_buf_begin_cpu_access(dma_buf, -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) - 0, dma_to_copy, +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, #endif - DMA_FROM_DEVICE); + DMA_FROM_DEVICE); if (ret) goto out_unlock; for (i = 0; i < dma_to_copy/PAGE_SIZE && target_page_nr < buf_data->nr_pages; i++) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE struct page *pg; void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); #else @@ -825,7 +827,7 @@ int kbase_mem_copy_from_extres(struct kbase_context *kctx, buf_data->nr_pages, &target_page_nr, offset); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE kunmap(pg); #else dma_buf_kunmap(dma_buf, i, extres_page); @@ -835,10 +837,10 @@ int kbase_mem_copy_from_extres(struct kbase_context *kctx, } } dma_buf_end_cpu_access(dma_buf, -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) - 0, dma_to_copy, +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, #endif - DMA_FROM_DEVICE); + DMA_FROM_DEVICE); break; } default: @@ -926,11 +928,6 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, #if !MALI_USE_CSF -#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE) -#define offsetofend(TYPE, MEMBER) \ - (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) -#endif - /* * Sizes of user data to copy for each just-in-time memory interface version * @@ -1654,7 +1651,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) struct base_fence fence; int fd; - if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + if (copy_from_user(&fence, + (__user void *)(uintptr_t)katom->jc, + sizeof(fence)) != 0) return -EINVAL; fd = kbase_sync_fence_out_create(katom, @@ -1663,7 +1662,8 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) return -EINVAL; fence.basep.fd = fd; - if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { + if (copy_to_user((__user void *)(uintptr_t)katom->jc, + &fence, sizeof(fence)) != 0) { kbase_sync_fence_out_remove(katom); kbase_sync_fence_close_fd(fd); fence.basep.fd = -EINVAL; @@ -1676,7 +1676,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) struct base_fence fence; int ret; - if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + if (copy_from_user(&fence, + (__user void *)(uintptr_t)katom->jc, + sizeof(fence)) != 0) return -EINVAL; /* Get a reference to the fence object */ diff --git a/mali_kbase/mali_kbase_strings.c b/mali_kbase/mali_kbase_strings.c index 22caa4a..5413dae 100644 --- a/mali_kbase/mali_kbase_strings.c +++ b/mali_kbase/mali_kbase_strings.c @@ -1,6 +1,7 @@ - /* +// SPDX-License-Identifier: GPL-2.0 +/* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -19,6 +20,7 @@ * SPDX-License-Identifier: GPL-2.0 * */ + #include "mali_kbase_strings.h" #define KBASE_DRV_NAME "mali" diff --git a/mali_kbase/mali_kbase_strings.h b/mali_kbase/mali_kbase_strings.h index d2f1825..56974a2 100644 --- a/mali_kbase/mali_kbase_strings.h +++ b/mali_kbase/mali_kbase_strings.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ extern const char kbase_drv_name[]; diff --git a/mali_kbase/mali_kbase_sync.h b/mali_kbase/mali_kbase_sync.h index d1921b7..f6426d8 100644 --- a/mali_kbase/mali_kbase_sync.h +++ b/mali_kbase/mali_kbase_sync.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** @@ -165,7 +184,7 @@ void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); */ static inline void kbase_sync_fence_close_fd(int fd) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) +#if KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE ksys_close(fd); #else sys_close(fd); diff --git a/mali_kbase/mali_kbase_sync_android.c b/mali_kbase/mali_kbase_sync_android.c index 75940fb..48945f1 100644 --- a/mali_kbase/mali_kbase_sync_android.c +++ b/mali_kbase/mali_kbase_sync_android.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,15 +51,6 @@ struct mali_sync_pt { int result; }; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -/* For backwards compatibility with kernels before 3.17. After 3.17 - * sync_pt_parent is included in the kernel. */ -static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) -{ - return pt->parent; -} -#endif - static struct mali_sync_timeline *to_mali_sync_timeline( struct sync_timeline *timeline) { @@ -196,6 +188,7 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) return 0; } +#if !MALI_USE_CSF /* Allocates a sync point within the timeline. * * The timeline must be the one allocated by kbase_sync_timeline_alloc @@ -225,10 +218,6 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) struct sync_timeline *tl; struct sync_pt *pt; struct sync_fence *fence; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) - struct files_struct *files; - struct fdtable *fdt; -#endif int fd; struct file *tl_file; @@ -259,29 +248,11 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) /* from here the fence owns the sync_pt */ /* create a fd representing the fence */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); if (fd < 0) { sync_fence_put(fence); goto out; } -#else - fd = get_unused_fd(); - if (fd < 0) { - sync_fence_put(fence); - goto out; - } - - files = current->files; - spin_lock(&files->file_lock); - fdt = files_fdtable(files); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) - __set_close_on_exec(fd, fdt); -#else - FD_SET(fd, fdt->close_on_exec); -#endif - spin_unlock(&files->file_lock); -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ /* bind fence to the new fd */ sync_fence_install(fence, fd); @@ -289,7 +260,8 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) katom->fence = sync_fence_fdget(fd); if (katom->fence == NULL) { /* The only way the fence can be NULL is if userspace closed it - * for us, so we don't need to clear it up */ + * for us, so we don't need to clear it up + */ fd = -EINVAL; goto out; } @@ -305,6 +277,7 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) katom->fence = sync_fence_fdget(fd); return katom->fence ? 0 : -ENOENT; } +#endif /* !MALI_USE_CSF */ int kbase_sync_fence_validate(int fd) { @@ -318,6 +291,7 @@ int kbase_sync_fence_validate(int fd) return 0; } +#if !MALI_USE_CSF /* Returns true if the specified timeline is allocated by Mali */ static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) { @@ -376,22 +350,14 @@ kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) if (!katom->fence) return BASE_JD_EVENT_JOB_CANCELLED; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - if (!list_is_singular(&katom->fence->pt_list_head)) { -#else if (katom->fence->num_fences != 1) { -#endif /* Not exactly one item in the list - so it didn't (directly) - * come from us */ + * come from us + */ return BASE_JD_EVENT_JOB_CANCELLED; } -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - pt = list_first_entry(&katom->fence->pt_list_head, - struct sync_pt, pt_list); -#else pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); -#endif timeline = sync_pt_parent(pt); if (!kbase_sync_timeline_is_ours(timeline)) { @@ -413,11 +379,7 @@ static inline int kbase_fence_get_status(struct sync_fence *fence) if (!fence) return -ENOENT; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - return fence->status; -#else return atomic_read(&fence->status); -#endif } static void kbase_fence_wait_callback(struct sync_fence *fence, @@ -461,7 +423,8 @@ int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) if (ret < 0) { katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; /* We should cause the dependent jobs in the bag to be failed, - * to do this we schedule the work queue to complete this job */ + * to do this we schedule the work queue to complete this job + */ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); queue_work(katom->kctx->jctx.job_done_wq, &katom->work); } @@ -473,7 +436,8 @@ void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) { if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { /* The wait wasn't cancelled - leave the cleanup for - * kbase_fence_wait_callback */ + * kbase_fence_wait_callback + */ return; } @@ -540,3 +504,4 @@ void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) sync_fence_wait(katom->fence, 1); } #endif +#endif /* !MALI_USE_CSF */ diff --git a/mali_kbase/mali_kbase_sync_common.c b/mali_kbase/mali_kbase_sync_common.c index 866894b..d509769 100644 --- a/mali_kbase/mali_kbase_sync_common.c +++ b/mali_kbase/mali_kbase_sync_common.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2016, 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/mali_kbase_sync_file.c b/mali_kbase/mali_kbase_sync_file.c index 5020692..2be6699 100644 --- a/mali_kbase/mali_kbase_sync_file.c +++ b/mali_kbase/mali_kbase_sync_file.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. @@ -62,7 +63,7 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) #if !MALI_USE_CSF int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; @@ -107,7 +108,7 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence = sync_file_get_fence(fd); #else struct dma_fence *fence = sync_file_get_fence(fd); @@ -124,7 +125,7 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) int kbase_sync_fence_validate(int fd) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence = sync_file_get_fence(fd); #else struct dma_fence *fence = sync_file_get_fence(fd); @@ -160,7 +161,7 @@ kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) static void kbase_fence_wait_callback(struct fence *fence, struct fence_cb *cb) #else @@ -203,7 +204,7 @@ static void kbase_fence_wait_callback(struct dma_fence *fence, int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) { int err; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; @@ -236,8 +237,8 @@ int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; /* We should cause the dependent jobs in the bag to be failed, - * to do this we schedule the work queue to complete this job */ - + * to do this we schedule the work queue to complete this job + */ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); queue_work(katom->kctx->jctx.job_done_wq, &katom->work); } @@ -249,7 +250,8 @@ void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) { if (!kbase_fence_free_callbacks(katom)) { /* The wait wasn't cancelled - - * leave the cleanup for kbase_fence_wait_callback */ + * leave the cleanup for kbase_fence_wait_callback + */ return; } @@ -325,7 +327,7 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; @@ -345,7 +347,7 @@ int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) { -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) struct fence *fence; #else struct dma_fence *fence; diff --git a/mali_kbase/mali_kbase_trace_gpu_mem.c b/mali_kbase/mali_kbase_trace_gpu_mem.c index d0e9f0b..5cbe432 100644 --- a/mali_kbase/mali_kbase_trace_gpu_mem.c +++ b/mali_kbase/mali_kbase_trace_gpu_mem.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. @@ -220,8 +221,3 @@ void kbase_add_dma_buf_usage(struct kbase_context *kctx, mutex_unlock(&kbdev->dma_buf_lock); } - -#if !defined(CONFIG_TRACE_GPU_MEM) && !MALI_CUSTOMER_RELEASE -#define CREATE_TRACE_POINTS -#include "mali_gpu_mem_trace.h" -#endif diff --git a/mali_kbase/mali_kbase_trace_gpu_mem.h b/mali_kbase/mali_kbase_trace_gpu_mem.h index 7e95956..5b25139 100644 --- a/mali_kbase/mali_kbase_trace_gpu_mem.h +++ b/mali_kbase/mali_kbase_trace_gpu_mem.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_TRACE_GPU_MEM_H_ @@ -25,8 +44,6 @@ #ifdef CONFIG_TRACE_GPU_MEM #include <trace/events/gpu_mem.h> -#elif !MALI_CUSTOMER_RELEASE -#include "mali_gpu_mem_trace.h" #endif #define DEVICE_TGID ((u32) 0U) @@ -34,9 +51,9 @@ static void kbase_trace_gpu_mem_usage(struct kbase_device *kbdev, struct kbase_context *kctx) { +#ifdef CONFIG_TRACE_GPU_MEM lockdep_assert_held(&kbdev->gpu_mem_usage_lock); -#if defined(CONFIG_TRACE_GPU_MEM) || !MALI_CUSTOMER_RELEASE trace_gpu_mem_total(kbdev->id, DEVICE_TGID, kbdev->total_gpu_pages << PAGE_SHIFT); diff --git a/mali_kbase/mali_kbase_utility.h b/mali_kbase/mali_kbase_utility.h index 8d4f044..e8e928f 100644 --- a/mali_kbase/mali_kbase_utility.h +++ b/mali_kbase/mali_kbase_utility.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2013, 2015, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2012-2013, 2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - #ifndef _KBASE_UTILITY_H #define _KBASE_UTILITY_H @@ -32,7 +49,7 @@ static inline void kbase_timer_setup(struct timer_list *timer, void (*callback)(struct timer_list *timer)) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE setup_timer(timer, (void (*)(unsigned long)) callback, (unsigned long) timer); #else diff --git a/mali_kbase/mali_kbase_vinstr.c b/mali_kbase/mali_kbase_vinstr.c index e0e828c..9b2ec6f 100644 --- a/mali_kbase/mali_kbase_vinstr.c +++ b/mali_kbase/mali_kbase_vinstr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. @@ -360,11 +361,7 @@ static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer) * cancelled, and the worker itself won't reschedule this timer if * suspend_count != 0. */ -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, &vctx->dump_work); -#else - queue_work(system_highpri_wq, &vctx->dump_work); -#endif + kbase_hwcnt_virtualizer_queue_work(vctx->hvirt, &vctx->dump_work); return HRTIMER_NORESTART; } @@ -567,11 +564,8 @@ void kbase_vinstr_resume(struct kbase_vinstr_context *vctx) } if (has_periodic_clients) -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, &vctx->dump_work); -#else - queue_work(system_highpri_wq, &vctx->dump_work); -#endif + kbase_hwcnt_virtualizer_queue_work( + vctx->hvirt, &vctx->dump_work); } } @@ -834,11 +828,8 @@ static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( * worker is already queued. */ if ((interval != 0) && (cli->vctx->suspend_count == 0)) -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, &cli->vctx->dump_work); -#else - queue_work(system_highpri_wq, &cli->vctx->dump_work); -#endif + kbase_hwcnt_virtualizer_queue_work(cli->vctx->hvirt, + &cli->vctx->dump_work); mutex_unlock(&cli->vctx->lock); diff --git a/mali_kbase/mali_kbase_vinstr.h b/mali_kbase/mali_kbase_vinstr.h index 81d315f..fafcd5a 100644 --- a/mali_kbase/mali_kbase_vinstr.h +++ b/mali_kbase/mali_kbase_vinstr.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* diff --git a/mali_kbase/mali_linux_trace.h b/mali_kbase/mali_linux_trace.h index 4d2c972..e17cf26 100644 --- a/mali_kbase/mali_linux_trace.h +++ b/mali_kbase/mali_linux_trace.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #undef TRACE_SYSTEM @@ -348,10 +367,7 @@ TRACE_EVENT(mali_jit_report, #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ #endif /* !MALI_USE_CSF */ -#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); -#endif - #if MALI_JIT_PRESSURE_LIMIT_BASE /* trace_mali_jit_report_pressure * diff --git a/mali_kbase/mali_malisw.h b/mali_kbase/mali_malisw.h index 3a4db10..61ff7ae 100644 --- a/mali_kbase/mali_malisw.h +++ b/mali_kbase/mali_malisw.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** @@ -28,23 +47,6 @@ #define _MALISW_H_ #include <linux/version.h> -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) -#define U8_MAX ((u8)~0U) -#define S8_MAX ((s8)(U8_MAX>>1)) -#define S8_MIN ((s8)(-S8_MAX - 1)) -#define U16_MAX ((u16)~0U) -#define S16_MAX ((s16)(U16_MAX>>1)) -#define S16_MIN ((s16)(-S16_MAX - 1)) -#define U32_MAX ((u32)~0U) -#define S32_MAX ((s32)(U32_MAX>>1)) -#define S32_MIN ((s32)(-S32_MAX - 1)) -#define U64_MAX ((u64)~0ULL) -#define S64_MAX ((s64)(U64_MAX>>1)) -#define S64_MIN ((s64)(-S64_MAX - 1)) -#endif /* LINUX_VERSION_CODE */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) -#define SIZE_MAX (~(size_t)0) -#endif /* LINUX_VERSION_CODE */ /** * MIN - Return the lesser of two values. diff --git a/mali_kbase/mali_power_gpu_frequency_trace.c b/mali_kbase/mali_power_gpu_frequency_trace.c index b6fb5a0..36fb9cf 100644 --- a/mali_kbase/mali_power_gpu_frequency_trace.c +++ b/mali_kbase/mali_power_gpu_frequency_trace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mali_power_gpu_frequency_trace.h b/mali_kbase/mali_power_gpu_frequency_trace.h index 3b90ae4..b131d62 100644 --- a/mali_kbase/mali_power_gpu_frequency_trace.h +++ b/mali_kbase/mali_power_gpu_frequency_trace.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _TRACE_POWER_GPU_FREQUENCY_MALI diff --git a/mali_kbase/mali_uk.h b/mali_kbase/mali_uk.h index 701f390..9c3e89b 100644 --- a/mali_kbase/mali_uk.h +++ b/mali_kbase/mali_uk.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010, 2012-2015, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,27 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010, 2012-2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ - - /** * @file mali_uk.h * Types and definitions that are common across OSs for both the user diff --git a/mali_kbase/mmu/backend/mali_kbase_mmu_csf.c b/mali_kbase/mmu/backend/mali_kbase_mmu_csf.c index b23d1ff..265e19a 100644 --- a/mali_kbase/mmu/backend/mali_kbase_mmu_csf.c +++ b/mali_kbase/mmu/backend/mali_kbase_mmu_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -69,17 +70,27 @@ void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, static void submit_work_pagefault(struct kbase_device *kbdev, u32 as_nr, struct kbase_fault *fault) { + unsigned long flags; struct kbase_as *const as = &kbdev->as[as_nr]; + struct kbase_context *kctx; - as->pf_data = (struct kbase_fault) { - .status = fault->status, - .addr = fault->addr, - }; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); - if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { - WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); - atomic_inc(&kbdev->faults_pending); + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + + as->pf_data = (struct kbase_fault) { + .status = fault->status, + .addr = fault->addr, + }; + + if (WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault))) + kbase_ctx_sched_release_ctx(kctx); + else + atomic_inc(&kbdev->faults_pending); } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, @@ -106,8 +117,14 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, /* Report MMU fault for all address spaces (except MCU_AS_NR) */ for (as_no = 1; as_no < kbdev->nr_hw_address_spaces; as_no++) - if (kbase_ctx_sched_as_to_ctx(kbdev, as_no)) - submit_work_pagefault(kbdev, as_no, fault); + submit_work_pagefault(kbdev, as_no, fault); + + /* MCU AS fault could mean hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); /* GPU reset is required to recover */ if (kbase_prepare_to_reset_gpu(kbdev)) @@ -482,18 +499,25 @@ static void submit_work_gpufault(struct kbase_device *kbdev, u32 status, { unsigned long flags; struct kbase_as *const as = &kbdev->as[as_nr]; + struct kbase_context *kctx; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as->gf_data = (struct kbase_fault) { + kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); + + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + + as->gf_data = (struct kbase_fault) { .status = status, .addr = address, - }; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + }; - if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { - WARN_ON(!queue_work(as->pf_wq, &as->work_gpufault)); - atomic_inc(&kbdev->faults_pending); + if (WARN_ON(!queue_work(as->pf_wq, &as->work_gpufault))) + kbase_ctx_sched_release_ctx(kctx); + else + atomic_inc(&kbdev->faults_pending); } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, diff --git a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c index b0187a4..ddb3fd2 100644 --- a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c +++ b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mmu/mali_kbase_mmu.c b/mali_kbase/mmu/mali_kbase_mmu.c index cb57dc9..402f024 100644 --- a/mali_kbase/mmu/mali_kbase_mmu.c +++ b/mali_kbase/mmu/mali_kbase_mmu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -1616,6 +1617,16 @@ static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, */ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); +#if MALI_USE_CSF + /* A GPU hang could mean hardware counters will stop working. + * Put the backend into the unrecoverable error state to cause + * current and subsequent counter operations to immediately + * fail, avoiding the risk of a hang. + */ + kbase_hwcnt_backend_csf_on_unrecoverable_error( + &kbdev->hwcnt_gpu_iface); +#endif /* MALI_USE_CSF */ + if (kbase_prepare_to_reset_gpu(kbdev)) kbase_reset_gpu(kbdev); } @@ -1649,10 +1660,10 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, kbdev = kctx->kbdev; #if !MALI_USE_CSF mutex_lock(&kbdev->js_data.queue_mutex); -#endif /* !MALI_USE_CSF */ ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); -#if !MALI_USE_CSF mutex_unlock(&kbdev->js_data.queue_mutex); +#else + ctx_is_in_runpool = kbase_ctx_sched_refcount_mmu_flush(kctx, sync); #endif /* !MALI_USE_CSF */ if (ctx_is_in_runpool) { @@ -1674,6 +1685,11 @@ void kbase_mmu_update(struct kbase_device *kbdev, KBASE_DEBUG_ASSERT(as_nr != KBASEP_AS_NR_INVALID); kbdev->mmu_mode->update(kbdev, mmut, as_nr); + +#if MALI_USE_CSF + if (mmut->kctx) + mmut->kctx->mmu_flush_pend_state = KCTX_MMU_FLUSH_NOT_PEND; +#endif } KBASE_EXPORT_TEST_API(kbase_mmu_update); @@ -1695,6 +1711,7 @@ void kbase_mmu_disable(struct kbase_context *kctx) KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); /* * The address space is being disabled, drain all knowledge of it out @@ -1706,6 +1723,10 @@ void kbase_mmu_disable(struct kbase_context *kctx) kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); + +#if MALI_USE_CSF + kctx->mmu_flush_pend_state = KCTX_MMU_FLUSH_NOT_PEND; +#endif } KBASE_EXPORT_TEST_API(kbase_mmu_disable); @@ -2295,3 +2316,30 @@ void kbase_flush_mmu_wqs(struct kbase_device *kbdev) flush_workqueue(as->pf_wq); } } + +#if MALI_USE_CSF +void kbase_mmu_deferred_flush_invalidate(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + if (kctx->as_nr == KBASEP_AS_NR_INVALID) + return; + + if (kctx->mmu_flush_pend_state == KCTX_MMU_FLUSH_NOT_PEND) + return; + + WARN_ON(!atomic_read(&kctx->refcount)); + + /* Specify the entire address space as the locked region. + * The flush of entire L2 cache and complete TLB invalidation will + * anyways happen for the exisiting CSF GPUs, regardless of the locked + * range. This may have to be revised later on. + */ + kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, + kctx->mmu_flush_pend_state == KCTX_MMU_FLUSH_PEND_SYNC); + + kctx->mmu_flush_pend_state = KCTX_MMU_FLUSH_NOT_PEND; +} +#endif diff --git a/mali_kbase/mmu/mali_kbase_mmu.h b/mali_kbase/mmu/mali_kbase_mmu.h index f2613e8..ba525e7 100644 --- a/mali_kbase/mmu/mali_kbase_mmu.h +++ b/mali_kbase/mmu/mali_kbase_mmu.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_MMU_H_ @@ -153,4 +172,21 @@ int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, u32 status, void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, u32 as_nr, u64 address, bool as_valid); +#if MALI_USE_CSF +/** + * kbase_mmu_deferred_flush_invalidate() - Perform deferred MMU flush + * operations for a Kbase context. + * @kctx: Pointer to the Kbase context for which MMU flush operations + * are pending. + * + * This function performs the MMU flush operations that are pending for a Kbase + * context. The flush operations will be deferred if the context is inactive, + * i.e. kctx->refcount is zero which happens when all the queue groups of a + * context have gone off CSG slots. + * This needs to be called when first queue group of the context is put back + * on the CSG slot. + */ +void kbase_mmu_deferred_flush_invalidate(struct kbase_context *kctx); +#endif + #endif /* _KBASE_MMU_H_ */ diff --git a/mali_kbase/mmu/mali_kbase_mmu_hw.h b/mali_kbase/mmu/mali_kbase_mmu_hw.h index e6eef86..bc7561d 100644 --- a/mali_kbase/mmu/mali_kbase_mmu_hw.h +++ b/mali_kbase/mmu/mali_kbase_mmu_hw.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/mmu/mali_kbase_mmu_hw_direct.c b/mali_kbase/mmu/mali_kbase_mmu_hw_direct.c index e9eef8b..78d84ad 100644 --- a/mali_kbase/mmu/mali_kbase_mmu_hw_direct.c +++ b/mali_kbase/mmu/mali_kbase_mmu_hw_direct.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/mmu/mali_kbase_mmu_internal.h b/mali_kbase/mmu/mali_kbase_mmu_internal.h index 8ecb14d..cc47c91 100644 --- a/mali_kbase/mmu/mali_kbase_mmu_internal.h +++ b/mali_kbase/mmu/mali_kbase_mmu_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_MMU_INTERNAL_H_ diff --git a/mali_kbase/mmu/mali_kbase_mmu_mode_aarch64.c b/mali_kbase/mmu/mali_kbase_mmu_mode_aarch64.c index 02493e9..f705663 100644 --- a/mali_kbase/mmu/mali_kbase_mmu_mode_aarch64.c +++ b/mali_kbase/mmu/mali_kbase_mmu_mode_aarch64.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2010-2014, 2016-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2014, 2016-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -48,25 +49,7 @@ */ static inline void page_table_entry_set(u64 *pte, u64 phy) { -#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE WRITE_ONCE(*pte, phy); -#else -#ifdef CONFIG_64BIT - barrier(); - *pte = phy; - barrier(); -#elif defined(CONFIG_ARM) - barrier(); - asm volatile("ldrd r0, [%1]\n\t" - "strd r0, %0\n\t" - : "=m" (*pte) - : "r" (&phy) - : "r0", "r1"); - barrier(); -#else -#error "64-bit atomic write must be implemented for your architecture" -#endif -#endif } static void mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, diff --git a/mali_kbase/mmu/mali_kbase_mmu_mode_lpae.c b/mali_kbase/mmu/mali_kbase_mmu_mode_lpae.c index 91a2d7a..b98d64e 100644 --- a/mali_kbase/mmu/mali_kbase_mmu_mode_lpae.c +++ b/mali_kbase/mmu/mali_kbase_mmu_mode_lpae.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,7 +21,6 @@ * */ - #include "mali_kbase.h" #include <gpu/mali_kbase_gpu_regmap.h> #include "mali_kbase_defs.h" @@ -45,25 +45,7 @@ */ static inline void page_table_entry_set(u64 *pte, u64 phy) { -#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE WRITE_ONCE(*pte, phy); -#else -#ifdef CONFIG_64BIT - barrier(); - *pte = phy; - barrier(); -#elif defined(CONFIG_ARM) - barrier(); - asm volatile("ldrd r0, [%1]\n\t" - "strd r0, %0\n\t" - : "=m" (*pte) - : "r" (&phy) - : "r0", "r1"); - barrier(); -#else -#error "64-bit atomic write must be implemented for your architecture" -#endif -#endif } static void mmu_get_as_setup(struct kbase_mmu_table *mmut, diff --git a/mali_kbase/platform/Kconfig b/mali_kbase/platform/Kconfig index ef9fb96..d5477b2 100644 --- a/mali_kbase/platform/Kconfig +++ b/mali_kbase/platform/Kconfig @@ -19,12 +19,9 @@ # # - - # Add your platform specific Kconfig file here # # "drivers/gpu/arm/midgard/platform/xxx/Kconfig" # # Where xxx is the platform name is the name set in MALI_PLATFORM_NAME # - diff --git a/mali_kbase/platform/devicetree/mali_kbase_clk_rate_trace.c b/mali_kbase/platform/devicetree/mali_kbase_clk_rate_trace.c index 47933a7..080514c 100644 --- a/mali_kbase/platform/devicetree/mali_kbase_clk_rate_trace.c +++ b/mali_kbase/platform/devicetree/mali_kbase_clk_rate_trace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2015, 2017-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/platform/devicetree/mali_kbase_config_devicetree.c b/mali_kbase/platform/devicetree/mali_kbase_config_devicetree.c index ccefddf..df82806 100644 --- a/mali_kbase/platform/devicetree/mali_kbase_config_devicetree.c +++ b/mali_kbase/platform/devicetree/mali_kbase_config_devicetree.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h index 2137b42..f16c9bc 100644 --- a/mali_kbase/platform/devicetree/mali_kbase_config_platform.h +++ b/mali_kbase/platform/devicetree/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/platform/devicetree/mali_kbase_runtime_pm.c b/mali_kbase/platform/devicetree/mali_kbase_runtime_pm.c index 8772edb..72639b5 100644 --- a/mali_kbase/platform/devicetree/mali_kbase_runtime_pm.c +++ b/mali_kbase/platform/devicetree/mali_kbase_runtime_pm.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2015, 2017-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/platform/vexpress/Kbuild b/mali_kbase/platform/vexpress/Kbuild index 6780e4c..bcd3167 100644 --- a/mali_kbase/platform/vexpress/Kbuild +++ b/mali_kbase/platform/vexpress/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2013, 2016-2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h index fac3cd5..a9988ae 100644 --- a/mali_kbase/platform/vexpress/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/platform/vexpress/mali_kbase_config_vexpress.c b/mali_kbase/platform/vexpress/mali_kbase_config_vexpress.c index d165ce2..d02ca3f 100644 --- a/mali_kbase/platform/vexpress/mali_kbase_config_vexpress.c +++ b/mali_kbase/platform/vexpress/mali_kbase_config_vexpress.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - #include <linux/ioport.h> #include <mali_kbase.h> #include <mali_kbase_defs.h> diff --git a/mali_kbase/platform/vexpress_1xv7_a57/Kbuild b/mali_kbase/platform/vexpress_1xv7_a57/Kbuild index 51b408e..ae03670 100644 --- a/mali_kbase/platform/vexpress_1xv7_a57/Kbuild +++ b/mali_kbase/platform/vexpress_1xv7_a57/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2013-2014, 2016-2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h index fac3cd5..a9988ae 100644 --- a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c index efca0a5..6d6aaf6 100644 --- a/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +++ b/mali_kbase/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2014, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h index fac3cd5..a9988ae 100644 --- a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +++ b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /** diff --git a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c index b6714b9..c3fbf21 100644 --- a/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +++ b/mali_kbase/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2014, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - #include <linux/ioport.h> #include <mali_kbase.h> #include <mali_kbase_defs.h> diff --git a/mali_kbase/protected_mode_switcher.h b/mali_kbase/protected_mode_switcher.h index 8778d81..bde31ec 100644 --- a/mali_kbase/protected_mode_switcher.h +++ b/mali_kbase/protected_mode_switcher.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _PROTECTED_MODE_SWITCH_H_ diff --git a/mali_kbase/tests/Mconfig b/mali_kbase/tests/Mconfig index be3fedb..cdbbaac 100644 --- a/mali_kbase/tests/Mconfig +++ b/mali_kbase/tests/Mconfig @@ -1,14 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 # # (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software # Foundation, and any use by you of this program is subject to the terms -# of such GNU licence. +# of such GNU license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. # -# A copy of the licence is included with the program, and can also be obtained -# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -# Boston, MA 02110-1301, USA. # config UNIT_TEST_KERNEL_MODULES diff --git a/mali_kbase/tests/include/kutf/kutf_helpers.h b/mali_kbase/tests/include/kutf/kutf_helpers.h index 858b9c3..ea87f9c 100644 --- a/mali_kbase/tests/include/kutf/kutf_helpers.h +++ b/mali_kbase/tests/include/kutf/kutf_helpers.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_HELPERS_H_ diff --git a/mali_kbase/tests/include/kutf/kutf_helpers_user.h b/mali_kbase/tests/include/kutf/kutf_helpers_user.h index 3b1300e..38f36fc 100644 --- a/mali_kbase/tests/include/kutf/kutf_helpers_user.h +++ b/mali_kbase/tests/include/kutf/kutf_helpers_user.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_HELPERS_USER_H_ @@ -63,7 +82,8 @@ struct kutf_helper_named_val { * unrecoverable) * * Positive values indicate correct access but invalid parsing (can be - * recovered from assuming data in the future is correct) */ + * recovered from assuming data in the future is correct) + */ enum kutf_helper_err { /* No error - must be zero */ KUTF_HELPER_ERR_NONE = 0, @@ -71,14 +91,16 @@ enum kutf_helper_err { KUTF_HELPER_ERR_INVALID_NAME, /* Named value parsing of string or u64 type encountered extra * characters after the value (after the last digit for a u64 type or - * after the string end delimiter for string type) */ + * after the string end delimiter for string type) + */ KUTF_HELPER_ERR_CHARS_AFTER_VAL, /* Named value parsing of string type couldn't find the string end * delimiter. * * This cannot be encountered when the NAME="value" message exceeds the * textbuf's maximum line length, because such messages are not checked - * for an end string delimiter */ + * for an end string delimiter + */ KUTF_HELPER_ERR_NO_END_DELIMITER, /* Named value didn't parse as any of the known types */ KUTF_HELPER_ERR_INVALID_VALUE, @@ -122,7 +144,8 @@ int kutf_helper_max_str_len_for_kern(const char *val_name, int kern_buf_sz); * * Any failure will be logged on the suite's current test fixture * - * Returns 0 on success, non-zero on failure */ + * Returns 0 on success, non-zero on failure + */ int kutf_helper_send_named_str(struct kutf_context *context, const char *val_name, const char *val_str); @@ -138,7 +161,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, * * Returns 0 on success. Negative value on failure to receive from the 'run' * file, positive value indicates an enum kutf_helper_err value for correct - * reception of data but invalid parsing */ + * reception of data but invalid parsing + */ int kutf_helper_receive_named_val( struct kutf_context *context, struct kutf_helper_named_val *named_val); @@ -165,7 +189,8 @@ int kutf_helper_receive_named_val( * - return value will be 0 to indicate success * * The rationale behind this is that we'd prefer to continue the rest of the - * test with failures propagated, rather than hitting a timeout */ + * test with failures propagated, rather than hitting a timeout + */ int kutf_helper_receive_check_val( struct kutf_helper_named_val *named_val, struct kutf_context *context, diff --git a/mali_kbase/tests/include/kutf/kutf_mem.h b/mali_kbase/tests/include/kutf/kutf_mem.h index 988559d..8a0e855 100644 --- a/mali_kbase/tests/include/kutf/kutf_mem.h +++ b/mali_kbase/tests/include/kutf/kutf_mem.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_MEM_H_ diff --git a/mali_kbase/tests/include/kutf/kutf_resultset.h b/mali_kbase/tests/include/kutf/kutf_resultset.h index 49ebeb4..f21de19 100644 --- a/mali_kbase/tests/include/kutf/kutf_resultset.h +++ b/mali_kbase/tests/include/kutf/kutf_resultset.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_RESULTSET_H_ diff --git a/mali_kbase/tests/include/kutf/kutf_suite.h b/mali_kbase/tests/include/kutf/kutf_suite.h index 8d75f50..f4af4fa 100644 --- a/mali_kbase/tests/include/kutf/kutf_suite.h +++ b/mali_kbase/tests/include/kutf/kutf_suite.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_SUITE_H_ @@ -264,9 +283,10 @@ struct kutf_suite { struct list_head test_list; }; -/* ============================================================================ - Application functions -============================================================================ */ +/** =========================================================================== + * Application functions + * ============================================================================ + */ /** * kutf_create_application() - Create an in kernel test application. @@ -284,9 +304,10 @@ struct kutf_application *kutf_create_application(const char *name); */ void kutf_destroy_application(struct kutf_application *app); -/* ============================================================================ - Suite functions -============================================================================ */ +/**============================================================================ + * Suite functions + * ============================================================================ + */ /** * kutf_create_suite() - Create a kernel test suite. @@ -416,10 +437,10 @@ void kutf_add_test_with_filters_and_data( unsigned int filters, union kutf_callback_data test_data); - -/* ============================================================================ - Test functions -============================================================================ */ +/** =========================================================================== + * Test functions + * ============================================================================ + */ /** * kutf_test_log_result_external() - Log a result which has been created * externally into a in a standard form diff --git a/mali_kbase/tests/include/kutf/kutf_utils.h b/mali_kbase/tests/include/kutf/kutf_utils.h index 25b8285..a60e569 100644 --- a/mali_kbase/tests/include/kutf/kutf_utils.h +++ b/mali_kbase/tests/include/kutf/kutf_utils.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KERNEL_UTF_UTILS_H_ diff --git a/mali_kbase/tests/kutf/Kbuild b/mali_kbase/tests/kutf/Kbuild index 2531d41..de68729 100644 --- a/mali_kbase/tests/kutf/Kbuild +++ b/mali_kbase/tests/kutf/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/kutf/Kconfig b/mali_kbase/tests/kutf/Kconfig index 0cdb474..3d1f92d 100644 --- a/mali_kbase/tests/kutf/Kconfig +++ b/mali_kbase/tests/kutf/Kconfig @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -19,7 +19,6 @@ # # - config MALI_KUTF tristate "Mali Kernel Unit Test Framework" default m diff --git a/mali_kbase/tests/kutf/Makefile b/mali_kbase/tests/kutf/Makefile index d848e87..b9e070b 100644 --- a/mali_kbase/tests/kutf/Makefile +++ b/mali_kbase/tests/kutf/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/kutf/build.bp b/mali_kbase/tests/kutf/build.bp index 32eab14..707a053 100644 --- a/mali_kbase/tests/kutf/build.bp +++ b/mali_kbase/tests/kutf/build.bp @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -5,11 +6,16 @@ * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/mali_kbase/tests/kutf/kutf_helpers.c b/mali_kbase/tests/kutf/kutf_helpers.c index 4463b04..13b6353 100644 --- a/mali_kbase/tests/kutf/kutf_helpers.c +++ b/mali_kbase/tests/kutf/kutf_helpers.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/tests/kutf/kutf_helpers_user.c b/mali_kbase/tests/kutf/kutf_helpers_user.c index 108fa82..00bc9c0 100644 --- a/mali_kbase/tests/kutf/kutf_helpers_user.c +++ b/mali_kbase/tests/kutf/kutf_helpers_user.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,7 +43,8 @@ static const char *get_val_type_name(enum kutf_helper_valtype valtype) * a) "<0 comparison on unsigned type" warning - if we did both upper * and lower bound check * b) incorrect range checking if it was a signed type - if we did - * upper bound check only */ + * upper bound check only + */ unsigned int type_idx = (unsigned int)valtype; if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) @@ -54,7 +56,8 @@ static const char *get_val_type_name(enum kutf_helper_valtype valtype) /* Check up to str_len chars of val_str to see if it's a valid value name: * * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator - * - And, each char is in the character set [A-Z0-9_] */ + * - And, each char is in the character set [A-Z0-9_] + */ static int validate_val_name(const char *val_str, int str_len) { int i = 0; @@ -87,7 +90,8 @@ static int validate_val_name(const char *val_str, int str_len) * e.g. "str" * * That is, before any '\\', '\n' or '"' characters. This is so we don't have - * to escape the string */ + * to escape the string + */ static int find_quoted_string_valid_len(const char *str) { char *ptr; @@ -207,7 +211,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; /* Using kmalloc() here instead of mempool since we know we need to free - * before we return */ + * before we return + */ str_buf = kmalloc(str_buf_sz, GFP_KERNEL); if (!str_buf) { errmsg = kutf_dsprintf(&context->fixture_pool, @@ -218,7 +223,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, copy_ptr = str_buf; /* Manually copy each string component instead of snprintf because - * val_str may need to end early, and less error path handling */ + * val_str may need to end early, and less error path handling + */ /* name */ memcpy(copy_ptr, val_name, val_name_len); @@ -331,7 +337,8 @@ int kutf_helper_receive_named_val( /* possibly a number value - strtoull will parse it */ err = kstrtoull(recv_str, 0, &u64val); /* unlike userspace can't get an end ptr, but if kstrtoull() - * reads characters after the number it'll report -EINVAL */ + * reads characters after the number it'll report -EINVAL + */ if (!err) { int len_remain = strnlen(recv_str, recv_sz); @@ -399,7 +406,8 @@ int kutf_helper_receive_check_val( goto out_fail_and_fixup; } - if (strcmp(named_val->val_name, expect_val_name) != 0) { + if (named_val->val_name != NULL && + strcmp(named_val->val_name, expect_val_name) != 0) { const char *msg = kutf_dsprintf(&context->fixture_pool, "Expecting to receive value named '%s' but got '%s'", expect_val_name, named_val->val_name); diff --git a/mali_kbase/tests/kutf/kutf_mem.c b/mali_kbase/tests/kutf/kutf_mem.c index fd98bea..88c2de8 100644 --- a/mali_kbase/tests/kutf/kutf_mem.c +++ b/mali_kbase/tests/kutf/kutf_mem.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/kutf/kutf_resultset.c b/mali_kbase/tests/kutf/kutf_resultset.c index 94ecfa4..ed6f02a 100644 --- a/mali_kbase/tests/kutf/kutf_resultset.c +++ b/mali_kbase/tests/kutf/kutf_resultset.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/kutf/kutf_suite.c b/mali_kbase/tests/kutf/kutf_suite.c index 9dc6e2b..382557b 100644 --- a/mali_kbase/tests/kutf/kutf_suite.c +++ b/mali_kbase/tests/kutf/kutf_suite.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2014, 2017-2020 ARM Limited. All rights reserved. @@ -21,7 +22,8 @@ */ /* Kernel UTF suite, test and fixture management including user to kernel - * interaction */ + * interaction + */ #include <linux/list.h> #include <linux/slab.h> @@ -598,7 +600,7 @@ static int create_fixture_variant(struct kutf_test_function *test_func, goto fail_file; } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) +#if KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE tmp = debugfs_create_file_unsafe( #else tmp = debugfs_create_file( @@ -634,7 +636,7 @@ static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) kfree(test_fix); } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) +#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE /* Adapting to the upstream debugfs_create_x32() change */ static int ktufp_u32_get(void *data, u64 *val) { @@ -679,7 +681,7 @@ void kutf_add_test_with_filters_and_data( } test_func->filters = filters; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) +#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE tmp = debugfs_create_file_unsafe("filters", S_IROTH, test_func->dir, &test_func->filters, &kutfp_fops_x32_ro); #else @@ -692,7 +694,7 @@ void kutf_add_test_with_filters_and_data( } test_func->test_id = id; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) +#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE debugfs_create_u32("test_id", S_IROTH, test_func->dir, &test_func->test_id); #else diff --git a/mali_kbase/tests/kutf/kutf_utils.c b/mali_kbase/tests/kutf/kutf_utils.c index 7f5ac51..7d1de25 100644 --- a/mali_kbase/tests/kutf/kutf_utils.c +++ b/mali_kbase/tests/kutf/kutf_utils.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014, 2017, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/build.bp b/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/build.bp index 0cc2904..c16b3de 100644 --- a/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/build.bp +++ b/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/build.bp @@ -1,14 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* + * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. */ bob_kernel_module { diff --git a/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index d74a278..88dfef4 100644 --- a/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/mali_kbase/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h b/mali_kbase/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h index f46afd5..0b7b84d 100644 --- a/mali_kbase/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h +++ b/mali_kbase/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KUTF_CLK_RATE_TRACE_TEST_H_ diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kbuild b/mali_kbase/tests/mali_kutf_irq_test/Kbuild index ca8c512..a1a4828 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/Kbuild +++ b/mali_kbase/tests/mali_kutf_irq_test/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/mali_kutf_irq_test/Kconfig b/mali_kbase/tests/mali_kutf_irq_test/Kconfig index 4a3863a..c49ec11 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/Kconfig +++ b/mali_kbase/tests/mali_kutf_irq_test/Kconfig @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/mali_kbase/tests/mali_kutf_irq_test/build.bp b/mali_kbase/tests/mali_kutf_irq_test/build.bp index 90efdcf..58021c8 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/build.bp +++ b/mali_kbase/tests/mali_kutf_irq_test/build.bp @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. @@ -5,11 +6,16 @@ * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * - * A copy of the licence is included with the program, and can also be obtained - * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c index 5f27c3a..3c22004 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +++ b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2016-2018, 2020 ARM Limited. All rights reserved. @@ -242,7 +243,7 @@ int mali_kutf_irq_test_main_init(void) irq_app = kutf_create_application("irq"); - if (NULL == irq_app) { + if (irq_app == NULL) { pr_warn("Creation of test application failed!\n"); return -ENOMEM; } @@ -251,7 +252,7 @@ int mali_kutf_irq_test_main_init(void) 1, mali_kutf_irq_default_create_fixture, mali_kutf_irq_default_remove_fixture); - if (NULL == suite) { + if (suite == NULL) { pr_warn("Creation of test suite failed!\n"); kutf_destroy_application(irq_app); return -ENOMEM; diff --git a/mali_kbase/thirdparty/mali_kbase_mmap.c b/mali_kbase/thirdparty/mali_kbase_mmap.c index 83a293d..6201cfc 100644 --- a/mali_kbase/thirdparty/mali_kbase_mmap.c +++ b/mali_kbase/thirdparty/mali_kbase_mmap.c @@ -209,7 +209,8 @@ check_current: return -ENOMEM; if (gap_start <= high_limit && gap_end - gap_start >= length) { /* We found a suitable gap. Clip it with the original - * high_limit. */ + * high_limit. + */ if (gap_end > info->high_limit) gap_end = info->high_limit; @@ -325,7 +326,8 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, /* kbase_check_alloc_sizes() already satisfies * these checks, but they're here to avoid * maintenance hazards due to the assumptions - * involved */ + * involved + */ WARN_ON(reg->extension > (ULONG_MAX >> PAGE_SHIFT)); WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); diff --git a/mali_kbase/tl/backend/mali_kbase_timeline_csf.c b/mali_kbase/tl/backend/mali_kbase_timeline_csf.c index abaa6bb..476214d 100644 --- a/mali_kbase/tl/backend/mali_kbase_timeline_csf.c +++ b/mali_kbase/tl/backend/mali_kbase_timeline_csf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -60,7 +61,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Lock the context list, to ensure no changes to the list are made * while we're summarizing the contexts and their contents. */ - mutex_lock(&kbdev->kctx_list_lock); + mutex_lock(&timeline->tl_kctx_list_lock); /* Hold the scheduler lock while we emit the current state * We also need to continue holding the lock until after the first body @@ -90,7 +91,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) mutex_unlock(&kbdev->csf.scheduler.lock); /* For each context in the device... */ - list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { size_t i; struct kbase_tlstream *body = &timeline->streams[TL_STREAM_TYPE_OBJ]; @@ -162,7 +163,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) */ }; - mutex_unlock(&kbdev->kctx_list_lock); + mutex_unlock(&timeline->tl_kctx_list_lock); /* Static object are placed into summary packet that needs to be * transmitted first. Flush all streams to make it available to diff --git a/mali_kbase/tl/backend/mali_kbase_timeline_jm.c b/mali_kbase/tl/backend/mali_kbase_timeline_jm.c index c368ac7..4babd1e 100644 --- a/mali_kbase/tl/backend/mali_kbase_timeline_jm.c +++ b/mali_kbase/tl/backend/mali_kbase_timeline_jm.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -66,10 +67,10 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Lock the context list, to ensure no changes to the list are made * while we're summarizing the contexts and their contents. */ - mutex_lock(&kbdev->kctx_list_lock); + mutex_lock(&timeline->tl_kctx_list_lock); /* For each context in the device... */ - list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { /* Summarize the context itself */ __kbase_tlstream_tl_new_ctx(summary, kctx, @@ -87,11 +88,11 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) */ kbase_timeline_streams_body_reset(timeline); - mutex_unlock(&kbdev->kctx_list_lock); + mutex_unlock(&timeline->tl_kctx_list_lock); /* Static object are placed into summary packet that needs to be * transmitted first. Flush all streams to make it available to * user space. */ kbase_timeline_streams_flush(timeline); -}
\ No newline at end of file +} diff --git a/mali_kbase/tl/mali_kbase_timeline.c b/mali_kbase/tl/mali_kbase_timeline.c index 8ebc075..98185e9 100644 --- a/mali_kbase/tl/mali_kbase_timeline.c +++ b/mali_kbase/tl/mali_kbase_timeline.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. @@ -116,7 +117,7 @@ int kbase_timeline_init(struct kbase_timeline **timeline, if (!timeline || !timeline_flags) return -EINVAL; - result = kzalloc(sizeof(*result), GFP_KERNEL); + result = vzalloc(sizeof(*result)); if (!result) return -ENOMEM; @@ -128,6 +129,10 @@ int kbase_timeline_init(struct kbase_timeline **timeline, kbase_tlstream_init(&result->streams[i], i, &result->event_queue); + /* Initialize the kctx list */ + mutex_init(&result->tl_kctx_list_lock); + INIT_LIST_HEAD(&result->tl_kctx_list); + /* Initialize autoflush timer. */ atomic_set(&result->autoflush_timer_active, 0); kbase_timer_setup(&result->autoflush_timer, @@ -154,10 +159,12 @@ void kbase_timeline_term(struct kbase_timeline *timeline) kbase_csf_tl_reader_term(&timeline->csf_tl_reader); #endif + WARN_ON(!list_empty(&timeline->tl_kctx_list)); + for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; i++) kbase_tlstream_term(&timeline->streams[i]); - kfree(timeline); + vfree(timeline); } #ifdef CONFIG_MALI_DEVFREQ @@ -172,11 +179,7 @@ static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) unsigned long cur_freq = 0; mutex_lock(&devfreq->lock); -#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE - cur_freq = kbdev->current_nominal_freq; -#else cur_freq = devfreq->last_status.current_frequency; -#endif KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)cur_freq); mutex_unlock(&devfreq->lock); } @@ -288,6 +291,74 @@ void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) #endif } +void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_timeline *timeline = kbdev->timeline; + + /* Remove the context from the list to ensure we don't try and + * summarize a context that is being destroyed. + * + * It's unsafe to try and summarize a context being destroyed as the + * locks we might normally attempt to acquire, and the data structures + * we would normally attempt to traverse could already be destroyed. + * + * In the case where the tlstream is acquired between this pre destroy + * call and the post destroy call, we will get a context destroy + * tracepoint without the corresponding context create tracepoint, + * but this will not affect the correctness of the object model. + */ + mutex_lock(&timeline->tl_kctx_list_lock); + list_del_init(&kctx->tl_kctx_list_node); + mutex_unlock(&timeline->tl_kctx_list_lock); +} + +void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_timeline *timeline = kbdev->timeline; + + /* On context create, add the context to the list to ensure it is + * summarized when timeline is acquired + */ + mutex_lock(&timeline->tl_kctx_list_lock); + + list_add(&kctx->tl_kctx_list_node, &timeline->tl_kctx_list); + + /* Fire the tracepoints with the lock held to ensure the tracepoints + * are either fired before or after the summarization, + * never in parallel with it. If fired in parallel, we could get + * duplicate creation tracepoints. + */ +#if MALI_USE_CSF + KBASE_TLSTREAM_TL_KBASE_NEW_CTX( + kbdev, kctx->id, kbdev->gpu_props.props.raw_props.gpu_id); +#endif + /* Trace with the AOM tracepoint even in CSF for dumping */ + KBASE_TLSTREAM_TL_NEW_CTX(kbdev, kctx, kctx->id, 0); + + mutex_unlock(&timeline->tl_kctx_list_lock); +} + +void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + /* Trace with the AOM tracepoint even in CSF for dumping */ + KBASE_TLSTREAM_TL_DEL_CTX(kbdev, kctx); +#if MALI_USE_CSF + KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kbdev, kctx->id); +#endif + + /* Flush the timeline stream, so the user can see the termination + * tracepoints being fired. + * The "if" statement below is for optimization. It is safe to call + * kbase_timeline_streams_flush when timeline is disabled. + */ + if (atomic_read(&kbdev->timeline_flags) != 0) + kbase_timeline_streams_flush(kbdev->timeline); +} + #if MALI_UNIT_TEST void kbase_timeline_stats(struct kbase_timeline *timeline, u32 *bytes_collected, u32 *bytes_generated) diff --git a/mali_kbase/tl/mali_kbase_timeline.h b/mali_kbase/tl/mali_kbase_timeline.h index cd48411..ea75a69 100644 --- a/mali_kbase/tl/mali_kbase_timeline.h +++ b/mali_kbase/tl/mali_kbase_timeline.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #if !defined(_KBASE_TIMELINE_H) @@ -81,6 +100,30 @@ void kbase_timeline_streams_flush(struct kbase_timeline *timeline); */ void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline); +/** + * kbase_timeline_post_kbase_context_create - Inform timeline that a new KBase + * Context has been created. + * @kctx: KBase Context + */ +void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx); + +/** + * kbase_timeline_pre_kbase_context_destroy - Inform timeline that a KBase + * Context is about to be destroyed. + * @kctx: KBase Context + */ +void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx); + +/** + * kbase_timeline_post_kbase_context_destroy - Inform timeline that a KBase + * Context has been destroyed. + * @kctx: KBase Context + * + * Should be called immediately before the memory is freed, and the context ID + * and kbdev pointer should still be valid. + */ +void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx); + #if MALI_UNIT_TEST /** * kbase_timeline_test - start timeline stream data generator diff --git a/mali_kbase/tl/mali_kbase_timeline_io.c b/mali_kbase/tl/mali_kbase_timeline_io.c index 724f5fa..a6d02b9 100644 --- a/mali_kbase/tl/mali_kbase_timeline_io.c +++ b/mali_kbase/tl/mali_kbase_timeline_io.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. diff --git a/mali_kbase/tl/mali_kbase_timeline_priv.h b/mali_kbase/tl/mali_kbase_timeline_priv.h index 35eec46..d305bb3 100644 --- a/mali_kbase/tl/mali_kbase_timeline_priv.h +++ b/mali_kbase/tl/mali_kbase_timeline_priv.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #if !defined(_KBASE_TIMELINE_PRIV_H) @@ -38,6 +57,8 @@ /** * struct kbase_timeline - timeline state structure * @streams: The timeline streams generated by kernel + * @tl_kctx_list: List of contexts for timeline. + * @tl_kctx_list_lock: Lock to protect @tl_kctx_list. * @autoflush_timer: Autoflush timer * @autoflush_timer_active: If non-zero autoflush timer is active * @reader_lock: Reader lock. Only one reader is allowed to @@ -51,6 +72,8 @@ */ struct kbase_timeline { struct kbase_tlstream streams[TL_STREAM_TYPE_COUNT]; + struct list_head tl_kctx_list; + struct mutex tl_kctx_list_lock; struct timer_list autoflush_timer; atomic_t autoflush_timer_active; struct mutex reader_lock; diff --git a/mali_kbase/tl/mali_kbase_tl_serialize.h b/mali_kbase/tl/mali_kbase_tl_serialize.h index 3e37827..78aee83 100644 --- a/mali_kbase/tl/mali_kbase_tl_serialize.h +++ b/mali_kbase/tl/mali_kbase_tl_serialize.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #if !defined(_KBASE_TL_SERIALIZE_H) diff --git a/mali_kbase/tl/mali_kbase_tlstream.c b/mali_kbase/tl/mali_kbase_tlstream.c index f4239cf..b682ecd 100644 --- a/mali_kbase/tl/mali_kbase_tlstream.c +++ b/mali_kbase/tl/mali_kbase_tlstream.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. @@ -149,12 +150,12 @@ void kbase_tlstream_init( unsigned int i; KBASE_DEBUG_ASSERT(stream); - KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + KBASE_DEBUG_ASSERT(stream_type < TL_STREAM_TYPE_COUNT); spin_lock_init(&stream->lock); /* All packets carrying tracepoints shall be numbered. */ - if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) + if (tl_stream_cfg[stream_type].pkt_type == TL_PACKET_TYPE_BODY) stream->numbered = 1; else stream->numbered = 0; @@ -217,7 +218,8 @@ static size_t kbasep_tlstream_msgbuf_submit( /* Increasing write buffer index will expose this packet to the reader. * As stream->lock is not taken on reader side we must make sure memory - * is updated correctly before this will happen. */ + * is updated correctly before this will happen. + */ smp_wmb(); atomic_inc(&stream->wbi); @@ -251,7 +253,7 @@ char *kbase_tlstream_msgbuf_acquire( wb_size = atomic_read(&stream->buffer[wb_idx].size); /* Select next buffer if data will not fit into current one. */ - if (PACKET_SIZE < wb_size + msg_size) { + if (wb_size + msg_size > PACKET_SIZE) { wb_size = kbasep_tlstream_msgbuf_submit( stream, wb_idx_raw, wb_size); wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; diff --git a/mali_kbase/tl/mali_kbase_tlstream.h b/mali_kbase/tl/mali_kbase_tlstream.h index faf88d6..4ad4ad1 100644 --- a/mali_kbase/tl/mali_kbase_tlstream.h +++ b/mali_kbase/tl/mali_kbase_tlstream.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #if !defined(_KBASE_TLSTREAM_H) diff --git a/mali_kbase/tl/mali_kbase_tracepoints.c b/mali_kbase/tl/mali_kbase_tracepoints.c index 0502c0d..9a2d7db 100644 --- a/mali_kbase/tl/mali_kbase_tracepoints.c +++ b/mali_kbase/tl/mali_kbase_tracepoints.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -135,6 +136,7 @@ enum tl_msg_id_aux { KBASE_AUX_PROTECTED_LEAVE_START, KBASE_AUX_PROTECTED_LEAVE_END, KBASE_AUX_JIT_STATS, + KBASE_AUX_TILER_HEAP_STATS, KBASE_AUX_EVENT_JOB_SLOT, KBASE_AUX_MSG_COUNT, }; @@ -541,6 +543,10 @@ const size_t obj_desc_header_size = sizeof(__obj_desc_header); "per-bin JIT statistics", \ "@IIIIII", \ "ctx_nr,bid,max_allocs,allocs,va_pages,ph_pages") \ + TRACEPOINT_DESC(KBASE_AUX_TILER_HEAP_STATS, \ + "Tiler Heap statistics", \ + "@ILIIIIIII", \ + "ctx_nr,heap_id,va_pages,ph_pages,max_chunks,chunk_size,chunk_count,target_in_flight,nr_in_flight") \ TRACEPOINT_DESC(KBASE_AUX_EVENT_JOB_SLOT, \ "event on a given job slot", \ "@pIII", \ @@ -1817,6 +1823,60 @@ void __kbase_tlstream_aux_jit_stats( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_aux_tiler_heap_stats( + struct kbase_tlstream *stream, + u32 ctx_nr, + u64 heap_id, + u32 va_pages, + u32 ph_pages, + u32 max_chunks, + u32 chunk_size, + u32 chunk_count, + u32 target_in_flight, + u32 nr_in_flight) +{ + const u32 msg_id = KBASE_AUX_TILER_HEAP_STATS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx_nr) + + sizeof(heap_id) + + sizeof(va_pages) + + sizeof(ph_pages) + + sizeof(max_chunks) + + sizeof(chunk_size) + + sizeof(chunk_count) + + sizeof(target_in_flight) + + sizeof(nr_in_flight) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &heap_id, sizeof(heap_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &va_pages, sizeof(va_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &ph_pages, sizeof(ph_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &max_chunks, sizeof(max_chunks)); + pos = kbasep_serialize_bytes(buffer, + pos, &chunk_size, sizeof(chunk_size)); + pos = kbasep_serialize_bytes(buffer, + pos, &chunk_count, sizeof(chunk_count)); + pos = kbasep_serialize_bytes(buffer, + pos, &target_in_flight, sizeof(target_in_flight)); + pos = kbasep_serialize_bytes(buffer, + pos, &nr_in_flight, sizeof(nr_in_flight)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_aux_event_job_slot( struct kbase_tlstream *stream, const void *ctx, diff --git a/mali_kbase/tl/mali_kbase_tracepoints.h b/mali_kbase/tl/mali_kbase_tracepoints.h index 3cd94e2..844bdf4 100644 --- a/mali_kbase/tl/mali_kbase_tracepoints.h +++ b/mali_kbase/tl/mali_kbase_tracepoints.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,25 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ /* @@ -277,6 +296,17 @@ void __kbase_tlstream_aux_jit_stats( u32 allocs, u32 va_pages, u32 ph_pages); +void __kbase_tlstream_aux_tiler_heap_stats( + struct kbase_tlstream *stream, + u32 ctx_nr, + u64 heap_id, + u32 va_pages, + u32 ph_pages, + u32 max_chunks, + u32 chunk_size, + u32 chunk_count, + u32 target_in_flight, + u32 nr_in_flight); void __kbase_tlstream_aux_event_job_slot( struct kbase_tlstream *stream, const void *ctx, @@ -1500,6 +1530,42 @@ struct kbase_tlstream; } while (0) /** + * KBASE_TLSTREAM_AUX_TILER_HEAP_STATS - + * Tiler Heap statistics + * + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @heap_id: Unique id used to represent a heap under a context + * @va_pages: Number of virtual pages allocated in this bin + * @ph_pages: Number of physical pages allocated in this bin + * @max_chunks: The maximum number of chunks that the heap should be allowed to use + * @chunk_size: Size of each chunk in tiler heap, in bytes + * @chunk_count: The number of chunks currently allocated in the tiler heap + * @target_in_flight: Number of render-passes that the driver should attempt + * to keep in flight for which allocation of new chunks is allowed + * @nr_in_flight: Number of render-passes that are in flight + */ +#define KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( \ + kbdev, \ + ctx_nr, \ + heap_id, \ + va_pages, \ + ph_pages, \ + max_chunks, \ + chunk_size, \ + chunk_count, \ + target_in_flight, \ + nr_in_flight \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_tiler_heap_stats( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, heap_id, va_pages, ph_pages, max_chunks, chunk_size, chunk_count, target_in_flight, nr_in_flight); \ + } while (0) + +/** * KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT - * event on a given job slot * |